X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FTransforms%2FIPO%2FInliner.cpp;h=57f3e772b5698fa416e51f38a8dfb19ae63a6123;hb=98a42b2fc39f3fc46905fb98078cce4ce4e0a827;hp=5725db1b7451705fac702716e83d5cf02006ff63;hpb=c0aa67950ae0f6e9611240d8f0e3ac49dc8195c0;p=oota-llvm.git diff --git a/lib/Transforms/IPO/Inliner.cpp b/lib/Transforms/IPO/Inliner.cpp index 5725db1b745..57f3e772b56 100644 --- a/lib/Transforms/IPO/Inliner.cpp +++ b/lib/Transforms/IPO/Inliner.cpp @@ -29,7 +29,6 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/Statistic.h" -#include using namespace llvm; STATISTIC(NumInlined, "Number of functions inlined"); @@ -38,14 +37,22 @@ STATISTIC(NumDeleted, "Number of functions deleted because all callers found"); STATISTIC(NumMergedAllocas, "Number of allocas merged together"); static cl::opt -InlineLimit("inline-threshold", cl::Hidden, cl::init(200), cl::ZeroOrMore, - cl::desc("Control the amount of inlining to perform (default = 200)")); +InlineLimit("inline-threshold", cl::Hidden, cl::init(225), cl::ZeroOrMore, + cl::desc("Control the amount of inlining to perform (default = 225)")); -Inliner::Inliner(void *ID) +static cl::opt +HintThreshold("inlinehint-threshold", cl::Hidden, cl::init(325), + cl::desc("Threshold for inlining functions with inline hint")); + +// Threshold to use when optsize is specified (and there is no -inline-limit). +const int OptSizeThreshold = 75; + +Inliner::Inliner(char &ID) : CallGraphSCCPass(ID), InlineThreshold(InlineLimit) {} -Inliner::Inliner(void *ID, int Threshold) - : CallGraphSCCPass(ID), InlineThreshold(Threshold) {} +Inliner::Inliner(char &ID, int Threshold) + : CallGraphSCCPass(ID), InlineThreshold(InlineLimit.getNumOccurrences() > 0 ? + InlineLimit : Threshold) {} /// getAnalysisUsage - For this class, we declare that we require and preserve /// the call graph. If the derived class implements this method, it should @@ -66,16 +73,15 @@ InlinedArrayAllocasTy; /// available from other functions inlined into the caller. If we are able to /// inline this call site we attempt to reuse already available allocas or add /// any new allocas to the set if not possible. -static bool InlineCallIfPossible(CallSite CS, CallGraph &CG, - const TargetData *TD, - InlinedArrayAllocasTy &InlinedArrayAllocas) { +static bool InlineCallIfPossible(CallSite CS, InlineFunctionInfo &IFI, + InlinedArrayAllocasTy &InlinedArrayAllocas, + int InlineHistory) { Function *Callee = CS.getCalledFunction(); Function *Caller = CS.getCaller(); // Try to inline the function. Get the list of static allocas that were // inlined. - SmallVector StaticAllocas; - if (!InlineFunction(CS, &CG, TD, &StaticAllocas)) + if (!InlineFunction(CS, IFI)) return false; // If the inlined function had a higher stack protection level than the @@ -86,7 +92,6 @@ static bool InlineCallIfPossible(CallSite CS, CallGraph &CG, !Caller->hasFnAttr(Attribute::StackProtectReq)) Caller->addFnAttr(Attribute::StackProtect); - // Look at all of the allocas that we inlined through this call site. If we // have already inlined other allocas through other calls into this function, // then we know that they have disjoint lifetimes and that we can merge them. @@ -110,11 +115,26 @@ static bool InlineCallIfPossible(CallSite CS, CallGraph &CG, // SmallPtrSet UsedAllocas; + // When processing our SCC, check to see if CS was inlined from some other + // call site. For example, if we're processing "A" in this code: + // A() { B() } + // B() { x = alloca ... C() } + // C() { y = alloca ... } + // Assume that C was not inlined into B initially, and so we're processing A + // and decide to inline B into A. Doing this makes an alloca available for + // reuse and makes a callsite (C) available for inlining. When we process + // the C call site we don't want to do any alloca merging between X and Y + // because their scopes are not disjoint. We could make this smarter by + // keeping track of the inline history for each alloca in the + // InlinedArrayAllocas but this isn't likely to be a significant win. + if (InlineHistory != -1) // Only do merging for top-level call sites in SCC. + return true; + // Loop over all the allocas we have so far and see if they can be merged with // a previously inlined alloca. If not, remember that we had it. - for (unsigned AllocaNo = 0, e = StaticAllocas.size(); + for (unsigned AllocaNo = 0, e = IFI.StaticAllocas.size(); AllocaNo != e; ++AllocaNo) { - AllocaInst *AI = StaticAllocas[AllocaNo]; + AllocaInst *AI = IFI.StaticAllocas[AllocaNo]; // Don't bother trying to merge array allocations (they will usually be // canonicalized to be an allocation *of* an array), or allocations whose @@ -147,19 +167,21 @@ static bool InlineCallIfPossible(CallSite CS, CallGraph &CG, // Otherwise, we *can* reuse it, RAUW AI into AvailableAlloca and declare // success! - DEBUG(dbgs() << " ***MERGED ALLOCA: " << *AI); + DEBUG(dbgs() << " ***MERGED ALLOCA: " << *AI << "\n\t\tINTO: " + << *AvailableAlloca << '\n'); AI->replaceAllUsesWith(AvailableAlloca); AI->eraseFromParent(); MergedAwayAlloca = true; ++NumMergedAllocas; + IFI.StaticAllocas[AllocaNo] = 0; break; } // If we already nuked the alloca, we're done with it. if (MergedAwayAlloca) continue; - + // If we were unable to merge away the alloca either because there are no // allocas of the right type available or because we reused them all // already, remember that this alloca came from an inlined function and mark @@ -171,7 +193,26 @@ static bool InlineCallIfPossible(CallSite CS, CallGraph &CG, return true; } - + +unsigned Inliner::getInlineThreshold(CallSite CS) const { + int thres = InlineThreshold; + + // Listen to optsize when -inline-limit is not given. + Function *Caller = CS.getCaller(); + if (Caller && !Caller->isDeclaration() && + Caller->hasFnAttr(Attribute::OptimizeForSize) && + InlineLimit.getNumOccurrences() == 0) + thres = OptSizeThreshold; + + // Listen to inlinehint when it would increase the threshold. + Function *Callee = CS.getCalledFunction(); + if (HintThreshold > thres && Callee && !Callee->isDeclaration() && + Callee->hasFnAttr(Attribute::InlineHint)) + thres = HintThreshold; + + return thres; +} + /// shouldInline - Return true if the inliner should attempt to inline /// at the given CallSite. bool Inliner::shouldInline(CallSite CS) { @@ -190,17 +231,13 @@ bool Inliner::shouldInline(CallSite CS) { } int Cost = IC.getValue(); - int CurrentThreshold = InlineThreshold; Function *Caller = CS.getCaller(); - if (Caller && !Caller->isDeclaration() && - Caller->hasFnAttr(Attribute::OptimizeForSize) && - InlineLimit.getNumOccurrences() == 0 && - InlineThreshold != 50) - CurrentThreshold = 50; - + int CurrentThreshold = getInlineThreshold(CS); float FudgeFactor = getInlineFudgeFactor(CS); - if (Cost >= (int)(CurrentThreshold * FudgeFactor)) { + int AdjThreshold = (int)(CurrentThreshold * FudgeFactor); + if (Cost >= AdjThreshold) { DEBUG(dbgs() << " NOT Inlining: cost=" << Cost + << ", thres=" << AdjThreshold << ", Call: " << *CS.getInstruction() << "\n"); return false; } @@ -214,43 +251,42 @@ bool Inliner::shouldInline(CallSite CS) { if (Caller->hasLocalLinkage()) { int TotalSecondaryCost = 0; bool outerCallsFound = false; - bool allOuterCallsWillBeInlined = true; - bool someOuterCallWouldNotBeInlined = false; + // This bool tracks what happens if we do NOT inline C into B. + bool callerWillBeRemoved = true; + // This bool tracks what happens if we DO inline C into B. + bool inliningPreventsSomeOuterInline = false; for (Value::use_iterator I = Caller->use_begin(), E =Caller->use_end(); I != E; ++I) { - CallSite CS2 = CallSite::get(*I); + CallSite CS2(*I); // If this isn't a call to Caller (it could be some other sort - // of reference) skip it. - if (CS2.getInstruction() == 0 || CS2.getCalledFunction() != Caller) + // of reference) skip it. Such references will prevent the caller + // from being removed. + if (!CS2 || CS2.getCalledFunction() != Caller) { + callerWillBeRemoved = false; continue; + } InlineCost IC2 = getInlineCost(CS2); if (IC2.isNever()) - allOuterCallsWillBeInlined = false; + callerWillBeRemoved = false; if (IC2.isAlways() || IC2.isNever()) continue; outerCallsFound = true; int Cost2 = IC2.getValue(); - int CurrentThreshold2 = InlineThreshold; - Function *Caller2 = CS2.getCaller(); - if (Caller2 && !Caller2->isDeclaration() && - Caller2->hasFnAttr(Attribute::OptimizeForSize) && - InlineThreshold != 50) - CurrentThreshold2 = 50; - + int CurrentThreshold2 = getInlineThreshold(CS2); float FudgeFactor2 = getInlineFudgeFactor(CS2); if (Cost2 >= (int)(CurrentThreshold2 * FudgeFactor2)) - allOuterCallsWillBeInlined = false; + callerWillBeRemoved = false; // See if we have this case. We subtract off the penalty // for the call instruction, which we would be deleting. if (Cost2 < (int)(CurrentThreshold2 * FudgeFactor2) && Cost2 + Cost - (InlineConstants::CallPenalty + 1) >= (int)(CurrentThreshold2 * FudgeFactor2)) { - someOuterCallWouldNotBeInlined = true; + inliningPreventsSomeOuterInline = true; TotalSecondaryCost += Cost2; } } @@ -258,10 +294,10 @@ bool Inliner::shouldInline(CallSite CS) { // one is set very low by getInlineCost, in anticipation that Caller will // be removed entirely. We did not account for this above unless there // is only one caller of Caller. - if (allOuterCallsWillBeInlined && Caller->use_begin() != Caller->use_end()) + if (callerWillBeRemoved && Caller->use_begin() != Caller->use_end()) TotalSecondaryCost += InlineConstants::LastCallToStaticBonus; - if (outerCallsFound && someOuterCallWouldNotBeInlined && + if (outerCallsFound && inliningPreventsSomeOuterInline && TotalSecondaryCost < Cost) { DEBUG(dbgs() << " NOT Inlining: " << *CS.getInstruction() << " Cost = " << Cost << @@ -271,18 +307,34 @@ bool Inliner::shouldInline(CallSite CS) { } DEBUG(dbgs() << " Inlining: cost=" << Cost + << ", thres=" << AdjThreshold << ", Call: " << *CS.getInstruction() << '\n'); return true; } -bool Inliner::runOnSCC(std::vector &SCC) { +/// InlineHistoryIncludes - Return true if the specified inline history ID +/// indicates an inline history that includes the specified function. +static bool InlineHistoryIncludes(Function *F, int InlineHistoryID, + const SmallVectorImpl > &InlineHistory) { + while (InlineHistoryID != -1) { + assert(unsigned(InlineHistoryID) < InlineHistory.size() && + "Invalid inline history ID"); + if (InlineHistory[InlineHistoryID].first == F) + return true; + InlineHistoryID = InlineHistory[InlineHistoryID].second; + } + return false; +} + + +bool Inliner::runOnSCC(CallGraphSCC &SCC) { CallGraph &CG = getAnalysis(); const TargetData *TD = getAnalysisIfAvailable(); SmallPtrSet SCCFunctions; DEBUG(dbgs() << "Inliner visiting SCC:"); - for (unsigned i = 0, e = SCC.size(); i != e; ++i) { - Function *F = SCC[i]->getFunction(); + for (CallGraphSCC::iterator I = SCC.begin(), E = SCC.end(); I != E; ++I) { + Function *F = (*I)->getFunction(); if (F) SCCFunctions.insert(F); DEBUG(dbgs() << " " << (F ? F->getName() : "INDIRECTNODE")); } @@ -290,18 +342,24 @@ bool Inliner::runOnSCC(std::vector &SCC) { // Scan through and identify all call sites ahead of time so that we only // inline call sites in the original functions, not call sites that result // from inlining other functions. - SmallVector CallSites; - - for (unsigned i = 0, e = SCC.size(); i != e; ++i) { - Function *F = SCC[i]->getFunction(); + SmallVector, 16> CallSites; + + // When inlining a callee produces new call sites, we want to keep track of + // the fact that they were inlined from the callee. This allows us to avoid + // infinite inlining in some obscure cases. To represent this, we use an + // index into the InlineHistory vector. + SmallVector, 8> InlineHistory; + + for (CallGraphSCC::iterator I = SCC.begin(), E = SCC.end(); I != E; ++I) { + Function *F = (*I)->getFunction(); if (!F) continue; for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) { - CallSite CS = CallSite::get(I); + CallSite CS(cast(I)); // If this isn't a call, or it is a call to an intrinsic, it can // never be inlined. - if (CS.getInstruction() == 0 || isa(I)) + if (!CS || isa(I)) continue; // If this is a direct call to an external function, we can never inline @@ -310,22 +368,27 @@ bool Inliner::runOnSCC(std::vector &SCC) { if (CS.getCalledFunction() && CS.getCalledFunction()->isDeclaration()) continue; - CallSites.push_back(CS); + CallSites.push_back(std::make_pair(CS, -1)); } } DEBUG(dbgs() << ": " << CallSites.size() << " call sites.\n"); + // If there are no calls in this function, exit early. + if (CallSites.empty()) + return false; + // Now that we have all of the call sites, move the ones to functions in the // current SCC to the end of the list. unsigned FirstCallInSCC = CallSites.size(); for (unsigned i = 0; i < FirstCallInSCC; ++i) - if (Function *F = CallSites[i].getCalledFunction()) + if (Function *F = CallSites[i].first.getCalledFunction()) if (SCCFunctions.count(F)) std::swap(CallSites[i--], CallSites[--FirstCallInSCC]); InlinedArrayAllocasTy InlinedArrayAllocas; + InlineFunctionInfo InlineInfo(&CG, TD); // Now that we have all of the call sites, loop over them and inline them if // it looks profitable to do so. @@ -336,7 +399,7 @@ bool Inliner::runOnSCC(std::vector &SCC) { // Iterate over the outer loop because inlining functions can cause indirect // calls to become direct calls. for (unsigned CSi = 0; CSi != CallSites.size(); ++CSi) { - CallSite CS = CallSites[CSi]; + CallSite CS = CallSites[CSi].first; Function *Caller = CS.getCaller(); Function *Callee = CS.getCalledFunction(); @@ -352,19 +415,51 @@ bool Inliner::runOnSCC(std::vector &SCC) { CG[Caller]->removeCallEdgeFor(CS); CS.getInstruction()->eraseFromParent(); ++NumCallsDeleted; + // Update the cached cost info with the missing call + growCachedCostInfo(Caller, NULL); } else { // We can only inline direct calls to non-declarations. if (Callee == 0 || Callee->isDeclaration()) continue; + // If this call site was obtained by inlining another function, verify + // that the include path for the function did not include the callee + // itself. If so, we'd be recursively inlining the same function, + // which would provide the same callsites, which would cause us to + // infinitely inline. + int InlineHistoryID = CallSites[CSi].second; + if (InlineHistoryID != -1 && + InlineHistoryIncludes(Callee, InlineHistoryID, InlineHistory)) + continue; + + // If the policy determines that we should inline this function, // try to do so. if (!shouldInline(CS)) continue; - // Attempt to inline the function... - if (!InlineCallIfPossible(CS, CG, TD, InlinedArrayAllocas)) + // Attempt to inline the function. + if (!InlineCallIfPossible(CS, InlineInfo, InlinedArrayAllocas, + InlineHistoryID)) continue; ++NumInlined; + + // If inlining this function gave us any new call sites, throw them + // onto our worklist to process. They are useful inline candidates. + if (!InlineInfo.InlinedCalls.empty()) { + // Create a new inline history entry for this, so that we remember + // that these new callsites came about due to inlining Callee. + int NewHistoryID = InlineHistory.size(); + InlineHistory.push_back(std::make_pair(Callee, InlineHistoryID)); + + for (unsigned i = 0, e = InlineInfo.InlinedCalls.size(); + i != e; ++i) { + Value *Ptr = InlineInfo.InlinedCalls[i]; + CallSites.push_back(std::make_pair(CallSite(Ptr), NewHistoryID)); + } + } + + // Update the cached cost info with the inlined call. + growCachedCostInfo(Caller, Callee); } // If we inlined or deleted the last possible call site to the function, @@ -390,18 +485,13 @@ bool Inliner::runOnSCC(std::vector &SCC) { delete CG.removeFunctionFromModule(CalleeNode); ++NumDeleted; } - - // Remove any cached cost info for this caller, as inlining the - // callee has increased the size of the caller (which may be the - // same as the callee). - resetCachedCostInfo(Caller); // Remove this call site from the list. If possible, use // swap/pop_back for efficiency, but do not use it if doing so would // move a call site to a function in this SCC before the // 'FirstCallInSCC' barrier. - if (SCC.size() == 1) { - std::swap(CallSites[CSi], CallSites.back()); + if (SCC.isSingular()) { + CallSites[CSi] = CallSites.back(); CallSites.pop_back(); } else { CallSites.erase(CallSites.begin()+CSi);