From 6db6cfa92b7232615fc76d201a309c84378ea1a1 Mon Sep 17 00:00:00 2001 From: Easwaran Raman Date: Tue, 22 Dec 2015 00:32:35 +0000 Subject: [PATCH] Determine callee's hotness and adjust threshold based on that. NFC. This uses the same criteria used in CFE's CodeGenPGO to identify hot and cold callees and uses values of inlinehint-threshold and inlinecold-threshold respectively as the thresholds for such callees. Differential Revision: http://reviews.llvm.org/D15245 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@256222 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Transforms/IPO/Inliner.cpp | 38 +++++++++++++++---- test/Transforms/Inline/inline-cold-callee.ll | 39 ++++++++++++++++++++ test/Transforms/Inline/inline-hot-callee.ll | 39 ++++++++++++++++++++ 3 files changed, 109 insertions(+), 7 deletions(-) create mode 100644 test/Transforms/Inline/inline-cold-callee.ll create mode 100644 test/Transforms/Inline/inline-hot-callee.ll diff --git a/lib/Transforms/IPO/Inliner.cpp b/lib/Transforms/IPO/Inliner.cpp index 14b1767721d..54505017136 100644 --- a/lib/Transforms/IPO/Inliner.cpp +++ b/lib/Transforms/IPO/Inliner.cpp @@ -288,18 +288,42 @@ unsigned Inliner::getInlineThreshold(CallSite CS) const { OptSizeThreshold < Threshold) Threshold = OptSizeThreshold; - // Listen to the inlinehint attribute when it would increase the threshold - // and the caller does not need to minimize its size. Function *Callee = CS.getCalledFunction(); - bool InlineHint = Callee && !Callee->isDeclaration() && - Callee->hasFnAttribute(Attribute::InlineHint); + if (!Callee || Callee->isDeclaration()) + return Threshold; + + // If profile information is available, use that to adjust threshold of hot + // and cold functions. + // FIXME: The heuristic used below for determining hotness and coldness are + // based on preliminary SPEC tuning and may not be optimal. Replace this with + // a well-tuned heuristic based on *callsite* hotness and not callee hotness. + uint64_t FunctionCount = 0, MaxFunctionCount = 0; + bool HasPGOCounts = false; + if (Callee->getEntryCount() && + Callee->getParent()->getMaximumFunctionCount()) { + HasPGOCounts = true; + FunctionCount = Callee->getEntryCount().getValue(); + MaxFunctionCount = + Callee->getParent()->getMaximumFunctionCount().getValue(); + } + + // Listen to the inlinehint attribute or profile based hotness information + // when it would increase the threshold and the caller does not need to + // minimize its size. + bool InlineHint = + Callee->hasFnAttribute(Attribute::InlineHint) || + (HasPGOCounts && + FunctionCount >= (uint64_t)(0.3 * (double)MaxFunctionCount)); if (InlineHint && HintThreshold > Threshold && !Caller->hasFnAttribute(Attribute::MinSize)) Threshold = HintThreshold; - // Listen to the cold attribute when it would decrease the threshold. - bool ColdCallee = Callee && !Callee->isDeclaration() && - Callee->hasFnAttribute(Attribute::Cold); + // Listen to the cold attribute or profile based coldness information + // when it would decrease the threshold. + bool ColdCallee = + Callee->hasFnAttribute(Attribute::Cold) || + (HasPGOCounts && + FunctionCount <= (uint64_t)(0.01 * (double)MaxFunctionCount)); // Command line argument for InlineLimit will override the default // ColdThreshold. If we have -inline-threshold but no -inlinecold-threshold, // do not use the default cold threshold even if it is smaller. diff --git a/test/Transforms/Inline/inline-cold-callee.ll b/test/Transforms/Inline/inline-cold-callee.ll new file mode 100644 index 00000000000..1fd9f105db5 --- /dev/null +++ b/test/Transforms/Inline/inline-cold-callee.ll @@ -0,0 +1,39 @@ +; RUN: opt < %s -inline -inlinecold-threshold=0 -S | FileCheck %s + +; This tests that a cold callee gets the (lower) inlinecold-threshold even without +; Cold hint and does not get inlined because the cost exceeds the inlinecold-threshold. +; A callee with identical body does gets inlined because cost fits within the +; inline-threshold + +define i32 @callee1(i32 %x) !prof !1 { + %x1 = add i32 %x, 1 + %x2 = add i32 %x1, 1 + %x3 = add i32 %x2, 1 + + ret i32 %x3 +} + +define i32 @callee2(i32 %x) !prof !2 { +; CHECK-LABEL: @callee2( + %x1 = add i32 %x, 1 + %x2 = add i32 %x1, 1 + %x3 = add i32 %x2, 1 + + ret i32 %x3 +} + +define i32 @caller2(i32 %y1) !prof !2 { +; CHECK-LABEL: @caller2( +; CHECK: call i32 @callee2 +; CHECK-NOT: call i32 @callee1 +; CHECK: ret i32 %x3.i + %y2 = call i32 @callee2(i32 %y1) + %y3 = call i32 @callee1(i32 %y2) + ret i32 %y3 +} + +!llvm.module.flags = !{!0} +!0 = !{i32 1, !"MaxFunctionCount", i32 1000} +!1 = !{!"function_entry_count", i64 100} +!2 = !{!"function_entry_count", i64 1} + diff --git a/test/Transforms/Inline/inline-hot-callee.ll b/test/Transforms/Inline/inline-hot-callee.ll new file mode 100644 index 00000000000..93ea9d43c78 --- /dev/null +++ b/test/Transforms/Inline/inline-hot-callee.ll @@ -0,0 +1,39 @@ +; RUN: opt < %s -inline -inline-threshold=0 -inlinehint-threshold=100 -S | FileCheck %s + +; This tests that a hot callee gets the (higher) inlinehint-threshold even without +; inline hints and gets inlined because the cost is less than inlinehint-threshold. +; A cold callee with identical body does not get inlined because cost exceeds the +; inline-threshold + +define i32 @callee1(i32 %x) !prof !1 { + %x1 = add i32 %x, 1 + %x2 = add i32 %x1, 1 + %x3 = add i32 %x2, 1 + + ret i32 %x3 +} + +define i32 @callee2(i32 %x) !prof !2 { +; CHECK-LABEL: @callee2( + %x1 = add i32 %x, 1 + %x2 = add i32 %x1, 1 + %x3 = add i32 %x2, 1 + + ret i32 %x3 +} + +define i32 @caller2(i32 %y1) !prof !2 { +; CHECK-LABEL: @caller2( +; CHECK: call i32 @callee2 +; CHECK-NOT: call i32 @callee1 +; CHECK: ret i32 %x3.i + %y2 = call i32 @callee2(i32 %y1) + %y3 = call i32 @callee1(i32 %y2) + ret i32 %y3 +} + +!llvm.module.flags = !{!0} +!0 = !{i32 1, !"MaxFunctionCount", i32 10} +!1 = !{!"function_entry_count", i64 10} +!2 = !{!"function_entry_count", i64 1} + -- 2.34.1