1 //===- LowerExpectIntrinsic.cpp - Lower expect intrinsic ------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass lowers the 'expect' intrinsic to LLVM metadata.
12 //===----------------------------------------------------------------------===//
14 #define DEBUG_TYPE "lower-expect-intrinsic"
15 #include "llvm/Constants.h"
16 #include "llvm/Function.h"
17 #include "llvm/BasicBlock.h"
18 #include "llvm/LLVMContext.h"
19 #include "llvm/Instructions.h"
20 #include "llvm/Intrinsics.h"
21 #include "llvm/Metadata.h"
22 #include "llvm/Pass.h"
23 #include "llvm/Transforms/Scalar.h"
24 #include "llvm/Support/CommandLine.h"
25 #include "llvm/Support/Debug.h"
26 #include "llvm/ADT/Statistic.h"
31 STATISTIC(IfHandled, "Number of 'expect' intrinsic intructions handled");
33 static cl::opt<uint32_t>
34 LikelyBranchWeight("likely-branch-weight", cl::Hidden, cl::init(64),
35 cl::desc("Weight of the branch likely to be taken (default = 64)"));
36 static cl::opt<uint32_t>
37 UnlikelyBranchWeight("unlikely-branch-weight", cl::Hidden, cl::init(4),
38 cl::desc("Weight of the branch unlikely to be taken (default = 4)"));
42 class LowerExpectIntrinsic : public FunctionPass {
44 bool HandleSwitchExpect(SwitchInst *SI);
46 bool HandleIfExpect(BranchInst *BI);
50 LowerExpectIntrinsic() : FunctionPass(ID) {
51 initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry());
54 bool runOnFunction(Function &F);
59 bool LowerExpectIntrinsic::HandleSwitchExpect(SwitchInst *SI) {
60 CallInst *CI = dyn_cast<CallInst>(SI->getCondition());
64 Function *Fn = CI->getCalledFunction();
65 if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
68 Value *ArgValue = CI->getArgOperand(0);
69 ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
73 LLVMContext &Context = CI->getContext();
74 Type *Int32Ty = Type::getInt32Ty(Context);
76 SwitchInst::CaseIt Case = SI->findCaseValue(ExpectedValue);
77 std::vector<Value *> Vec;
78 unsigned n = SI->getNumCases();
79 Vec.resize(n + 1 + 1); // +1 for MDString and +1 for default case
81 Vec[0] = MDString::get(Context, "branch_weights");
82 Vec[1] = ConstantInt::get(Int32Ty, Case == SI->caseDefault() ?
83 LikelyBranchWeight : UnlikelyBranchWeight);
84 for (unsigned i = 0; i < n; ++i) {
85 Vec[i + 1 + 1] = ConstantInt::get(Int32Ty, i == Case.getCaseIndex() ?
86 LikelyBranchWeight : UnlikelyBranchWeight);
89 MDNode *WeightsNode = llvm::MDNode::get(Context, Vec);
90 SI->setMetadata(LLVMContext::MD_prof, WeightsNode);
92 SI->setCondition(ArgValue);
97 bool LowerExpectIntrinsic::HandleIfExpect(BranchInst *BI) {
98 if (BI->isUnconditional())
101 // Handle non-optimized IR code like:
102 // %expval = call i64 @llvm.expect.i64.i64(i64 %conv1, i64 1)
103 // %tobool = icmp ne i64 %expval, 0
104 // br i1 %tobool, label %if.then, label %if.end
106 ICmpInst *CmpI = dyn_cast<ICmpInst>(BI->getCondition());
107 if (!CmpI || CmpI->getPredicate() != CmpInst::ICMP_NE)
110 CallInst *CI = dyn_cast<CallInst>(CmpI->getOperand(0));
114 Function *Fn = CI->getCalledFunction();
115 if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
118 Value *ArgValue = CI->getArgOperand(0);
119 ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
123 LLVMContext &Context = CI->getContext();
124 Type *Int32Ty = Type::getInt32Ty(Context);
125 bool Likely = ExpectedValue->isOne();
127 // If expect value is equal to 1 it means that we are more likely to take
128 // branch 0, in other case more likely is branch 1.
130 MDString::get(Context, "branch_weights"),
131 ConstantInt::get(Int32Ty, Likely ? LikelyBranchWeight : UnlikelyBranchWeight),
132 ConstantInt::get(Int32Ty, Likely ? UnlikelyBranchWeight : LikelyBranchWeight)
135 MDNode *WeightsNode = MDNode::get(Context, Ops);
136 BI->setMetadata(LLVMContext::MD_prof, WeightsNode);
138 CmpI->setOperand(0, ArgValue);
143 bool LowerExpectIntrinsic::runOnFunction(Function &F) {
144 for (Function::iterator I = F.begin(), E = F.end(); I != E;) {
145 BasicBlock *BB = I++;
147 // Create "block_weights" metadata.
148 if (BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator())) {
149 if (HandleIfExpect(BI))
151 } else if (SwitchInst *SI = dyn_cast<SwitchInst>(BB->getTerminator())) {
152 if (HandleSwitchExpect(SI))
156 // remove llvm.expect intrinsics.
157 for (BasicBlock::iterator BI = BB->begin(), BE = BB->end();
159 CallInst *CI = dyn_cast<CallInst>(BI++);
163 Function *Fn = CI->getCalledFunction();
164 if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) {
165 Value *Exp = CI->getArgOperand(0);
166 CI->replaceAllUsesWith(Exp);
167 CI->eraseFromParent();
176 char LowerExpectIntrinsic::ID = 0;
177 INITIALIZE_PASS(LowerExpectIntrinsic, "lower-expect", "Lower 'expect' "
178 "Intrinsics", false, false)
180 FunctionPass *llvm::createLowerExpectIntrinsicPass() {
181 return new LowerExpectIntrinsic();