1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
11 // It also builds the data structures and initialization code needed for
12 // updating execution counts and emitting the profile at runtime.
14 //===----------------------------------------------------------------------===//
16 #include "llvm/ADT/Triple.h"
17 #include "llvm/IR/IRBuilder.h"
18 #include "llvm/IR/IntrinsicInst.h"
19 #include "llvm/IR/Module.h"
20 #include "llvm/ProfileData/InstrProf.h"
21 #include "llvm/Transforms/Instrumentation.h"
22 #include "llvm/Transforms/Utils/ModuleUtils.h"
26 #define DEBUG_TYPE "instrprof"
30 class InstrProfiling : public ModulePass {
34 InstrProfiling() : ModulePass(ID) {}
36 InstrProfiling(const InstrProfOptions &Options)
37 : ModulePass(ID), Options(Options) {}
39 const char *getPassName() const override {
40 return "Frontend instrumentation-based coverage lowering";
43 bool runOnModule(Module &M) override;
45 void getAnalysisUsage(AnalysisUsage &AU) const override {
50 InstrProfOptions Options;
52 typedef struct PerFunctionProfileData {
53 uint32_t NumValueSites[IPVK_Last+1];
54 GlobalVariable* RegionCounters;
55 GlobalVariable* DataVar;
56 PerFunctionProfileData() : RegionCounters(nullptr), DataVar(nullptr) {
57 memset(NumValueSites, 0, sizeof(uint32_t) * (IPVK_Last+1));
59 } PerFunctionProfileData;
60 DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
61 std::vector<Value *> UsedVars;
63 bool isMachO() const {
64 return Triple(M->getTargetTriple()).isOSBinFormatMachO();
67 /// Get the section name for the counter variables.
68 StringRef getCountersSection() const {
69 return getInstrProfCountersSectionName(isMachO());
72 /// Get the section name for the name variables.
73 StringRef getNameSection() const {
74 return getInstrProfNameSectionName(isMachO());
77 /// Get the section name for the profile data variables.
78 StringRef getDataSection() const {
79 return getInstrProfDataSectionName(isMachO());
82 /// Get the section name for the coverage mapping data.
83 StringRef getCoverageSection() const {
84 return getInstrProfCoverageSectionName(isMachO());
87 /// Count the number of instrumented value sites for the function.
88 void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
90 /// Replace instrprof_value_profile with a call to runtime library.
91 void lowerValueProfileInst(InstrProfValueProfileInst *Ins);
93 /// Replace instrprof_increment with an increment of the appropriate value.
94 void lowerIncrement(InstrProfIncrementInst *Inc);
96 /// Force emitting of name vars for unused functions.
97 void lowerCoverageData(GlobalVariable *CoverageNamesVar);
99 /// Get the region counters for an increment, creating them if necessary.
101 /// If the counter array doesn't yet exist, the profile data variables
102 /// referring to them will also be created.
103 GlobalVariable *getOrCreateRegionCounters(InstrProfIncrementInst *Inc);
105 /// Emit runtime registration functions for each profile data variable.
106 void emitRegistration();
108 /// Emit the necessary plumbing to pull in the runtime initialization.
109 void emitRuntimeHook();
111 /// Add uses of our data variables and runtime hook.
114 /// Create a static initializer for our data, on platforms that need it,
115 /// and for any profile output file that was specified.
116 void emitInitialization();
119 } // anonymous namespace
121 char InstrProfiling::ID = 0;
122 INITIALIZE_PASS(InstrProfiling, "instrprof",
123 "Frontend instrumentation-based coverage lowering.", false,
126 ModulePass *llvm::createInstrProfilingPass(const InstrProfOptions &Options) {
127 return new InstrProfiling(Options);
130 bool InstrProfiling::runOnModule(Module &M) {
131 bool MadeChange = false;
134 ProfileDataMap.clear();
137 // We did not know how many value sites there would be inside
138 // the instrumented function. This is counting the number of instrumented
139 // target value sites to enter it as field in the profile data variable.
140 for (Function &F : M)
141 for (BasicBlock &BB : F)
142 for (auto I = BB.begin(), E = BB.end(); I != E;)
143 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I++))
144 computeNumValueSiteCounts(Ind);
146 for (Function &F : M)
147 for (BasicBlock &BB : F)
148 for (auto I = BB.begin(), E = BB.end(); I != E;) {
150 if (auto *Inc = dyn_cast<InstrProfIncrementInst>(Instr)) {
153 } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
154 lowerValueProfileInst(Ind);
159 if (GlobalVariable *CoverageNamesVar =
160 M.getNamedGlobal(getCoverageNamesVarName())) {
161 lowerCoverageData(CoverageNamesVar);
171 emitInitialization();
175 static Constant *getOrInsertValueProfilingCall(Module &M) {
176 LLVMContext &Ctx = M.getContext();
177 auto *ReturnTy = Type::getVoidTy(M.getContext());
178 Type *ParamTypes[] = {
179 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
180 #include "llvm/ProfileData/InstrProfData.inc"
182 auto *ValueProfilingCallTy =
183 FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
184 return M.getOrInsertFunction(getInstrProfValueProfFuncName(),
185 ValueProfilingCallTy);
188 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
190 GlobalVariable *Name = Ind->getName();
191 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
192 uint64_t Index = Ind->getIndex()->getZExtValue();
193 auto It = ProfileDataMap.find(Name);
194 if (It == ProfileDataMap.end()) {
195 PerFunctionProfileData PD;
196 PD.NumValueSites[ValueKind] = Index + 1;
197 ProfileDataMap[Name] = PD;
198 } else if (It->second.NumValueSites[ValueKind] <= Index)
199 It->second.NumValueSites[ValueKind] = Index + 1;
202 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
204 GlobalVariable *Name = Ind->getName();
205 auto It = ProfileDataMap.find(Name);
206 assert(It != ProfileDataMap.end() && It->second.DataVar &&
207 "value profiling detected in function with no counter incerement");
209 GlobalVariable *DataVar = It->second.DataVar;
210 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
211 uint64_t Index = Ind->getIndex()->getZExtValue();
212 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
213 Index += It->second.NumValueSites[Kind];
215 IRBuilder<> Builder(Ind);
216 Value* Args[3] = {Ind->getTargetValue(),
217 Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
218 Builder.getInt32(Index)};
219 Ind->replaceAllUsesWith(
220 Builder.CreateCall(getOrInsertValueProfilingCall(*M), Args));
221 Ind->eraseFromParent();
224 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
225 GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
227 IRBuilder<> Builder(Inc);
228 uint64_t Index = Inc->getIndex()->getZExtValue();
229 Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
230 Value *Count = Builder.CreateLoad(Addr, "pgocount");
231 Count = Builder.CreateAdd(Count, Builder.getInt64(1));
232 Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
233 Inc->eraseFromParent();
236 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
238 ConstantArray *Names =
239 cast<ConstantArray>(CoverageNamesVar->getInitializer());
240 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
241 Constant *NC = Names->getOperand(I);
242 Value *V = NC->stripPointerCasts();
243 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
244 GlobalVariable *Name = cast<GlobalVariable>(V);
246 // Move the name variable to the right section.
247 Name->setSection(getNameSection());
248 Name->setAlignment(1);
252 /// Get the name of a profiling variable for a particular function.
253 static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
254 StringRef NamePrefix = getInstrProfNameVarPrefix();
255 StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
256 return (Prefix + Name).str();
259 static inline bool shouldRecordFunctionAddr(Function *F) {
261 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
262 !F->hasAvailableExternallyLinkage())
264 // Check uses of this function for other than direct calls or invokes to it.
265 return F->hasAddressTaken();
268 static inline Comdat *getOrCreateProfileComdat(Module &M,
269 InstrProfIncrementInst *Inc) {
270 // COFF format requires a COMDAT section to have a key symbol with the same
271 // name. The linker targeting COFF also requires that the COMDAT section
272 // a section is associated to must precede the associating section. For this
273 // reason, we must choose the name var's name as the name of the comdat.
274 StringRef ComdatPrefix = (Triple(M.getTargetTriple()).isOSBinFormatCOFF()
275 ? getInstrProfNameVarPrefix()
276 : getInstrProfComdatPrefix());
277 return M.getOrInsertComdat(StringRef(getVarName(Inc, ComdatPrefix)));
281 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
282 GlobalVariable *NamePtr = Inc->getName();
283 auto It = ProfileDataMap.find(NamePtr);
284 PerFunctionProfileData PD;
285 if (It != ProfileDataMap.end()) {
286 if (It->second.RegionCounters)
287 return It->second.RegionCounters;
291 // Move the name variable to the right section. Place them in a COMDAT group
292 // if the associated function is a COMDAT. This will make sure that
293 // only one copy of counters of the COMDAT function will be emitted after
295 Function *Fn = Inc->getParent()->getParent();
296 Comdat *ProfileVarsComdat = nullptr;
298 ProfileVarsComdat = getOrCreateProfileComdat(*M, Inc);
299 NamePtr->setSection(getNameSection());
300 NamePtr->setAlignment(1);
301 NamePtr->setComdat(ProfileVarsComdat);
303 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
304 LLVMContext &Ctx = M->getContext();
305 ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
307 // Create the counters variable.
309 new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(),
310 Constant::getNullValue(CounterTy),
311 getVarName(Inc, getInstrProfCountersVarPrefix()));
312 CounterPtr->setVisibility(NamePtr->getVisibility());
313 CounterPtr->setSection(getCountersSection());
314 CounterPtr->setAlignment(8);
315 CounterPtr->setComdat(ProfileVarsComdat);
317 // Create data variable.
318 auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
319 auto *Int16Ty = Type::getInt16Ty(Ctx);
320 auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last+1);
321 Type *DataTypes[] = {
322 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
323 #include "llvm/ProfileData/InstrProfData.inc"
325 auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
327 Constant *FunctionAddr = shouldRecordFunctionAddr(Fn) ?
328 ConstantExpr::getBitCast(Fn, Int8PtrTy) :
329 ConstantPointerNull::get(Int8PtrTy);
331 Constant *Int16ArrayVals[IPVK_Last+1];
332 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
333 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
335 Constant *DataVals[] = {
336 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
337 #include "llvm/ProfileData/InstrProfData.inc"
339 auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(),
340 ConstantStruct::get(DataTy, DataVals),
341 getVarName(Inc, getInstrProfDataVarPrefix()));
342 Data->setVisibility(NamePtr->getVisibility());
343 Data->setSection(getDataSection());
344 Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
345 Data->setComdat(ProfileVarsComdat);
347 PD.RegionCounters = CounterPtr;
349 ProfileDataMap[NamePtr] = PD;
351 // Mark the data variable as used so that it isn't stripped out.
352 UsedVars.push_back(Data);
357 void InstrProfiling::emitRegistration() {
358 // Don't do this for Darwin. compiler-rt uses linker magic.
359 if (Triple(M->getTargetTriple()).isOSDarwin())
362 // Use linker script magic to get data/cnts/name start/end.
363 if (Triple(M->getTargetTriple()).isOSLinux() ||
364 Triple(M->getTargetTriple()).isOSFreeBSD())
367 // Construct the function.
368 auto *VoidTy = Type::getVoidTy(M->getContext());
369 auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
370 auto *RegisterFTy = FunctionType::get(VoidTy, false);
371 auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
372 getInstrProfRegFuncsName(), M);
373 RegisterF->setUnnamedAddr(true);
374 if (Options.NoRedZone) RegisterF->addFnAttr(Attribute::NoRedZone);
376 auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
377 auto *RuntimeRegisterF =
378 Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
379 getInstrProfRegFuncName(), M);
381 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
382 for (Value *Data : UsedVars)
383 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
387 void InstrProfiling::emitRuntimeHook() {
389 // We expect the linker to be invoked with -u<hook_var> flag for linux,
390 // for which case there is no need to emit the user function.
391 if (Triple(M->getTargetTriple()).isOSLinux())
394 // If the module's provided its own runtime, we don't need to do anything.
395 if (M->getGlobalVariable(getInstrProfRuntimeHookVarName())) return;
397 // Declare an external variable that will pull in the runtime initialization.
398 auto *Int32Ty = Type::getInt32Ty(M->getContext());
400 new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
401 nullptr, getInstrProfRuntimeHookVarName());
403 // Make a function that uses it.
404 auto *User = Function::Create(FunctionType::get(Int32Ty, false),
405 GlobalValue::LinkOnceODRLinkage,
406 getInstrProfRuntimeHookVarUseFuncName(), M);
407 User->addFnAttr(Attribute::NoInline);
408 if (Options.NoRedZone) User->addFnAttr(Attribute::NoRedZone);
409 User->setVisibility(GlobalValue::HiddenVisibility);
411 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
412 auto *Load = IRB.CreateLoad(Var);
415 // Mark the user variable as used so that it isn't stripped out.
416 UsedVars.push_back(User);
419 void InstrProfiling::emitUses() {
420 if (UsedVars.empty())
423 GlobalVariable *LLVMUsed = M->getGlobalVariable("llvm.used");
424 std::vector<Constant *> MergedVars;
426 // Collect the existing members of llvm.used.
427 ConstantArray *Inits = cast<ConstantArray>(LLVMUsed->getInitializer());
428 for (unsigned I = 0, E = Inits->getNumOperands(); I != E; ++I)
429 MergedVars.push_back(Inits->getOperand(I));
430 LLVMUsed->eraseFromParent();
433 Type *i8PTy = Type::getInt8PtrTy(M->getContext());
434 // Add uses for our data.
435 for (auto *Value : UsedVars)
436 MergedVars.push_back(
437 ConstantExpr::getBitCast(cast<Constant>(Value), i8PTy));
439 // Recreate llvm.used.
440 ArrayType *ATy = ArrayType::get(i8PTy, MergedVars.size());
442 new GlobalVariable(*M, ATy, false, GlobalValue::AppendingLinkage,
443 ConstantArray::get(ATy, MergedVars), "llvm.used");
444 LLVMUsed->setSection("llvm.metadata");
447 void InstrProfiling::emitInitialization() {
448 std::string InstrProfileOutput = Options.InstrProfileOutput;
450 Constant *RegisterF = M->getFunction(getInstrProfRegFuncsName());
451 if (!RegisterF && InstrProfileOutput.empty()) return;
453 // Create the initialization function.
454 auto *VoidTy = Type::getVoidTy(M->getContext());
455 auto *F = Function::Create(FunctionType::get(VoidTy, false),
456 GlobalValue::InternalLinkage,
457 getInstrProfInitFuncName(), M);
458 F->setUnnamedAddr(true);
459 F->addFnAttr(Attribute::NoInline);
460 if (Options.NoRedZone) F->addFnAttr(Attribute::NoRedZone);
462 // Add the basic block and the necessary calls.
463 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
465 IRB.CreateCall(RegisterF, {});
466 if (!InstrProfileOutput.empty()) {
467 auto *Int8PtrTy = Type::getInt8PtrTy(M->getContext());
468 auto *SetNameTy = FunctionType::get(VoidTy, Int8PtrTy, false);
469 auto *SetNameF = Function::Create(SetNameTy, GlobalValue::ExternalLinkage,
470 getInstrProfFileOverriderFuncName(), M);
472 // Create variable for profile name.
473 Constant *ProfileNameConst =
474 ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true);
475 GlobalVariable *ProfileName =
476 new GlobalVariable(*M, ProfileNameConst->getType(), true,
477 GlobalValue::PrivateLinkage, ProfileNameConst);
479 IRB.CreateCall(SetNameF, IRB.CreatePointerCast(ProfileName, Int8PtrTy));
483 appendToGlobalCtors(*M, F, 0);