1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass lowers instrprof_increment intrinsics emitted by a frontend for
11 // profiling. It also builds the data structures and initialization code needed
12 // for updating execution counts and emitting the profile at runtime.
14 //===----------------------------------------------------------------------===//
16 #include "llvm/Transforms/Instrumentation.h"
18 #include "llvm/ADT/Triple.h"
19 #include "llvm/IR/IRBuilder.h"
20 #include "llvm/IR/IntrinsicInst.h"
21 #include "llvm/IR/Module.h"
22 #include "llvm/Transforms/Utils/ModuleUtils.h"
26 #define DEBUG_TYPE "instrprof"
30 class InstrProfiling : public ModulePass {
34 InstrProfiling() : ModulePass(ID) {}
36 InstrProfiling(const InstrProfOptions &Options)
37 : ModulePass(ID), Options(Options) {}
39 const char *getPassName() const override {
40 return "Frontend instrumentation-based coverage lowering";
43 bool runOnModule(Module &M) override;
45 void getAnalysisUsage(AnalysisUsage &AU) const override {
50 InstrProfOptions Options;
52 DenseMap<GlobalVariable *, GlobalVariable *> RegionCounters;
53 std::vector<Value *> UsedVars;
55 bool isMachO() const {
56 return Triple(M->getTargetTriple()).isOSBinFormatMachO();
59 /// Get the section name for the counter variables.
60 StringRef getCountersSection() const {
61 return isMachO() ? "__DATA,__llvm_prf_cnts" : "__llvm_prf_cnts";
64 /// Get the section name for the name variables.
65 StringRef getNameSection() const {
66 return isMachO() ? "__DATA,__llvm_prf_names" : "__llvm_prf_names";
69 /// Get the section name for the profile data variables.
70 StringRef getDataSection() const {
71 return isMachO() ? "__DATA,__llvm_prf_data" : "__llvm_prf_data";
74 /// Get the section name for the coverage mapping data.
75 StringRef getCoverageSection() const {
76 return isMachO() ? "__DATA,__llvm_covmap" : "__llvm_covmap";
79 /// Replace instrprof_increment with an increment of the appropriate value.
80 void lowerIncrement(InstrProfIncrementInst *Inc);
82 /// Set up the section and uses for coverage data and its references.
83 void lowerCoverageData(GlobalVariable *CoverageData);
85 /// Get the region counters for an increment, creating them if necessary.
87 /// If the counter array doesn't yet exist, the profile data variables
88 /// referring to them will also be created.
89 GlobalVariable *getOrCreateRegionCounters(InstrProfIncrementInst *Inc);
91 /// Emit runtime registration functions for each profile data variable.
92 void emitRegistration();
94 /// Emit the necessary plumbing to pull in the runtime initialization.
95 void emitRuntimeHook();
97 /// Add uses of our data variables and runtime hook.
100 /// Create a static initializer for our data, on platforms that need it,
101 /// and for any profile output file that was specified.
102 void emitInitialization();
105 } // anonymous namespace
107 char InstrProfiling::ID = 0;
108 INITIALIZE_PASS(InstrProfiling, "instrprof",
109 "Frontend instrumentation-based coverage lowering.", false,
112 ModulePass *llvm::createInstrProfilingPass(const InstrProfOptions &Options) {
113 return new InstrProfiling(Options);
116 bool InstrProfiling::runOnModule(Module &M) {
117 bool MadeChange = false;
120 RegionCounters.clear();
123 for (Function &F : M)
124 for (BasicBlock &BB : F)
125 for (auto I = BB.begin(), E = BB.end(); I != E;)
126 if (auto *Inc = dyn_cast<InstrProfIncrementInst>(I++)) {
130 if (GlobalVariable *Coverage = M.getNamedGlobal("__llvm_coverage_mapping")) {
131 lowerCoverageData(Coverage);
140 emitInitialization();
144 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
145 GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
147 IRBuilder<> Builder(Inc);
148 uint64_t Index = Inc->getIndex()->getZExtValue();
149 Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
150 Value *Count = Builder.CreateLoad(Addr, "pgocount");
151 Count = Builder.CreateAdd(Count, Builder.getInt64(1));
152 Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
153 Inc->eraseFromParent();
156 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageData) {
157 CoverageData->setSection(getCoverageSection());
158 CoverageData->setAlignment(8);
160 Constant *Init = CoverageData->getInitializer();
161 // We're expecting { i32, i32, i32, i32, [n x { i8*, i32, i32 }], [m x i8] }
162 // for some C. If not, the frontend's given us something broken.
163 assert(Init->getNumOperands() == 6 && "bad number of fields in coverage map");
164 assert(isa<ConstantArray>(Init->getAggregateElement(4)) &&
165 "invalid function list in coverage map");
166 ConstantArray *Records = cast<ConstantArray>(Init->getAggregateElement(4));
167 for (unsigned I = 0, E = Records->getNumOperands(); I < E; ++I) {
168 Constant *Record = Records->getOperand(I);
169 Value *V = const_cast<Value *>(Record->getOperand(0))->stripPointerCasts();
171 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
172 GlobalVariable *Name = cast<GlobalVariable>(V);
174 // If we have region counters for this name, we've already handled it.
175 auto It = RegionCounters.find(Name);
176 if (It != RegionCounters.end())
179 // Move the name variable to the right section.
180 Name->setSection(getNameSection());
181 Name->setAlignment(1);
185 /// Get the name of a profiling variable for a particular function.
186 static std::string getVarName(InstrProfIncrementInst *Inc, StringRef VarName) {
187 auto *Arr = cast<ConstantDataArray>(Inc->getName()->getInitializer());
188 StringRef Name = Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
189 return ("__llvm_profile_" + VarName + "_" + Name).str();
193 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
194 GlobalVariable *Name = Inc->getName();
195 auto It = RegionCounters.find(Name);
196 if (It != RegionCounters.end())
199 // Move the name variable to the right section. Place them in a COMDAT group
200 // if the associated function is a COMDAT. This will make sure that
201 // only one copy of counters of the COMDAT function will be emitted after
203 Function *Fn = Inc->getParent()->getParent();
204 Comdat *ProfileVarsComdat = nullptr;
206 ProfileVarsComdat = M->getOrInsertComdat(StringRef(getVarName(Inc, "vars")));
207 Name->setSection(getNameSection());
208 Name->setAlignment(1);
209 Name->setComdat(ProfileVarsComdat);
211 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
212 LLVMContext &Ctx = M->getContext();
213 ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
215 // Create the counters variable.
216 auto *Counters = new GlobalVariable(*M, CounterTy, false, Name->getLinkage(),
217 Constant::getNullValue(CounterTy),
218 getVarName(Inc, "counters"));
219 Counters->setVisibility(Name->getVisibility());
220 Counters->setSection(getCountersSection());
221 Counters->setAlignment(8);
222 Counters->setComdat(ProfileVarsComdat);
224 RegionCounters[Inc->getName()] = Counters;
226 // Create data variable.
227 auto *NameArrayTy = Name->getType()->getPointerElementType();
228 auto *Int32Ty = Type::getInt32Ty(Ctx);
229 auto *Int64Ty = Type::getInt64Ty(Ctx);
230 auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
231 auto *Int64PtrTy = Type::getInt64PtrTy(Ctx);
233 Type *DataTypes[] = {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int64PtrTy};
234 auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
235 Constant *DataVals[] = {
236 ConstantInt::get(Int32Ty, NameArrayTy->getArrayNumElements()),
237 ConstantInt::get(Int32Ty, NumCounters),
238 ConstantInt::get(Int64Ty, Inc->getHash()->getZExtValue()),
239 ConstantExpr::getBitCast(Name, Int8PtrTy),
240 ConstantExpr::getBitCast(Counters, Int64PtrTy)};
241 auto *Data = new GlobalVariable(*M, DataTy, true, Name->getLinkage(),
242 ConstantStruct::get(DataTy, DataVals),
243 getVarName(Inc, "data"));
244 Data->setVisibility(Name->getVisibility());
245 Data->setSection(getDataSection());
246 Data->setAlignment(8);
247 Data->setComdat(ProfileVarsComdat);
249 // Mark the data variable as used so that it isn't stripped out.
250 UsedVars.push_back(Data);
255 void InstrProfiling::emitRegistration() {
256 // Don't do this for Darwin. compiler-rt uses linker magic.
257 if (Triple(M->getTargetTriple()).isOSDarwin())
260 // Construct the function.
261 auto *VoidTy = Type::getVoidTy(M->getContext());
262 auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
263 auto *RegisterFTy = FunctionType::get(VoidTy, false);
264 auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
265 "__llvm_profile_register_functions", M);
266 RegisterF->setUnnamedAddr(true);
267 if (Options.NoRedZone)
268 RegisterF->addFnAttr(Attribute::NoRedZone);
270 auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
271 auto *RuntimeRegisterF =
272 Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
273 "__llvm_profile_register_function", M);
275 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
276 for (Value *Data : UsedVars)
277 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
281 void InstrProfiling::emitRuntimeHook() {
282 const char *const RuntimeVarName = "__llvm_profile_runtime";
283 const char *const RuntimeUserName = "__llvm_profile_runtime_user";
285 // If the module's provided its own runtime, we don't need to do anything.
286 if (M->getGlobalVariable(RuntimeVarName))
289 // Declare an external variable that will pull in the runtime initialization.
290 auto *Int32Ty = Type::getInt32Ty(M->getContext());
292 new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
293 nullptr, RuntimeVarName);
295 // Make a function that uses it.
297 Function::Create(FunctionType::get(Int32Ty, false),
298 GlobalValue::LinkOnceODRLinkage, RuntimeUserName, M);
299 User->addFnAttr(Attribute::NoInline);
300 if (Options.NoRedZone)
301 User->addFnAttr(Attribute::NoRedZone);
302 User->setVisibility(GlobalValue::HiddenVisibility);
304 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
305 auto *Load = IRB.CreateLoad(Var);
308 // Mark the user variable as used so that it isn't stripped out.
309 UsedVars.push_back(User);
312 void InstrProfiling::emitUses() {
313 if (UsedVars.empty())
316 GlobalVariable *LLVMUsed = M->getGlobalVariable("llvm.used");
317 std::vector<Constant *> MergedVars;
319 // Collect the existing members of llvm.used.
320 ConstantArray *Inits = cast<ConstantArray>(LLVMUsed->getInitializer());
321 for (unsigned I = 0, E = Inits->getNumOperands(); I != E; ++I)
322 MergedVars.push_back(Inits->getOperand(I));
323 LLVMUsed->eraseFromParent();
326 Type *i8PTy = Type::getInt8PtrTy(M->getContext());
327 // Add uses for our data.
328 for (auto *Value : UsedVars)
329 MergedVars.push_back(
330 ConstantExpr::getBitCast(cast<Constant>(Value), i8PTy));
332 // Recreate llvm.used.
333 ArrayType *ATy = ArrayType::get(i8PTy, MergedVars.size());
335 new GlobalVariable(*M, ATy, false, GlobalValue::AppendingLinkage,
336 ConstantArray::get(ATy, MergedVars), "llvm.used");
338 LLVMUsed->setSection("llvm.metadata");
341 void InstrProfiling::emitInitialization() {
342 std::string InstrProfileOutput = Options.InstrProfileOutput;
344 Constant *RegisterF = M->getFunction("__llvm_profile_register_functions");
345 if (!RegisterF && InstrProfileOutput.empty())
348 // Create the initialization function.
349 auto *VoidTy = Type::getVoidTy(M->getContext());
351 Function::Create(FunctionType::get(VoidTy, false),
352 GlobalValue::InternalLinkage, "__llvm_profile_init", M);
353 F->setUnnamedAddr(true);
354 F->addFnAttr(Attribute::NoInline);
355 if (Options.NoRedZone)
356 F->addFnAttr(Attribute::NoRedZone);
358 // Add the basic block and the necessary calls.
359 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
361 IRB.CreateCall(RegisterF, {});
362 if (!InstrProfileOutput.empty()) {
363 auto *Int8PtrTy = Type::getInt8PtrTy(M->getContext());
364 auto *SetNameTy = FunctionType::get(VoidTy, Int8PtrTy, false);
366 Function::Create(SetNameTy, GlobalValue::ExternalLinkage,
367 "__llvm_profile_override_default_filename", M);
369 // Create variable for profile name.
370 Constant *ProfileNameConst =
371 ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true);
372 GlobalVariable *ProfileName =
373 new GlobalVariable(*M, ProfileNameConst->getType(), true,
374 GlobalValue::PrivateLinkage, ProfileNameConst);
376 IRB.CreateCall(SetNameF, IRB.CreatePointerCast(ProfileName, Int8PtrTy));
380 appendToGlobalCtors(*M, F, 0);