DK_OptimizationRemarkAnalysisAliasing,
DK_OptimizationFailure,
DK_MIRParser,
- DK_PGOProfile,
DK_FirstPluginKind
};
const Twine &Msg;
};
-/// Diagnostic information for the PGO profiler.
-class DiagnosticInfoPGOProfile : public DiagnosticInfo {
-public:
- DiagnosticInfoPGOProfile(const char *FileName, const Twine &Msg,
- DiagnosticSeverity Severity = DS_Error)
- : DiagnosticInfo(DK_PGOProfile, Severity), FileName(FileName), Msg(Msg) {}
-
- /// \see DiagnosticInfo::print.
- void print(DiagnosticPrinter &DP) const override;
-
- static bool classof(const DiagnosticInfo *DI) {
- return DI->getKind() == DK_PGOProfile;
- }
-
- const char *getFileName() const { return FileName; }
- const Twine &getMsg() const { return Msg; }
-
-private:
- /// Name of the input file associated with this diagnostic.
- const char *FileName;
-
- /// Message to report.
- const Twine &Msg;
-};
-
/// Common features for diagnostics dealing with optimization remarks.
class DiagnosticInfoOptimizationBase : public DiagnosticInfo {
public:
void initializeExpandPostRAPass(PassRegistry&);
void initializeAAResultsWrapperPassPass(PassRegistry &);
void initializeGCOVProfilerPass(PassRegistry&);
-void initializePGOInstrumentationGenPass(PassRegistry&);
-void initializePGOInstrumentationUsePass(PassRegistry&);
void initializeInstrProfilingPass(PassRegistry&);
void initializeAddressSanitizerPass(PassRegistry&);
void initializeAddressSanitizerModulePass(PassRegistry&);
(void) llvm::createDomOnlyViewerPass();
(void) llvm::createDomViewerPass();
(void) llvm::createGCOVProfilerPass();
- (void) llvm::createPGOInstrumentationGenPass();
- (void) llvm::createPGOInstrumentationUsePass();
(void) llvm::createInstrProfilingPass();
(void) llvm::createFunctionInliningPass();
(void) llvm::createAlwaysInlinerPass();
ModulePass *createGCOVProfilerPass(const GCOVOptions &Options =
GCOVOptions::getDefault());
-// PGO Instrumention
-ModulePass *createPGOInstrumentationGenPass();
-ModulePass *
-createPGOInstrumentationUsePass(StringRef Filename = StringRef(""));
-
/// Options for the frontend instrumentation based profiling pass.
struct InstrProfOptions {
InstrProfOptions() : NoRedZone(false) {}
/// protect against stack-based overflow vulnerabilities.
FunctionPass *createSafeStackPass(const TargetMachine *TM = nullptr);
-/// \brief Calculate what to divide by to scale counts.
-///
-/// Given the maximum count, calculate a divisor that will scale all the
-/// weights to strictly less than UINT32_MAX.
-static inline uint64_t calculateCountScale(uint64_t MaxCount) {
- return MaxCount < UINT32_MAX ? 1 : MaxCount / UINT32_MAX + 1;
-}
-
-/// \brief Scale an individual branch count.
-///
-/// Scale a 64-bit weight down to 32-bits using \c Scale.
-///
-static inline uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale) {
- uint64_t Scaled = Count / Scale;
- assert(Scaled <= UINT32_MAX && "overflow 32-bits");
- return Scaled;
-}
-
} // End llvm namespace
#endif
DP << getMsg();
}
-void DiagnosticInfoPGOProfile::print(DiagnosticPrinter &DP) const {
- if (getFileName())
- DP << getFileName() << ": ";
- DP << getMsg();
-}
-
bool DiagnosticInfoOptimizationBase::isLocationAvailable() const {
return getDebugLoc();
}
name = IPO
parent = Transforms
library_name = ipo
-required_libraries = Analysis Core InstCombine IRReader Linker Object ProfileData Scalar Support TransformUtils Vectorize Instrumentation
+required_libraries = Analysis Core InstCombine IRReader Linker Object ProfileData Scalar Support TransformUtils Vectorize
+++ /dev/null
-//===-- CFGMST.h - Minimum Spanning Tree for CFG -------*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements Union-find algorithm to compute Minimum Spanning Tree
-// for a given CFG.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Support/BranchProbability.h"
-#include "llvm/Transforms/Utils/BasicBlockUtils.h"
-#include "llvm/Analysis/CFG.h"
-#include "llvm/Analysis/BranchProbabilityInfo.h"
-#include "llvm/Analysis/BlockFrequencyInfo.h"
-#include <string>
-#include <vector>
-#include <utility>
-
-namespace llvm {
-
-#define DEBUG_TYPE "cfgmst"
-
-template <class Edge, class BBInfo> class CFGMST {
-public:
- Function &F;
-
- // Store all the edges in CFG. It may contain some stale edges
- // when Removed is set.
- std::vector<std::unique_ptr<Edge>> AllEdges;
-
- // This map records the auxiliary information for each BB.
- DenseMap<const BasicBlock *, std::unique_ptr<BBInfo>> BBInfos;
-
- // Find the root group of the G and compress the path from G to the root.
- BBInfo *findAndCompressGroup(BBInfo *G) {
- if (G->Group != G)
- G->Group = findAndCompressGroup(static_cast<BBInfo *>(G->Group));
- return static_cast<BBInfo *>(G->Group);
- }
-
- // Union BB1 and BB2 into the same group and return true.
- // Returns false if BB1 and BB2 are already in the same group.
- bool unionGroups(const BasicBlock *BB1, const BasicBlock *BB2) {
- BBInfo *BB1G = findAndCompressGroup(&getBBInfo(BB1));
- BBInfo *BB2G = findAndCompressGroup(&getBBInfo(BB2));
-
- if (BB1G == BB2G)
- return false;
-
- // Make the smaller rank tree a direct child or the root of high rank tree.
- if (BB1G->Rank < BB2G->Rank)
- BB1G->Group = BB2G;
- else {
- BB2G->Group = BB1G;
- // If the ranks are the same, increment root of one tree by one.
- if (BB1G->Rank == BB2G->Rank)
- BB1G->Rank++;
- }
- return true;
- }
-
- // Give BB, return the auxiliary information.
- BBInfo &getBBInfo(const BasicBlock *BB) const {
- auto It = BBInfos.find(BB);
- assert(It->second.get() != nullptr);
- return *It->second.get();
- }
-
- // Traverse the CFG using a stack. Find all the edges and assign the weight.
- // Edges with large weight will be put into MST first so they are less likely
- // to be instrumented.
- void buildEdges() {
- DEBUG(dbgs() << "Build Edge on " << F.getName() << "\n");
-
- const BasicBlock *BB = &(F.getEntryBlock());
- // Add a fake edge to the entry.
- addEdge(nullptr, BB, BFI->getEntryFreq());
-
- // Special handling for single BB functions.
- if (succ_empty(BB)) {
- addEdge(BB, nullptr, BFI->getEntryFreq());
- return;
- }
-
- static const uint32_t CriticalEdgeMultiplier = 1000;
-
- for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) {
- TerminatorInst *TI = BB->getTerminator();
- uint64_t BBWeight = BFI->getBlockFreq(&*BB).getFrequency();
- uint64_t Weight;
- if (int successors = TI->getNumSuccessors()) {
- for (int i = 0; i != successors; ++i) {
- BasicBlock *TargetBB = TI->getSuccessor(i);
- bool Critical = isCriticalEdge(TI, i);
- uint64_t scaleFactor = BBWeight;
- if (Critical) {
- if (scaleFactor < UINT64_MAX / CriticalEdgeMultiplier)
- scaleFactor *= CriticalEdgeMultiplier;
- else
- scaleFactor = UINT64_MAX;
- }
- Weight = BPI->getEdgeProbability(&*BB, TargetBB).scale(scaleFactor);
- addEdge(&*BB, TargetBB, Weight).IsCritical = Critical;
- DEBUG(dbgs() << " Edge: from " << BB->getName() << " to "
- << TargetBB->getName() << " w=" << Weight << "\n");
- }
- } else {
- addEdge(&*BB, nullptr, BBWeight);
- DEBUG(dbgs() << " Edge: from " << BB->getName() << " to exit"
- << " w = " << BBWeight << "\n");
- }
- }
- }
-
- // Sort CFG edges based on its weight.
- void sortEdgesByWeight() {
- std::sort(
- AllEdges.begin(), AllEdges.end(),
- [](const std::unique_ptr<Edge> &lhs, const std::unique_ptr<Edge> &rhs) {
- return lhs->Weight > rhs->Weight;
- });
- }
-
- // Traverse all the edges and compute the Minimum Weight Spanning Tree
- // using union-find algorithm.
- void computeMinimumSpanningTree() {
- // First, put all the critical edge with landing-pad as the Dest to MST.
- // This works around the insufficient support of critical edges split
- // when destination BB is a landing pad.
- for (auto &Ei : AllEdges) {
- if (Ei->Removed)
- continue;
- if (Ei->IsCritical) {
- if (Ei->DestBB && Ei->DestBB->isLandingPad()) {
- if (unionGroups(Ei->SrcBB, Ei->DestBB))
- Ei->InMST = true;
- }
- }
- }
-
- for (auto &Ei : AllEdges) {
- if (Ei->Removed)
- continue;
- if (unionGroups(Ei->SrcBB, Ei->DestBB))
- Ei->InMST = true;
- }
- }
-
- // Dump the Debug information about the instrumentation.
- void dumpEdges(raw_ostream &OS, const StringRef Message = StringRef()) const {
- if (!Message.empty())
- OS << Message << "\n";
- OS << " Number of Basic Blocks: " << BBInfos.size() << "\n";
- for (auto &BI : BBInfos) {
- const BasicBlock *BB = BI.first;
- OS << " BB: " << (BB == nullptr ? "FakeNode" : BB->getName()) << " "
- << BI.second->infoString() << "\n";
- }
-
- OS << " Number of Edges: " << AllEdges.size()
- << " (*: Instrument, C: CriticalEdge, -: Removed)\n";
- uint32_t Count = 0;
- for (auto &EI : AllEdges) {
- OS << " Edge " << Count++ << ": " << getBBInfo(EI->SrcBB).Index << "-->"
- << getBBInfo(EI->DestBB).Index << EI->infoString() << "\n";
- };
- }
-
- // Add an edge to AllEdges with weight W.
- Edge &addEdge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W) {
- uint32_t Index = BBInfos.size();
- auto Iter = BBInfos.end();
- bool Inserted;
- std::tie(Iter, Inserted) = BBInfos.insert(std::make_pair(Src, nullptr));
- if (Inserted) {
- // Newly inserted, update the real info.
- Iter->second = std::move(llvm::make_unique<BBInfo>(Index));
- Index++;
- }
- std::tie(Iter, Inserted) = BBInfos.insert(std::make_pair(Dest, nullptr));
- if (Inserted)
- // Newly inserted, update the real info.
- Iter->second = std::move(llvm::make_unique<BBInfo>(Index));
- AllEdges.emplace_back(new Edge(Src, Dest, W));
- return *AllEdges.back();
- }
-
- BranchProbabilityInfo *BPI;
- BlockFrequencyInfo *BFI;
-
-public:
- CFGMST(Function &Func, BranchProbabilityInfo *BPI_ = nullptr,
- BlockFrequencyInfo *BFI_ = nullptr)
- : F(Func), BPI(BPI_), BFI(BFI_) {
- buildEdges();
- sortEdgesByWeight();
- computeMinimumSpanningTree();
- }
-};
-
-#undef DEBUG_TYPE // "cfgmst"
-} // end namespace llvm
MemorySanitizer.cpp
Instrumentation.cpp
InstrProfiling.cpp
- PGOInstrumentation.cpp
SafeStack.cpp
SanitizerCoverage.cpp
ThreadSanitizer.cpp
initializeAddressSanitizerModulePass(Registry);
initializeBoundsCheckingPass(Registry);
initializeGCOVProfilerPass(Registry);
- initializePGOInstrumentationGenPass(Registry);
- initializePGOInstrumentationUsePass(Registry);
initializeInstrProfilingPass(Registry);
initializeMemorySanitizerPass(Registry);
initializeThreadSanitizerPass(Registry);
type = Library
name = Instrumentation
parent = Transforms
-required_libraries = Analysis Core MC Support TransformUtils ProfileData
+required_libraries = Analysis Core MC Support TransformUtils
+++ /dev/null
-//===- PGOInstru.cpp - PGO Instrumentation --------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements PGO instrumentation using a minimum spanning tree based
-// on the following paper.
-// [1] Donald E. Knuth, Francis R. Stevenson. Optimal measurement of points
-// for program frequency counts. BIT Numerical Mathematics 1973, Volume 13,
-// Issue 3, pp 313-322
-// The idea of the algorithm based on the fact that for each node (except for
-// the entry and exit), the sum of incoming edge counts equals the sum of
-// outgoing edge counts. The count of edge on spanning tree can be derived from
-// those edges not on the spanning tree. Knuth proves this method instruments
-// the minimum number of edges.
-//
-// The minimal spanning tree here is actually a maximum weight tree -- on-tree
-// edges have higher frequencies (most likely to execute). The idea is to
-// instrument those less frequently executed edges which speeds up the
-// instrumented binaries.
-//
-// This file contains two passes:
-// (1) Pass PGOInstrumentationGen which instruments the IR to generate edge
-// count profile, and
-// (2) Pass PGOInstrumentationUse which reads the edge count profile and
-// annotates the branch weight.
-// These two passes are mutually exclusive, and they are called at the same
-// compilation point (so they see the same IR). For PGOInstrumentationGen,
-// the real work is done instrumentOneFunc(). For PGOInstrumentationUse, the
-// real work in done in class PGOUseFunc and the profile is opened in module
-// level and passed to each PGOUseFunc instance.
-// The shared code for PGOInstrumentationGen and PGOInstrumentationUse is put
-// in class FuncPGOInstrumentation.
-//
-// Class PGOEdge represents a CFG edge and some auxiliary information. Class
-// BBInfo contains auxiliary information for a BB. These two classes are used
-// in PGOGenFunc. Class PGOUseEdge and UseBBInfo are the derived class of
-// PGOEdge and BBInfo, respectively. They contains extra data structure used
-// in populating profile counters.
-// The MST implementation is in Class CFGMST.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Transforms/Instrumentation.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/IR/InstIterator.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/MDBuilder.h"
-#include "llvm/IR/DiagnosticInfo.h"
-#include "llvm/Pass.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/BranchProbability.h"
-#include "llvm/Support/JamCRC.h"
-#include "llvm/Transforms/Utils/BasicBlockUtils.h"
-#include "llvm/ProfileData/InstrProfReader.h"
-#include "llvm/Analysis/CFG.h"
-#include "llvm/Analysis/BranchProbabilityInfo.h"
-#include "llvm/Analysis/BlockFrequencyInfo.h"
-#include <string>
-#include <vector>
-#include <utility>
-#include "CFGMST.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "pgo-instr"
-
-STATISTIC(NumOfPGOInstrument, "Number of edges instrumented.");
-STATISTIC(NumOfPGOEdge, "Number of edges.");
-STATISTIC(NumOfPGOBB, "Number of basic-blocks.");
-STATISTIC(NumOfPGOSplit, "Number of critical edge splits.");
-STATISTIC(NumOfPGOFunc, "Number of functions having valid profile counts.");
-STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile.");
-STATISTIC(NumOfPGOMissing, "Number of functions without profile.");
-
-static cl::opt<std::string>
- PGOProfileFile("pgo-profile-file", cl::init(""), cl::Hidden,
- cl::value_desc("filename"),
- cl::desc("Specify the path of profile data file"));
-
-namespace {
-class PGOInstrumentationGen : public ModulePass {
-public:
- static char ID;
-
- PGOInstrumentationGen() : ModulePass(ID) {
- initializePGOInstrumentationGenPass(*PassRegistry::getPassRegistry());
- }
-
- const char *getPassName() const override {
- return "PGOInstrumentationGenPass";
- }
-
-private:
- bool runOnModule(Module &M) override;
-
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.addRequired<BlockFrequencyInfoWrapperPass>();
- AU.addRequired<BranchProbabilityInfoWrapperPass>();
- }
-};
-
-class PGOInstrumentationUse : public ModulePass {
-public:
- static char ID;
-
- // Provide the profile filename as the parameter.
- PGOInstrumentationUse(StringRef Filename = StringRef(""))
- : ModulePass(ID), ProfileFileName(Filename) {
- if (!PGOProfileFile.empty())
- ProfileFileName = StringRef(PGOProfileFile);
- initializePGOInstrumentationUsePass(*PassRegistry::getPassRegistry());
- }
-
- const char *getPassName() const override {
- return "PGOInstrumentationUsePass";
- }
-
-private:
- StringRef ProfileFileName;
- std::unique_ptr<IndexedInstrProfReader> PGOReader;
- bool runOnModule(Module &M) override;
-
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.addRequired<BlockFrequencyInfoWrapperPass>();
- AU.addRequired<BranchProbabilityInfoWrapperPass>();
- }
-};
-} // end anonymous namespace
-
-char PGOInstrumentationGen::ID = 0;
-INITIALIZE_PASS_BEGIN(PGOInstrumentationGen, "pgo-instr-gen",
- "PGO instrumentation.", false, false)
-INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)
-INITIALIZE_PASS_END(PGOInstrumentationGen, "pgo-instr-gen",
- "PGO instrumentation.", false, false)
-
-ModulePass *llvm::createPGOInstrumentationGenPass() {
- return new PGOInstrumentationGen();
-}
-
-char PGOInstrumentationUse::ID = 0;
-INITIALIZE_PASS_BEGIN(PGOInstrumentationUse, "pgo-instr-use",
- "Read PGO instrumentation profile.", false, false)
-INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)
-INITIALIZE_PASS_END(PGOInstrumentationUse, "pgo-instr-use",
- "Read PGO instrumentation profile.", false, false)
-
-ModulePass *llvm::createPGOInstrumentationUsePass(StringRef Filename) {
- return new PGOInstrumentationUse(Filename);
-}
-
-namespace {
-/// \brief An MST based instrumentation for PGO
-///
-/// Implements a Minimum Spanning Tree (MST) based instrumentation for PGO
-/// in the function level.
-//
-// This class implements the CFG edges. Note the CFG can be a multi-graph.
-struct PGOEdge {
- const BasicBlock *SrcBB;
- const BasicBlock *DestBB;
- uint64_t Weight;
- bool InMST;
- bool Removed;
- bool IsCritical;
- PGOEdge(const BasicBlock *Src, const BasicBlock *Dest, unsigned W = 1)
- : SrcBB(Src), DestBB(Dest), Weight(W), InMST(false), Removed(false),
- IsCritical(false) {}
- // Return the information string of an edge.
- const std::string infoString() const {
- std::string Str = (Removed ? "-" : " ");
- Str += (InMST ? " " : "*");
- Str += (IsCritical ? "c" : " ");
- Str += " W=" + std::to_string(Weight);
- return Str;
- }
-};
-
-// This class stores the auxiliary information for each BB.
-struct BBInfo {
- BBInfo *Group;
- uint32_t Index;
- uint32_t Rank;
-
- BBInfo(unsigned IX) : Group(this), Index(IX), Rank(0) {}
-
- // Return the information string of this object.
- const std::string infoString() const {
- return "Index=" + std::to_string(Index);
- }
-};
-
-// This class implements the CFG edges. Note the CFG can be a multi-graph.
-template <class Edge, class BBInfo> class FuncPGOInstrumentation {
-private:
- Function &F;
- void computeCFGHash();
-
-public:
- std::string FuncName;
- GlobalVariable *FuncNameVar;
- // CFG hash value for this function.
- uint64_t FunctionHash;
-
- // The Minimum Spanning Tree of function CFG.
- CFGMST<Edge, BBInfo> MST;
-
- // Give an edge, find the BB that will be instrumented.
- // Return nullptr if there is no BB to be instrumented.
- BasicBlock *getInstrBB(Edge *E);
-
- // Return the auxiliary BB information.
- BBInfo &getBBInfo(const BasicBlock *BB) const { return MST.getBBInfo(BB); }
-
- // Dump edges and BB information.
- void dumpInfo(std::string Str = "") const {
- std::string Message = "Dump Function " + FuncName + " Hash: " +
- std::to_string(FunctionHash) + "\t" + Str;
- MST.dumpEdges(dbgs(), Message);
- }
-
- FuncPGOInstrumentation(Function &Func, bool CreateGlobalVar = false,
- BranchProbabilityInfo *BPI_ = nullptr,
- BlockFrequencyInfo *BFI_ = nullptr)
- : F(Func), FunctionHash(0), MST(F, BPI_, BFI_) {
- FuncName = getPGOFuncName(F);
- computeCFGHash();
- DEBUG(dumpInfo("after CFGMST"));
-
- NumOfPGOBB += MST.BBInfos.size();
- for (auto &Ei : MST.AllEdges) {
- if (Ei->Removed)
- continue;
- NumOfPGOEdge++;
- if (!Ei->InMST)
- NumOfPGOInstrument++;
- }
-
- if (CreateGlobalVar)
- FuncNameVar = createPGOFuncNameVar(F, FuncName);
- };
-};
-
-// Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index
-// value of each BB in the CFG. The higher 32 bits record the number of edges.
-template <class Edge, class BBInfo>
-void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
- std::vector<char> Indexes;
- JamCRC JC;
- for (auto &BB : F) {
- const TerminatorInst *TI = BB.getTerminator();
- for (unsigned s = 0, e = TI->getNumSuccessors(); s != e; ++s) {
- BasicBlock *Succ = TI->getSuccessor(s);
- uint32_t Index = getBBInfo(Succ).Index;
- for (int i = 0; i < 4; i++)
- Indexes.push_back((char)(Index >> i));
- }
- }
- JC.update(Indexes);
- FunctionHash = MST.AllEdges.size() << 32 | JC.getCRC();
-}
-
-template <class Edge, class BBInfo>
-BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {
- if (E->InMST || E->Removed)
- return nullptr;
-
- BasicBlock *SrcBB = const_cast<BasicBlock *>(E->SrcBB);
- BasicBlock *DestBB = const_cast<BasicBlock *>(E->DestBB);
- // For a fake edge, instrument the real BB.
- if (SrcBB == nullptr)
- return DestBB;
- if (DestBB == nullptr)
- return SrcBB;
-
- // Instrument the SrcBB if it has a single successor,
- // otherwise, the DestBB if this is not a critical edge.
- TerminatorInst *TI = SrcBB->getTerminator();
- if (TI->getNumSuccessors() <= 1)
- return SrcBB;
- if (!E->IsCritical)
- return DestBB;
-
- // For a critical edge, we have to split. Instrument the newly
- // created BB.
- NumOfPGOSplit++;
- DEBUG(dbgs() << "Split critical edge: " << getBBInfo(SrcBB).Index << " --> "
- << getBBInfo(DestBB).Index << "\n");
- unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB);
- BasicBlock *InstrBB = SplitCriticalEdge(TI, SuccNum);
- assert(InstrBB && "Critical edge is not split");
-
- E->Removed = true;
- return InstrBB;
-}
-
-// Visit all edge and instrument the edges not in MST.
-// Critical edges will be split.
-static void instrumentOneFunc(Function &F, Module *M,
- BranchProbabilityInfo *BPI,
- BlockFrequencyInfo *BFI) {
- unsigned NumCounters = 0;
- FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(F, true, BPI, BFI);
- for (auto &Ei : FuncInfo.MST.AllEdges) {
- if (!Ei->InMST && !Ei->Removed)
- NumCounters++;
- }
-
- uint32_t j = 0;
- for (auto &Ei : FuncInfo.MST.AllEdges) {
- BasicBlock *InstrBB = FuncInfo.getInstrBB(Ei.get());
- if (!InstrBB)
- continue;
-
- IRBuilder<> Builder(InstrBB, InstrBB->getFirstInsertionPt());
- assert(Builder.GetInsertPoint() != InstrBB->end() &&
- "Cannot get the Instrumentation point");
- auto *I8PtrTy = Type::getInt8PtrTy(M->getContext());
- Builder.CreateCall(
- Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment),
- {llvm::ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
- Builder.getInt64(FuncInfo.FunctionHash), Builder.getInt32(NumCounters),
- Builder.getInt32(j++)});
- }
-}
-
-struct PGOUseEdge : public PGOEdge {
- bool CountValid;
- uint64_t CountValue;
- PGOUseEdge(const BasicBlock *Src, const BasicBlock *Dest, unsigned W = 1)
- : PGOEdge(Src, Dest, W), CountValid(false), CountValue(0) {}
-
- // Set edge count value
- void setEdgeCount(uint64_t Value) {
- CountValue = Value;
- CountValid = true;
- }
-
- // Return the information string for this object.
- const std::string infoString() const {
- if (!CountValid)
- return PGOEdge::infoString();
- return PGOEdge::infoString() + " Count=" + std::to_string(CountValue);
- }
-};
-
-typedef SmallVector<PGOUseEdge *, 2> DirectEdges;
-
-// This class stores the auxiliary information for each BB.
-struct UseBBInfo : public BBInfo {
- uint64_t CountValue;
- bool CountValid;
- int32_t UnknownCountInEdge;
- int32_t UnknownCountOutEdge;
- DirectEdges InEdges;
- DirectEdges OutEdges;
- UseBBInfo(unsigned IX)
- : BBInfo(IX), CountValue(0), CountValid(false), UnknownCountInEdge(0),
- UnknownCountOutEdge(0) {}
- UseBBInfo(unsigned IX, uint64_t C)
- : BBInfo(IX), CountValue(C), CountValid(true), UnknownCountInEdge(0),
- UnknownCountOutEdge(0) {}
-
- // Set the profile count value for this BB.
- void setBBInfoCount(uint64_t Value) {
- CountValue = Value;
- CountValid = true;
- }
-
- // Return the information string of this object.
- const std::string infoString() const {
- if (!CountValid)
- return BBInfo::infoString();
- return BBInfo::infoString() + " Count=" + std::to_string(CountValue);
- }
-};
-
-// Sum up the count values for all the edges.
-static uint64_t sumEdgeCount(const ArrayRef<PGOUseEdge *> Edges) {
- uint64_t Total = 0;
- for (auto &Ei : Edges) {
- if (Ei->Removed)
- continue;
- Total += Ei->CountValue;
- }
- return Total;
-}
-
-class PGOUseFunc {
-private:
- Function &F;
- Module *M;
- // This member stores the shared information with class PGOGenFunc.
- FuncPGOInstrumentation<PGOUseEdge, UseBBInfo> FuncInfo;
-
- // Return the auxiliary BB information.
- UseBBInfo &getBBInfo(const BasicBlock *BB) const {
- return FuncInfo.getBBInfo(BB);
- }
-
- // The maximum count value in the profile. This is only used in PGO use
- // compilation.
- uint64_t ProgramMaxCount;
-
- // Find the Instrumented BB and set the value.
- void setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile);
-
- // Set the edge counter value for the unknown edge -- there should be only
- // one unknown edge.
- void setEdgeCount(DirectEdges &Edges, uint64_t Value);
-
- // Return FuncName string;
- const std::string getFuncName() const { return FuncInfo.FuncName; }
-
- // Set the hot/cold inline hints based on the count values.
- void applyFunctionAttributes(uint64_t EntryCount, uint64_t MaxCount) {
- if (ProgramMaxCount == 0)
- return;
- // Threshold of the hot functions.
- const BranchProbability HotFunctionThreshold(1, 100);
- // Threshold of the cold functions.
- const BranchProbability ColdFunctionThreshold(2, 10000);
- if (EntryCount >= HotFunctionThreshold.scale(ProgramMaxCount))
- F.addFnAttr(llvm::Attribute::InlineHint);
- else if (MaxCount <= ColdFunctionThreshold.scale(ProgramMaxCount))
- F.addFnAttr(llvm::Attribute::Cold);
- }
-
-public:
- PGOUseFunc(Function &Func, Module *Modu,
- BranchProbabilityInfo *BPI_ = nullptr,
- BlockFrequencyInfo *BFI_ = nullptr)
- : F(Func), M(Modu), FuncInfo(Func, false, BPI_, BFI_) {}
-
- // Read counts for the instrumented BB from profile.
- bool readCounters(IndexedInstrProfReader *PGOReader);
-
- // Populate the counts for all BBs.
- void populateCounters();
-
- // Set the branch weights based on the count values.
- void setBranchWeights();
-};
-
-// Visit all the edges and assign the count value for the instrumented
-// edges and the BB.
-void PGOUseFunc::setInstrumentedCounts(
- const std::vector<uint64_t> &CountFromProfile) {
-
- // Use a worklist as we will update the vector during the iteration.
- std::vector<PGOUseEdge *> WorkList;
- for (auto &Ei : FuncInfo.MST.AllEdges)
- WorkList.push_back(Ei.get());
-
- uint32_t j = 0;
- for (auto &Ei : WorkList) {
- BasicBlock *InstrBB = FuncInfo.getInstrBB(Ei);
- if (!InstrBB)
- continue;
- uint64_t CountValue = CountFromProfile[j++];
- if (!Ei->Removed) {
- getBBInfo(InstrBB).setBBInfoCount(CountValue);
- Ei->setEdgeCount(CountValue);
- continue;
- }
-
- // Need to add two new edges.
- BasicBlock *SrcBB = const_cast<BasicBlock *>(Ei->SrcBB);
- BasicBlock *DestBB = const_cast<BasicBlock *>(Ei->DestBB);
- // Add new edge of SrcBB->InstrBB.
- PGOUseEdge &NewEdge = FuncInfo.MST.addEdge(SrcBB, InstrBB, 0);
- NewEdge.setEdgeCount(CountValue);
- // Add new edge of InstrBB->DestBB.
- PGOUseEdge &NewEdge1 = FuncInfo.MST.addEdge(InstrBB, DestBB, 0);
- NewEdge1.setEdgeCount(CountValue);
- NewEdge1.InMST = true;
- getBBInfo(InstrBB).setBBInfoCount(CountValue);
- }
-}
-
-// Set the count value for the unknown edges. There should be one and only one
-// unknown edge in Edges vector.
-void PGOUseFunc::setEdgeCount(DirectEdges &Edges, uint64_t Value) {
- for (auto &Ei : Edges) {
- if (Ei->CountValid)
- continue;
- Ei->setEdgeCount(Value);
-
- getBBInfo(Ei->SrcBB).UnknownCountOutEdge--;
- getBBInfo(Ei->DestBB).UnknownCountInEdge--;
- return;
- }
- llvm_unreachable("Cannot find the unknown count edge");
-}
-
-// Read the profile from ProfileFileName and assign the value to the
-// instrumented BB and the edges. This function also updates ProgramMaxCount.
-// Return true if the profile are successfully read, and false on errors.
-bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader) {
- auto &Ctx = M->getContext();
- ErrorOr<InstrProfRecord> Result =
- PGOReader->getInstrProfRecord(FuncInfo.FuncName, FuncInfo.FunctionHash);
- if (std::error_code EC = Result.getError()) {
- if (EC == instrprof_error::unknown_function)
- NumOfPGOMissing++;
- else if (EC == instrprof_error::hash_mismatch ||
- EC == llvm::instrprof_error::malformed)
- NumOfPGOMismatch++;
-
- std::string Msg = EC.message() + std::string(" ") + F.getName().str();
- Ctx.diagnose(
- DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
- return false;
- }
- std::vector<uint64_t> &CountFromProfile = Result.get().Counts;
-
- NumOfPGOFunc++;
- DEBUG(dbgs() << CountFromProfile.size() << " counts\n");
- uint64_t ValueSum = 0;
- for (unsigned i = 0, e = CountFromProfile.size(); i < e; i++) {
- DEBUG(dbgs() << " " << i << ": " << CountFromProfile[i] << "\n");
- ValueSum += CountFromProfile[i];
- }
-
- DEBUG(dbgs() << "SUM = " << ValueSum << "\n");
-
- getBBInfo(nullptr).UnknownCountOutEdge = 2;
- getBBInfo(nullptr).UnknownCountInEdge = 2;
-
- setInstrumentedCounts(CountFromProfile);
- ProgramMaxCount = PGOReader->getMaximumFunctionCount();
- return true;
-}
-
-// Populate the counters from instrumented BBs to all BBs.
-// In the end of this operation, all BBs should have a valid count value.
-void PGOUseFunc::populateCounters() {
- // First set up Count variable for all BBs.
- for (auto &Ei : FuncInfo.MST.AllEdges) {
- if (Ei->Removed)
- continue;
-
- const BasicBlock *SrcBB = Ei->SrcBB;
- const BasicBlock *DestBB = Ei->DestBB;
- UseBBInfo &SrcInfo = getBBInfo(SrcBB);
- UseBBInfo &DestInfo = getBBInfo(DestBB);
- SrcInfo.OutEdges.push_back(Ei.get());
- DestInfo.InEdges.push_back(Ei.get());
- SrcInfo.UnknownCountOutEdge++;
- DestInfo.UnknownCountInEdge++;
-
- if (!Ei->CountValid)
- continue;
- DestInfo.UnknownCountInEdge--;
- SrcInfo.UnknownCountOutEdge--;
- }
-
- bool Changes = true;
- unsigned NumPasses = 0;
- while (Changes) {
- NumPasses++;
- Changes = false;
-
- // For efficient traversal, it's better to start from the end as most
- // of the instrumented edges are at the end.
- for (auto &BB : reverse(F)) {
- UseBBInfo &Count = getBBInfo(&BB);
- if (!Count.CountValid) {
- if (Count.UnknownCountOutEdge == 0) {
- Count.CountValue = sumEdgeCount(Count.OutEdges);
- Count.CountValid = true;
- Changes = true;
- } else if (Count.UnknownCountInEdge == 0) {
- Count.CountValue = sumEdgeCount(Count.InEdges);
- Count.CountValid = true;
- Changes = true;
- }
- }
- if (Count.CountValid) {
- if (Count.UnknownCountOutEdge == 1) {
- uint64_t Total = Count.CountValue - sumEdgeCount(Count.OutEdges);
- setEdgeCount(Count.OutEdges, Total);
- Changes = true;
- }
- if (Count.UnknownCountInEdge == 1) {
- uint64_t Total = Count.CountValue - sumEdgeCount(Count.InEdges);
- setEdgeCount(Count.InEdges, Total);
- Changes = true;
- }
- }
- }
- }
-
- DEBUG(dbgs() << "Populate counts in " << NumPasses << " passes.\n");
- // Assert every BB has a valid counter.
- uint64_t FuncEntryCount = getBBInfo(&*F.begin()).CountValue;
- uint64_t FuncMaxCount = FuncEntryCount;
- for (auto &BB : F) {
- assert(getBBInfo(&BB).CountValid && "BB count is not valid");
- uint64_t Count = getBBInfo(&BB).CountValue;
- if (Count > FuncMaxCount)
- FuncMaxCount = Count;
- }
- applyFunctionAttributes(FuncEntryCount, FuncMaxCount);
-
- DEBUG(FuncInfo.dumpInfo("after reading profile."));
-}
-
-// Assign the scaled count values to the BB with multiple out edges.
-void PGOUseFunc::setBranchWeights() {
- // Generate MD_prof metadata for every branch instruction.
- DEBUG(dbgs() << "\nSetting branch weights.\n");
- MDBuilder MDB(M->getContext());
- for (auto &BB : F) {
- TerminatorInst *TI = BB.getTerminator();
- if (TI->getNumSuccessors() < 2)
- continue;
- if (!isa<BranchInst>(TI) && !isa<SwitchInst>(TI))
- continue;
- if (getBBInfo(&BB).CountValue == 0)
- continue;
-
- // We have a non-zero Branch BB.
- const UseBBInfo &BBCountInfo = getBBInfo(&BB);
- unsigned Size = BBCountInfo.OutEdges.size();
- SmallVector<unsigned, 2> EdgeCounts(Size, 0);
- uint64_t MaxCount = 0;
- for (unsigned s = 0; s < Size; s++) {
- const PGOUseEdge *E = BBCountInfo.OutEdges[s];
- const BasicBlock *SrcBB = E->SrcBB;
- const BasicBlock *DestBB = E->DestBB;
- if (DestBB == 0)
- continue;
- unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB);
- uint64_t EdgeCount = E->CountValue;
- if (EdgeCount > MaxCount)
- MaxCount = EdgeCount;
- EdgeCounts[SuccNum] = EdgeCount;
- }
- assert(MaxCount > 0 && "Bad max count");
- uint64_t Scale = calculateCountScale(MaxCount);
- SmallVector<unsigned, 4> Weights;
- for (const auto &ECI : EdgeCounts)
- Weights.push_back(scaleBranchCount(ECI, Scale));
-
- TI->setMetadata(llvm::LLVMContext::MD_prof,
- MDB.createBranchWeights(Weights));
- DEBUG(dbgs() << "Weight is: "; for (const auto &W
- : Weights) dbgs()
- << W << " ";
- dbgs() << "\n";);
- }
-}
-} // end anonymous namespace
-
-bool PGOInstrumentationGen::runOnModule(Module &M) {
- for (auto &F : M) {
- if (F.isDeclaration())
- continue;
- BranchProbabilityInfo *BPI =
- &(getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI());
- BlockFrequencyInfo *BFI =
- &(getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI());
- instrumentOneFunc(F, &M, BPI, BFI);
- }
- return true;
-}
-
-static void setPGOCountOnFunc(PGOUseFunc &Func,
- IndexedInstrProfReader *PGOReader) {
- if (Func.readCounters(PGOReader)) {
- Func.populateCounters();
- Func.setBranchWeights();
- }
-}
-
-bool PGOInstrumentationUse::runOnModule(Module &M) {
- DEBUG(dbgs() << "Read in profile counters: ");
- auto &Ctx = M.getContext();
- // Read the counter array from file.
- auto ReaderOrErr = IndexedInstrProfReader::create(ProfileFileName);
- if (std::error_code EC = ReaderOrErr.getError()) {
- Ctx.diagnose(
- DiagnosticInfoPGOProfile(ProfileFileName.data(), EC.message()));
- return false;
- }
-
- PGOReader = std::move(ReaderOrErr.get());
- if (!PGOReader) {
- Ctx.diagnose(DiagnosticInfoPGOProfile(ProfileFileName.data(),
- "Cannot get PGOReader"));
- return false;
- }
-
- for (auto &F : M) {
- if (F.isDeclaration())
- continue;
- BranchProbabilityInfo *BPI =
- &(getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI());
- BlockFrequencyInfo *BFI =
- &(getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI());
- PGOUseFunc Func(F, &M, BPI, BFI);
- setPGOCountOnFunc(Func, PGOReader.get());
- }
- return true;
-}
+++ /dev/null
-; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-; CHECK: @__llvm_profile_name__Z9test_br_1i = private constant [13 x i8] c"_Z9test_br_1i"
-
-define i32 @_Z9test_br_1i(i32 %i) {
-entry:
- %cmp = icmp sgt i32 %i, 0
- br i1 %cmp, label %if.then, label %if.end
-
-if.then:
-; CHECK: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @__llvm_profile_name__Z9test_br_1i, i32 0, i32 0), i64 23925403969, i32 2, i32 1)
- %add = add nsw i32 %i, 2
- br label %if.end
-
-if.end:
- %retv = phi i32 [ %add, %if.then ], [ %i, %entry ]
-; CHECK: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @__llvm_profile_name__Z9test_br_1i, i32 0, i32 0), i64 23925403969, i32 2, i32 0)
- ret i32 %retv
-}
+++ /dev/null
-; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-; CHECK: @__llvm_profile_name__Z9test_br_2i = private constant [13 x i8] c"_Z9test_br_2i"
-
-define i32 @_Z9test_br_2i(i32 %i) {
-entry:
- %cmp = icmp sgt i32 %i, 0
- br i1 %cmp, label %if.then, label %if.else
-
-if.then:
-; CHECK: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @__llvm_profile_name__Z9test_br_2i, i32 0, i32 0), i64 29368252703, i32 2, i32 0)
- %add = add nsw i32 %i, 2
- br label %if.end
-
-if.else:
-; CHECK: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @__llvm_profile_name__Z9test_br_2i, i32 0, i32 0), i64 29368252703, i32 2, i32 1)
- %sub = sub nsw i32 %i, 2
- br label %if.end
-
-if.end:
- %retv = phi i32 [ %add, %if.then ], [ %sub, %if.else ]
- ret i32 %retv
-}
+++ /dev/null
-; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-; CHECK: @__llvm_profile_name__Z17test_criticalEdgeii = private constant [23 x i8] c"_Z17test_criticalEdgeii"
-; CHECK: @"__llvm_profile_name_<stdin>:_ZL3bari" = private constant [16 x i8] c"<stdin>:_ZL3bari"
-
-define i32 @_Z17test_criticalEdgeii(i32 %i, i32 %j) {
-entry:
- switch i32 %i, label %sw.default [
- i32 1, label %sw.bb
- i32 2, label %sw.bb1
- i32 3, label %sw.bb3
- i32 4, label %sw.bb3
-; CHECK: i32 3, label %entry.sw.bb3_crit_edge
-; CHECK: i32 4, label %entry.sw.bb3_crit_edge1
- i32 5, label %sw.bb3
- ]
-
-; CHECK: entry.sw.bb3_crit_edge1:
-; CHECK: br label %sw.bb3
-
-; CHECK: entry.sw.bb3_crit_edge:
-; CHECK: br label %sw.bb3
-
-sw.bb:
- %call = call i32 @_ZL3bari(i32 2)
- br label %sw.epilog
-
-sw.bb1:
- %call2 = call i32 @_ZL3bari(i32 1024)
- br label %sw.epilog
-
-sw.bb3:
- %cmp = icmp eq i32 %j, 2
- br i1 %cmp, label %if.then, label %if.end
-
-if.then:
- %call4 = call i32 @_ZL3bari(i32 4)
- br label %return
-
-if.end:
- %call5 = call i32 @_ZL3bari(i32 8)
- br label %sw.epilog
-
-sw.default:
- %call6 = call i32 @_ZL3bari(i32 32)
- %cmp7 = icmp sgt i32 %j, 10
- br i1 %cmp7, label %if.then8, label %if.end9
-
-if.then8:
- %add = add nsw i32 %call6, 10
- br label %if.end9
-
-if.end9:
- %res.0 = phi i32 [ %add, %if.then8 ], [ %call6, %sw.default ]
- br label %sw.epilog
-
-sw.epilog:
- %res.1 = phi i32 [ %res.0, %if.end9 ], [ %call5, %if.end ], [ %call2, %sw.bb1 ], [ %call, %sw.bb ]
- br label %return
-
-return:
- %retval = phi i32 [ %res.1, %sw.epilog ], [ %call4, %if.then ]
- ret i32 %retval
-}
-
-define internal i32 @_ZL3bari(i32 %i) {
-entry:
- ret i32 %i
-}
+++ /dev/null
-; RUN: llvm-profdata merge %S/Inputs/criticaledge.proftext -o %T/criticaledge.profdata
-; RUN: opt < %s -pgo-instr-use -pgo-profile-file=%T/criticaledge.profdata -S | FileCheck %s
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-define i32 @_Z17test_criticalEdgeii(i32 %i, i32 %j) {
-entry:
- switch i32 %i, label %sw.default [
- i32 1, label %sw.bb
- i32 2, label %sw.bb1
- i32 3, label %sw.bb3
- i32 4, label %sw.bb3
-; CHECK: i32 3, label %entry.sw.bb3_crit_edge
-; CHECK: i32 4, label %entry.sw.bb3_crit_edge1
- i32 5, label %sw.bb3
- ]
-; CHECK: !prof !
-
-; CHECK: entry.sw.bb3_crit_edge1:
-; CHECK: br label %sw.bb3
-; CHECK: entry.sw.bb3_crit_edge:
-; CHECK: br label %sw.bb3
-
-sw.bb:
- %call = call i32 @_ZL3bari(i32 2)
- br label %sw.epilog
-
-sw.bb1:
- %call2 = call i32 @_ZL3bari(i32 1024)
- br label %sw.epilog
-
-sw.bb3:
- %cmp = icmp eq i32 %j, 2
- br i1 %cmp, label %if.then, label %if.end
-; CHECK: !prof !
-
-if.then:
- %call4 = call i32 @_ZL3bari(i32 4)
- br label %return
-
-if.end:
- %call5 = call i32 @_ZL3bari(i32 8)
- br label %sw.epilog
-
-sw.default:
- %call6 = call i32 @_ZL3bari(i32 32)
- %cmp7 = icmp sgt i32 %j, 10
- br i1 %cmp7, label %if.then8, label %if.end9
-; CHECK: !prof !
-
-if.then8:
- %add = add nsw i32 %call6, 10
- br label %if.end9
-
-if.end9:
- %res.0 = phi i32 [ %add, %if.then8 ], [ %call6, %sw.default ]
- br label %sw.epilog
-
-sw.epilog:
- %res.1 = phi i32 [ %res.0, %if.end9 ], [ %call5, %if.end ], [ %call2, %sw.bb1 ], [ %call, %sw.bb ]
- br label %return
-
-return:
- %retval = phi i32 [ %res.1, %sw.epilog ], [ %call4, %if.then ]
- ret i32 %retval
-}
-
-define internal i32 @_ZL3bari(i32 %i) {
-entry:
- ret i32 %i
-}
-
-; CHECK-DAG: !{!"branch_weights", i32 2, i32 2}
-; CHECK-DAG: !{!"branch_weights", i32 1, i32 1}
+++ /dev/null
-; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-@val = global i32 0, align 4
-@_ZTIi = external constant i8*
-; CHECK: @__llvm_profile_name__Z3bari = private constant [7 x i8] c"_Z3bari"
-; CHECK: @__llvm_profile_name__Z3fooi = private constant [7 x i8] c"_Z3fooi"
-
-define i32 @_Z3bari(i32 %i) {
-entry:
- %rem = srem i32 %i, 3
- %tobool = icmp ne i32 %rem, 0
- br i1 %tobool, label %if.then, label %if.end
-
-if.then:
-; CHECK: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @__llvm_profile_name__Z3bari, i32 0, i32 0), i64 23319865734, i32 2, i32 1)
- %exception = call i8* @__cxa_allocate_exception(i64 4)
- %tmp = bitcast i8* %exception to i32*
- store i32 %i, i32* %tmp, align 16
- call void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi to i8*), i8* null)
- unreachable
-
-if.end:
-; CHECK: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @__llvm_profile_name__Z3bari, i32 0, i32 0), i64 23319865734, i32 2, i32 0)
- ret i32 0
-}
-
-declare i8* @__cxa_allocate_exception(i64)
-
-declare void @__cxa_throw(i8*, i8*, i8*)
-
-define i32 @_Z3fooi(i32 %i) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
-entry:
- %rem = srem i32 %i, 2
- %tobool = icmp ne i32 %rem, 0
- br i1 %tobool, label %if.then, label %if.end
-
-if.then:
- %mul = mul nsw i32 %i, 7
- %call = invoke i32 @_Z3bari(i32 %mul)
- to label %invoke.cont unwind label %lpad
-
-invoke.cont:
-; CHECK: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @__llvm_profile_name__Z3fooi, i32 0, i32 0), i64 57451243578, i32 4, i32 1)
- br label %if.end
-
-lpad:
- %tmp = landingpad { i8*, i32 }
- catch i8* bitcast (i8** @_ZTIi to i8*)
- %tmp1 = extractvalue { i8*, i32 } %tmp, 0
- %tmp2 = extractvalue { i8*, i32 } %tmp, 1
- br label %catch.dispatch
-
-catch.dispatch:
- %tmp3 = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*))
- %matches = icmp eq i32 %tmp2, %tmp3
- br i1 %matches, label %catch, label %eh.resume
-
-catch:
-; CHECK: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @__llvm_profile_name__Z3fooi, i32 0, i32 0), i64 57451243578, i32 4, i32 2)
- %tmp4 = call i8* @__cxa_begin_catch(i8* %tmp1)
- %tmp5 = bitcast i8* %tmp4 to i32*
- %tmp6 = load i32, i32* %tmp5, align 4
- %tmp7 = load i32, i32* @val, align 4
- %sub = sub nsw i32 %tmp7, %tmp6
- store i32 %sub, i32* @val, align 4
- call void @__cxa_end_catch()
- br label %try.cont
-
-try.cont:
- ret i32 -1
-
-if.end:
-; CHECK: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @__llvm_profile_name__Z3fooi, i32 0, i32 0), i64 57451243578, i32 4, i32 0)
- %tmp8 = load i32, i32* @val, align 4
- %add = add nsw i32 %tmp8, %i
- store i32 %add, i32* @val, align 4
- br label %try.cont
-
-eh.resume:
-; CHECK: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @__llvm_profile_name__Z3fooi, i32 0, i32 0), i64 57451243578, i32 4, i32 3)
- %lpad.val = insertvalue { i8*, i32 } undef, i8* %tmp1, 0
- %lpad.val3 = insertvalue { i8*, i32 } %lpad.val, i32 %tmp2, 1
- resume { i8*, i32 } %lpad.val3
-}
-
-declare i32 @__gxx_personality_v0(...)
-
-declare i32 @llvm.eh.typeid.for(i8*)
-
-declare i8* @__cxa_begin_catch(i8*)
-
-declare void @__cxa_end_catch()
-
+++ /dev/null
-; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-; CHECK: @__llvm_profile_name__Z15test_nested_foriii = private constant [22 x i8] c"_Z15test_nested_foriii"
-
-define i32 @_Z15test_nested_foriii(i32 %r, i32 %s, i32 %t) {
-entry:
- br label %for.cond
-
-for.cond:
- %i.0 = phi i32 [ 0, %entry ], [ %inc12, %for.inc11 ]
- %nested_for_sum.0 = phi i32 [ 1, %entry ], [ %nested_for_sum.1, %for.inc11 ]
- %cmp = icmp slt i32 %i.0, %r
- br i1 %cmp, label %for.body, label %for.end13
-
-for.body:
- br label %for.cond1
-
-for.cond1:
- %j.0 = phi i32 [ 0, %for.body ], [ %inc9, %for.inc8 ]
- %nested_for_sum.1 = phi i32 [ %nested_for_sum.0, %for.body ], [ %nested_for_sum.2, %for.inc8 ]
- %cmp2 = icmp slt i32 %j.0, %s
- br i1 %cmp2, label %for.body3, label %for.end10
-
-for.body3:
- br label %for.cond4
-
-for.cond4:
- %k.0 = phi i32 [ 0, %for.body3 ], [ %inc7, %for.inc ]
- %nested_for_sum.2 = phi i32 [ %nested_for_sum.1, %for.body3 ], [ %inc, %for.inc ]
- %cmp5 = icmp slt i32 %k.0, %t
- br i1 %cmp5, label %for.body6, label %for.end
-
-for.body6:
- %inc = add nsw i32 %nested_for_sum.2, 1
- br label %for.inc
-
-for.inc:
- %inc7 = add nsw i32 %k.0, 1
- br label %for.cond4
-
-for.end:
- br label %for.inc8
-
-for.inc8:
- %inc9 = add nsw i32 %j.0, 1
- br label %for.cond1
-
-for.end10:
- br label %for.inc11
-
-for.inc11:
- %inc12 = add nsw i32 %i.0, 1
- br label %for.cond
-
-for.end13:
- ret i32 %nested_for_sum.0
-}
+++ /dev/null
-; RUN: llvm-profdata merge %S/Inputs/loop3.proftext -o %T/loop3.profdata
-; RUN: opt < %s -pgo-instr-use -pgo-profile-file=%T/loop3.profdata -S | FileCheck %s
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-define i32 @_Z15test_nested_foriii(i32 %r, i32 %s, i32 %t) {
-entry:
- br label %for.cond
-
-for.cond:
- %i.0 = phi i32 [ 0, %entry ], [ %inc12, %for.inc11 ]
- %nested_for_sum.0 = phi i32 [ 1, %entry ], [ %nested_for_sum.1, %for.inc11 ]
- %cmp = icmp slt i32 %i.0, %r
- br i1 %cmp, label %for.body, label %for.end13
-; CHECK: !prof
-
-for.body:
- br label %for.cond1
-
-for.cond1:
- %j.0 = phi i32 [ 0, %for.body ], [ %inc9, %for.inc8 ]
- %nested_for_sum.1 = phi i32 [ %nested_for_sum.0, %for.body ], [ %nested_for_sum.2, %for.inc8 ]
- %cmp2 = icmp slt i32 %j.0, %s
- br i1 %cmp2, label %for.body3, label %for.end10
-; CHECK: !prof
-
-for.body3:
- br label %for.cond4
-
-for.cond4:
- %k.0 = phi i32 [ 0, %for.body3 ], [ %inc7, %for.inc ]
- %nested_for_sum.2 = phi i32 [ %nested_for_sum.1, %for.body3 ], [ %inc, %for.inc ]
- %cmp5 = icmp slt i32 %k.0, %t
- br i1 %cmp5, label %for.body6, label %for.end
-; CHECK: !prof
-
-for.body6:
- %inc = add nsw i32 %nested_for_sum.2, 1
- br label %for.inc
-
-for.inc:
- %inc7 = add nsw i32 %k.0, 1
- br label %for.cond4
-
-for.end:
- br label %for.inc8
-
-for.inc8:
- %inc9 = add nsw i32 %j.0, 1
- br label %for.cond1
-
-for.end10:
- br label %for.inc11
-
-for.inc11:
- %inc12 = add nsw i32 %i.0, 1
- br label %for.cond
-
-for.end13:
- ret i32 %nested_for_sum.0
-}
-
-; CHECK: !{!"branch_weights"
+++ /dev/null
-; RUN: not opt < %s -pgo-instr-use -pgo-profile-file=%T/notexisting.profdata -S
-
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-define i32 @_Z9single_bbv() {
-entry:
- ret i32 0
-}
+++ /dev/null
-; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-; CHECK: @__llvm_profile_name__Z13test_switch_1i = private constant [18 x i8] c"_Z13test_switch_1i"
-
-define i32 @_Z13test_switch_1i(i32 %i) {
-entry:
- switch i32 %i, label %sw.default [
- i32 1, label %sw.bb
- i32 2, label %sw.bb1
- i32 3, label %sw.bb3
- ]
-
-sw.bb:
- %add = add nsw i32 %i, 2
- br label %sw.epilog
-
-sw.bb1:
- %add2 = add nsw i32 %i, 100
- br label %sw.epilog
-
-sw.bb3:
- %add4 = add nsw i32 %i, 4
- br label %sw.epilog
-
-sw.default:
- %add5 = add nsw i32 %i, 1
- br label %sw.epilog
-
-sw.epilog:
- %retv = phi i32 [ %add5, %sw.default ], [ %add4, %sw.bb3 ], [ %add2, %sw.bb1 ], [ %add, %sw.bb ]
- ret i32 %retv
-}
+++ /dev/null
-; RUN: llvm-profdata merge %S/Inputs/switch.proftext -o %T/switch.profdata
-; RUN: opt < %s -pgo-instr-use -pgo-profile-file=%T/switch.profdata -S | FileCheck %s
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-define i32 @_Z13test_switch_1i(i32 %i) {
-entry:
- switch i32 %i, label %sw.default [
- i32 1, label %sw.bb
- i32 2, label %sw.bb1
- i32 3, label %sw.bb3
- ]
-; CHECK: !prof !0
-
-sw.bb:
- %add = add nsw i32 %i, 2
- br label %sw.epilog
-
-sw.bb1:
- %add2 = add nsw i32 %i, 100
- br label %sw.epilog
-
-sw.bb3:
- %add4 = add nsw i32 %i, 4
- br label %sw.epilog
-
-sw.default:
- %add5 = add nsw i32 %i, 1
- br label %sw.epilog
-
-sw.epilog:
- %retv = phi i32 [ %add5, %sw.default ], [ %add4, %sw.bb3 ], [ %add2, %sw.bb1 ], [ %add, %sw.bb ]
- ret i32 %retv
-}
-
-; CHECK: !0 = !{!"branch_weights"
projects / oota-llvm.git / commitdiff