1 //===- BranchProbability.h - Branch Probability Wrapper ---------*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // Definition of BranchProbability shared by IR and Machine Instructions.
12 //===----------------------------------------------------------------------===//
14 #ifndef LLVM_SUPPORT_BRANCHPROBABILITY_H
15 #define LLVM_SUPPORT_BRANCHPROBABILITY_H
17 #include "llvm/Support/DataTypes.h"
27 // This class represents Branch Probability as a non-negative fraction that is
28 // no greater than 1. It uses a fixed-point-like implementation, in which the
29 // denominator is always a constant value (here we use 1<<31 for maximum
31 class BranchProbability {
35 // Denominator, which is a constant value.
36 static const uint32_t D = 1u << 31;
37 static const uint32_t UnknownN = UINT32_MAX;
39 // Construct a BranchProbability with only numerator assuming the denominator
40 // is 1<<31. For internal use only.
41 explicit BranchProbability(uint32_t n) : N(n) {}
44 BranchProbability() : N(UnknownN) {}
45 BranchProbability(uint32_t Numerator, uint32_t Denominator);
47 bool isZero() const { return N == 0; }
48 bool isUnknown() const { return N == UnknownN; }
50 static BranchProbability getZero() { return BranchProbability(0); }
51 static BranchProbability getOne() { return BranchProbability(D); }
52 static BranchProbability getUnknown() { return BranchProbability(UnknownN); }
53 // Create a BranchProbability object with the given numerator and 1<<31
55 static BranchProbability getRaw(uint32_t N) { return BranchProbability(N); }
56 // Create a BranchProbability object from 64-bit integers.
57 static BranchProbability getBranchProbability(uint64_t Numerator,
58 uint64_t Denominator);
60 // Normalize given probabilties so that the sum of them becomes approximate
62 template <class ProbabilityIter>
63 static void normalizeProbabilities(ProbabilityIter Begin,
66 // Normalize a list of weights by scaling them down so that the sum of them
67 // doesn't exceed UINT32_MAX.
68 template <class WeightListIter>
69 static void normalizeEdgeWeights(WeightListIter Begin, WeightListIter End);
71 uint32_t getNumerator() const { return N; }
72 static uint32_t getDenominator() { return D; }
74 // Return (1 - Probability).
75 BranchProbability getCompl() const { return BranchProbability(D - N); }
77 raw_ostream &print(raw_ostream &OS) const;
81 /// \brief Scale a large integer.
83 /// Scales \c Num. Guarantees full precision. Returns the floor of the
86 /// \return \c Num times \c this.
87 uint64_t scale(uint64_t Num) const;
89 /// \brief Scale a large integer by the inverse.
91 /// Scales \c Num by the inverse of \c this. Guarantees full precision.
92 /// Returns the floor of the result.
94 /// \return \c Num divided by \c this.
95 uint64_t scaleByInverse(uint64_t Num) const;
97 BranchProbability &operator+=(BranchProbability RHS) {
98 assert(N != UnknownN && RHS.N != UnknownN &&
99 "Unknown probability cannot participate in arithmetics.");
100 // Saturate the result in case of overflow.
101 N = (uint64_t(N) + RHS.N > D) ? D : N + RHS.N;
105 BranchProbability &operator-=(BranchProbability RHS) {
106 assert(N != UnknownN && RHS.N != UnknownN &&
107 "Unknown probability cannot participate in arithmetics.");
108 // Saturate the result in case of underflow.
109 N = N < RHS.N ? 0 : N - RHS.N;
113 BranchProbability &operator*=(BranchProbability RHS) {
114 assert(N != UnknownN && RHS.N != UnknownN &&
115 "Unknown probability cannot participate in arithmetics.");
116 N = (static_cast<uint64_t>(N) * RHS.N + D / 2) / D;
120 BranchProbability operator+(BranchProbability RHS) const {
121 BranchProbability Prob(*this);
125 BranchProbability operator-(BranchProbability RHS) const {
126 BranchProbability Prob(*this);
130 BranchProbability operator*(BranchProbability RHS) const {
131 BranchProbability Prob(*this);
135 bool operator==(BranchProbability RHS) const { return N == RHS.N; }
136 bool operator!=(BranchProbability RHS) const { return !(*this == RHS); }
138 bool operator<(BranchProbability RHS) const {
139 assert(N != UnknownN && RHS.N != UnknownN &&
140 "Unknown probability cannot participate in comparisons.");
144 bool operator>(BranchProbability RHS) const {
145 assert(N != UnknownN && RHS.N != UnknownN &&
146 "Unknown probability cannot participate in comparisons.");
150 bool operator<=(BranchProbability RHS) const {
151 assert(N != UnknownN && RHS.N != UnknownN &&
152 "Unknown probability cannot participate in comparisons.");
153 return !(RHS < *this);
156 bool operator>=(BranchProbability RHS) const {
157 assert(N != UnknownN && RHS.N != UnknownN &&
158 "Unknown probability cannot participate in comparisons.");
159 return !(*this < RHS);
163 inline raw_ostream &operator<<(raw_ostream &OS, BranchProbability Prob) {
164 return Prob.print(OS);
167 inline BranchProbability operator/(BranchProbability LHS, uint32_t RHS) {
168 assert(LHS != BranchProbability::getUnknown() &&
169 "Unknown probability cannot participate in arithmetics.");
170 return BranchProbability::getRaw(LHS.getNumerator() / RHS);
173 template <class ProbabilityIter>
174 void BranchProbability::normalizeProbabilities(ProbabilityIter Begin,
175 ProbabilityIter End) {
179 auto UnknownProbCount =
180 std::count(Begin, End, BranchProbability::getUnknown());
181 assert((UnknownProbCount == 0 ||
182 UnknownProbCount == std::distance(Begin, End)) &&
183 "Cannot normalize probabilities with known and unknown ones.");
184 (void)UnknownProbCount;
186 uint64_t Sum = std::accumulate(
187 Begin, End, uint64_t(0),
188 [](uint64_t S, const BranchProbability &BP) { return S + BP.N; });
191 BranchProbability BP(1, std::distance(Begin, End));
192 std::fill(Begin, End, BP);
196 for (auto I = Begin; I != End; ++I)
197 I->N = (I->N * uint64_t(D) + Sum / 2) / Sum;
200 template <class WeightListIter>
201 void BranchProbability::normalizeEdgeWeights(WeightListIter Begin,
202 WeightListIter End) {
203 // First we compute the sum with 64-bits of precision.
204 uint64_t Sum = std::accumulate(Begin, End, uint64_t(0));
206 if (Sum > UINT32_MAX) {
207 // Compute the scale necessary to cause the weights to fit, and re-sum with
208 // that scale applied.
209 assert(Sum / UINT32_MAX < UINT32_MAX &&
210 "The sum of weights exceeds UINT32_MAX^2!");
211 uint32_t Scale = Sum / UINT32_MAX + 1;
212 for (auto I = Begin; I != End; ++I)
214 Sum = std::accumulate(Begin, End, uint64_t(0));
217 // Eliminate zero weights.
218 auto ZeroWeightNum = std::count(Begin, End, 0u);
219 if (ZeroWeightNum > 0) {
220 // If all weights are zeros, replace them by 1.
222 std::fill(Begin, End, 1u);
224 // We are converting zeros into ones, and here we need to make sure that
225 // after this the sum won't exceed UINT32_MAX.
226 if (Sum + ZeroWeightNum > UINT32_MAX) {
227 for (auto I = Begin; I != End; ++I)
229 ZeroWeightNum = std::count(Begin, End, 0u);
230 Sum = std::accumulate(Begin, End, uint64_t(0));
232 // Scale up non-zero weights and turn zero weights into ones.
233 uint64_t ScalingFactor = (UINT32_MAX - ZeroWeightNum) / Sum;
234 assert(ScalingFactor >= 1);
235 if (ScalingFactor > 1)
236 for (auto I = Begin; I != End; ++I)
238 std::replace(Begin, End, 0u, 1u);