From e93b8e1539857d770cdd5a815b4bc3f3eb474e51 Mon Sep 17 00:00:00 2001 From: Cong Hou Date: Tue, 22 Dec 2015 18:56:14 +0000 Subject: [PATCH] [BPI] Replace weights by probabilities in BPI. This patch removes all weight-related interfaces from BPI and replace them by probability versions. With this patch, we won't use edge weight anymore in either IR or MC passes. Edge probabilitiy is a better representation in terms of CFG update and validation. Differential revision: http://reviews.llvm.org/D15519 llvm-svn: 256263 --- .../llvm/Analysis/BranchProbabilityInfo.h | 39 +-- llvm/include/llvm/Support/BranchProbability.h | 48 --- llvm/lib/Analysis/BranchProbabilityInfo.cpp | 295 ++++++++---------- llvm/lib/CodeGen/StackProtector.cpp | 11 +- llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp | 20 +- llvm/lib/Transforms/Scalar/JumpThreading.cpp | 28 +- .../BranchProbabilityInfo/noreturn.ll | 10 +- .../JumpThreading/update-edge-weight.ll | 2 +- 8 files changed, 169 insertions(+), 284 deletions(-) diff --git a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h index 69dae5e90785..cfdf218491bd 100644 --- a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h +++ b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h @@ -84,36 +84,14 @@ public: raw_ostream &printEdgeProbability(raw_ostream &OS, const BasicBlock *Src, const BasicBlock *Dst) const; - /// \brief Get the raw edge weight calculated for the edge. + /// \brief Set the raw edge probability for the given edge. /// - /// This returns the raw edge weight. It is guaranteed to fall between 1 and - /// UINT32_MAX. Note that the raw edge weight is not meaningful in isolation. - /// This interface should be very carefully, and primarily by routines that - /// are updating the analysis by later calling setEdgeWeight. - uint32_t getEdgeWeight(const BasicBlock *Src, - unsigned IndexInSuccessors) const; - - /// \brief Get the raw edge weight calculated for the block pair. - /// - /// This returns the sum of all raw edge weights from Src to Dst. - /// It is guaranteed to fall between 1 and UINT32_MAX. - uint32_t getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const; - - uint32_t getEdgeWeight(const BasicBlock *Src, - succ_const_iterator Dst) const; - - /// \brief Set the raw edge weight for a given edge. - /// - /// This allows a pass to explicitly set the edge weight for an edge. It can - /// be used when updating the CFG to update and preserve the branch + /// This allows a pass to explicitly set the edge probability for an edge. It + /// can be used when updating the CFG to update and preserve the branch /// probability information. Read the implementation of how these edge - /// weights are calculated carefully before using! - void setEdgeWeight(const BasicBlock *Src, unsigned IndexInSuccessors, - uint32_t Weight); - - static uint32_t getBranchWeightStackProtector(bool IsLikely) { - return IsLikely ? (1u << 20) - 1 : 1; - } + /// probabilities are calculated carefully before using! + void setEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors, + BranchProbability Prob); static BranchProbability getBranchProbStackProtector(bool IsLikely) { static const BranchProbability LikelyProb((1u << 20) - 1, 1u << 20); @@ -135,7 +113,7 @@ private: // weight to just "inherit" the non-zero weight of an adjacent successor. static const uint32_t DEFAULT_WEIGHT = 16; - DenseMap Weights; + DenseMap Probs; /// \brief Track the last function we run over for printing. Function *LastF; @@ -146,9 +124,6 @@ private: /// \brief Track the set of blocks that always lead to a cold call. SmallPtrSet PostDominatedByColdCall; - /// \brief Get sum of the block successors' weights. - uint32_t getSumForBlock(const BasicBlock *BB) const; - bool calcUnreachableHeuristics(BasicBlock *BB); bool calcMetadataWeights(BasicBlock *BB); bool calcColdCallHeuristics(BasicBlock *BB); diff --git a/llvm/include/llvm/Support/BranchProbability.h b/llvm/include/llvm/Support/BranchProbability.h index 1d6a7d84fbf4..26bc888d1cab 100644 --- a/llvm/include/llvm/Support/BranchProbability.h +++ b/llvm/include/llvm/Support/BranchProbability.h @@ -63,11 +63,6 @@ public: static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End); - // Normalize a list of weights by scaling them down so that the sum of them - // doesn't exceed UINT32_MAX. - template - static void normalizeEdgeWeights(WeightListIter Begin, WeightListIter End); - uint32_t getNumerator() const { return N; } static uint32_t getDenominator() { return D; } @@ -219,49 +214,6 @@ void BranchProbability::normalizeProbabilities(ProbabilityIter Begin, I->N = (I->N * uint64_t(D) + Sum / 2) / Sum; } -template -void BranchProbability::normalizeEdgeWeights(WeightListIter Begin, - WeightListIter End) { - // First we compute the sum with 64-bits of precision. - uint64_t Sum = std::accumulate(Begin, End, uint64_t(0)); - - if (Sum > UINT32_MAX) { - // Compute the scale necessary to cause the weights to fit, and re-sum with - // that scale applied. - assert(Sum / UINT32_MAX < UINT32_MAX && - "The sum of weights exceeds UINT32_MAX^2!"); - uint32_t Scale = Sum / UINT32_MAX + 1; - for (auto I = Begin; I != End; ++I) - *I /= Scale; - Sum = std::accumulate(Begin, End, uint64_t(0)); - } - - // Eliminate zero weights. - auto ZeroWeightNum = std::count(Begin, End, 0u); - if (ZeroWeightNum > 0) { - // If all weights are zeros, replace them by 1. - if (Sum == 0) - std::fill(Begin, End, 1u); - else { - // We are converting zeros into ones, and here we need to make sure that - // after this the sum won't exceed UINT32_MAX. - if (Sum + ZeroWeightNum > UINT32_MAX) { - for (auto I = Begin; I != End; ++I) - *I /= 2; - ZeroWeightNum = std::count(Begin, End, 0u); - Sum = std::accumulate(Begin, End, uint64_t(0)); - } - // Scale up non-zero weights and turn zero weights into ones. - uint64_t ScalingFactor = (UINT32_MAX - ZeroWeightNum) / Sum; - assert(ScalingFactor >= 1); - if (ScalingFactor > 1) - for (auto I = Begin; I != End; ++I) - *I *= ScalingFactor; - std::replace(Begin, End, 0u, 1u); - } - } -} - } #endif diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp index a0d6123b5835..c7c2f436e8c4 100644 --- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp +++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp @@ -108,13 +108,6 @@ static const uint32_t IH_TAKEN_WEIGHT = 1024 * 1024 - 1; /// instruction. This is essentially never taken. static const uint32_t IH_NONTAKEN_WEIGHT = 1; -// Standard weight value. Used when none of the heuristics set weight for -// the edge. -static const uint32_t NORMAL_WEIGHT = 16; - -// Minimum weight of an edge. Please note, that weight is NEVER 0. -static const uint32_t MIN_WEIGHT = 1; - /// \brief Calculate edge weights for successors lead to unreachable. /// /// Predict that a successor which leads necessarily to an @@ -157,22 +150,24 @@ bool BranchProbabilityInfo::calcUnreachableHeuristics(BasicBlock *BB) { return false; } - uint32_t UnreachableWeight = - std::max(UR_TAKEN_WEIGHT / (unsigned)UnreachableEdges.size(), MIN_WEIGHT); - for (SmallVectorImpl::iterator I = UnreachableEdges.begin(), - E = UnreachableEdges.end(); - I != E; ++I) - setEdgeWeight(BB, *I, UnreachableWeight); - - if (ReachableEdges.empty()) + if (ReachableEdges.empty()) { + BranchProbability Prob(1, UnreachableEdges.size()); + for (unsigned SuccIdx : UnreachableEdges) + setEdgeProbability(BB, SuccIdx, Prob); return true; - uint32_t ReachableWeight = - std::max(UR_NONTAKEN_WEIGHT / (unsigned)ReachableEdges.size(), - NORMAL_WEIGHT); - for (SmallVectorImpl::iterator I = ReachableEdges.begin(), - E = ReachableEdges.end(); - I != E; ++I) - setEdgeWeight(BB, *I, ReachableWeight); + } + + BranchProbability UnreachableProb(UR_TAKEN_WEIGHT, + (UR_TAKEN_WEIGHT + UR_NONTAKEN_WEIGHT) * + UnreachableEdges.size()); + BranchProbability ReachableProb(UR_NONTAKEN_WEIGHT, + (UR_TAKEN_WEIGHT + UR_NONTAKEN_WEIGHT) * + ReachableEdges.size()); + + for (unsigned SuccIdx : UnreachableEdges) + setEdgeProbability(BB, SuccIdx, UnreachableProb); + for (unsigned SuccIdx : ReachableEdges) + setEdgeProbability(BB, SuccIdx, ReachableProb); return true; } @@ -223,10 +218,12 @@ bool BranchProbabilityInfo::calcMetadataWeights(BasicBlock *BB) { WeightSum = 0; for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) { - uint32_t W = Weights[i] / ScalingFactor; - WeightSum += W; - setEdgeWeight(BB, i, W); + Weights[i] /= ScalingFactor; + WeightSum += Weights[i]; } + for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) + setEdgeProbability(BB, i, {Weights[i], static_cast(WeightSum)}); + assert(WeightSum <= UINT32_MAX && "Expected weights to scale down to 32 bits"); @@ -275,21 +272,24 @@ bool BranchProbabilityInfo::calcColdCallHeuristics(BasicBlock *BB) { if (TI->getNumSuccessors() == 1 || ColdEdges.empty()) return false; - uint32_t ColdWeight = - std::max(CC_TAKEN_WEIGHT / (unsigned) ColdEdges.size(), MIN_WEIGHT); - for (SmallVectorImpl::iterator I = ColdEdges.begin(), - E = ColdEdges.end(); - I != E; ++I) - setEdgeWeight(BB, *I, ColdWeight); - - if (NormalEdges.empty()) + if (NormalEdges.empty()) { + BranchProbability Prob(1, ColdEdges.size()); + for (unsigned SuccIdx : ColdEdges) + setEdgeProbability(BB, SuccIdx, Prob); return true; - uint32_t NormalWeight = std::max( - CC_NONTAKEN_WEIGHT / (unsigned) NormalEdges.size(), NORMAL_WEIGHT); - for (SmallVectorImpl::iterator I = NormalEdges.begin(), - E = NormalEdges.end(); - I != E; ++I) - setEdgeWeight(BB, *I, NormalWeight); + } + + BranchProbability ColdProb(CC_TAKEN_WEIGHT, + (CC_TAKEN_WEIGHT + CC_NONTAKEN_WEIGHT) * + ColdEdges.size()); + BranchProbability NormalProb(CC_NONTAKEN_WEIGHT, + (CC_TAKEN_WEIGHT + CC_NONTAKEN_WEIGHT) * + NormalEdges.size()); + + for (unsigned SuccIdx : ColdEdges) + setEdgeProbability(BB, SuccIdx, ColdProb); + for (unsigned SuccIdx : NormalEdges) + setEdgeProbability(BB, SuccIdx, NormalProb); return true; } @@ -322,8 +322,10 @@ bool BranchProbabilityInfo::calcPointerHeuristics(BasicBlock *BB) { if (!isProb) std::swap(TakenIdx, NonTakenIdx); - setEdgeWeight(BB, TakenIdx, PH_TAKEN_WEIGHT); - setEdgeWeight(BB, NonTakenIdx, PH_NONTAKEN_WEIGHT); + BranchProbability TakenProb(PH_TAKEN_WEIGHT, + PH_TAKEN_WEIGHT + PH_NONTAKEN_WEIGHT); + setEdgeProbability(BB, TakenIdx, TakenProb); + setEdgeProbability(BB, NonTakenIdx, TakenProb.getCompl()); return true; } @@ -351,37 +353,35 @@ bool BranchProbabilityInfo::calcLoopBranchHeuristics(BasicBlock *BB, if (BackEdges.empty() && ExitingEdges.empty()) return false; - if (uint32_t numBackEdges = BackEdges.size()) { - uint32_t backWeight = LBH_TAKEN_WEIGHT / numBackEdges; - if (backWeight < NORMAL_WEIGHT) - backWeight = NORMAL_WEIGHT; + // Collect the sum of probabilities of back-edges/in-edges/exiting-edges, and + // normalize them so that they sum up to one. + SmallVector Probs(3, BranchProbability::getZero()); + unsigned Denom = (BackEdges.empty() ? 0 : LBH_TAKEN_WEIGHT) + + (InEdges.empty() ? 0 : LBH_TAKEN_WEIGHT) + + (ExitingEdges.empty() ? 0 : LBH_NONTAKEN_WEIGHT); + if (!BackEdges.empty()) + Probs[0] = BranchProbability(LBH_TAKEN_WEIGHT, Denom); + if (!InEdges.empty()) + Probs[1] = BranchProbability(LBH_TAKEN_WEIGHT, Denom); + if (!ExitingEdges.empty()) + Probs[2] = BranchProbability(LBH_NONTAKEN_WEIGHT, Denom); - for (SmallVectorImpl::iterator EI = BackEdges.begin(), - EE = BackEdges.end(); EI != EE; ++EI) { - setEdgeWeight(BB, *EI, backWeight); - } + if (uint32_t numBackEdges = BackEdges.size()) { + auto Prob = Probs[0] / numBackEdges; + for (unsigned SuccIdx : BackEdges) + setEdgeProbability(BB, SuccIdx, Prob); } if (uint32_t numInEdges = InEdges.size()) { - uint32_t inWeight = LBH_TAKEN_WEIGHT / numInEdges; - if (inWeight < NORMAL_WEIGHT) - inWeight = NORMAL_WEIGHT; - - for (SmallVectorImpl::iterator EI = InEdges.begin(), - EE = InEdges.end(); EI != EE; ++EI) { - setEdgeWeight(BB, *EI, inWeight); - } + auto Prob = Probs[1] / numInEdges; + for (unsigned SuccIdx : InEdges) + setEdgeProbability(BB, SuccIdx, Prob); } if (uint32_t numExitingEdges = ExitingEdges.size()) { - uint32_t exitWeight = LBH_NONTAKEN_WEIGHT / numExitingEdges; - if (exitWeight < MIN_WEIGHT) - exitWeight = MIN_WEIGHT; - - for (SmallVectorImpl::iterator EI = ExitingEdges.begin(), - EE = ExitingEdges.end(); EI != EE; ++EI) { - setEdgeWeight(BB, *EI, exitWeight); - } + auto Prob = Probs[2] / numExitingEdges; + for (unsigned SuccIdx : ExitingEdges) + setEdgeProbability(BB, SuccIdx, Prob); } return true; @@ -463,9 +463,10 @@ bool BranchProbabilityInfo::calcZeroHeuristics(BasicBlock *BB) { if (!isProb) std::swap(TakenIdx, NonTakenIdx); - setEdgeWeight(BB, TakenIdx, ZH_TAKEN_WEIGHT); - setEdgeWeight(BB, NonTakenIdx, ZH_NONTAKEN_WEIGHT); - + BranchProbability TakenProb(ZH_TAKEN_WEIGHT, + ZH_TAKEN_WEIGHT + ZH_NONTAKEN_WEIGHT); + setEdgeProbability(BB, TakenIdx, TakenProb); + setEdgeProbability(BB, NonTakenIdx, TakenProb.getCompl()); return true; } @@ -499,9 +500,10 @@ bool BranchProbabilityInfo::calcFloatingPointHeuristics(BasicBlock *BB) { if (!isProb) std::swap(TakenIdx, NonTakenIdx); - setEdgeWeight(BB, TakenIdx, FPH_TAKEN_WEIGHT); - setEdgeWeight(BB, NonTakenIdx, FPH_NONTAKEN_WEIGHT); - + BranchProbability TakenProb(FPH_TAKEN_WEIGHT, + FPH_TAKEN_WEIGHT + FPH_NONTAKEN_WEIGHT); + setEdgeProbability(BB, TakenIdx, TakenProb); + setEdgeProbability(BB, NonTakenIdx, TakenProb.getCompl()); return true; } @@ -510,13 +512,15 @@ bool BranchProbabilityInfo::calcInvokeHeuristics(BasicBlock *BB) { if (!II) return false; - setEdgeWeight(BB, 0/*Index for Normal*/, IH_TAKEN_WEIGHT); - setEdgeWeight(BB, 1/*Index for Unwind*/, IH_NONTAKEN_WEIGHT); + BranchProbability TakenProb(IH_TAKEN_WEIGHT, + IH_TAKEN_WEIGHT + IH_NONTAKEN_WEIGHT); + setEdgeProbability(BB, 0 /*Index for Normal*/, TakenProb); + setEdgeProbability(BB, 1 /*Index for Unwind*/, TakenProb.getCompl()); return true; } void BranchProbabilityInfo::releaseMemory() { - Weights.clear(); + Probs.clear(); } void BranchProbabilityInfo::print(raw_ostream &OS) const { @@ -532,20 +536,6 @@ void BranchProbabilityInfo::print(raw_ostream &OS) const { } } -uint32_t BranchProbabilityInfo::getSumForBlock(const BasicBlock *BB) const { - uint32_t Sum = 0; - - for (succ_const_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) { - uint32_t Weight = getEdgeWeight(BB, I.getSuccessorIndex()); - uint32_t PrevSum = Sum; - - Sum += Weight; - assert(Sum >= PrevSum); (void) PrevSum; - } - - return Sum; -} - bool BranchProbabilityInfo:: isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const { // Hot probability is at least 4/5 = 80% @@ -554,107 +544,39 @@ isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const { } BasicBlock *BranchProbabilityInfo::getHotSucc(BasicBlock *BB) const { - uint32_t Sum = 0; - uint32_t MaxWeight = 0; + auto MaxProb = BranchProbability::getZero(); BasicBlock *MaxSucc = nullptr; for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) { BasicBlock *Succ = *I; - uint32_t Weight = getEdgeWeight(BB, Succ); - uint32_t PrevSum = Sum; - - Sum += Weight; - assert(Sum > PrevSum); (void) PrevSum; - - if (Weight > MaxWeight) { - MaxWeight = Weight; + auto Prob = getEdgeProbability(BB, Succ); + if (Prob > MaxProb) { + MaxProb = Prob; MaxSucc = Succ; } } // Hot probability is at least 4/5 = 80% - if (BranchProbability(MaxWeight, Sum) > BranchProbability(4, 5)) + if (MaxProb > BranchProbability(4, 5)) return MaxSucc; return nullptr; } -/// Get the raw edge weight for the edge. If can't find it, return -/// DEFAULT_WEIGHT value. Here an edge is specified using PredBlock and an index -/// to the successors. -uint32_t BranchProbabilityInfo:: -getEdgeWeight(const BasicBlock *Src, unsigned IndexInSuccessors) const { - DenseMap::const_iterator I = - Weights.find(std::make_pair(Src, IndexInSuccessors)); +/// Get the raw edge probability for the edge. If can't find it, return a +/// default probability 1/N where N is the number of successors. Here an edge is +/// specified using PredBlock and an +/// index to the successors. +BranchProbability +BranchProbabilityInfo::getEdgeProbability(const BasicBlock *Src, + unsigned IndexInSuccessors) const { + auto I = Probs.find(std::make_pair(Src, IndexInSuccessors)); - if (I != Weights.end()) + if (I != Probs.end()) return I->second; - return DEFAULT_WEIGHT; -} - -uint32_t BranchProbabilityInfo::getEdgeWeight(const BasicBlock *Src, - succ_const_iterator Dst) const { - return getEdgeWeight(Src, Dst.getSuccessorIndex()); -} - -/// Get the raw edge weight calculated for the block pair. This returns the sum -/// of all raw edge weights from Src to Dst. -uint32_t BranchProbabilityInfo:: -getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const { - uint32_t Weight = 0; - bool FoundWeight = false; - DenseMap::const_iterator MapI; - for (succ_const_iterator I = succ_begin(Src), E = succ_end(Src); I != E; ++I) - if (*I == Dst) { - MapI = Weights.find(std::make_pair(Src, I.getSuccessorIndex())); - if (MapI != Weights.end()) { - FoundWeight = true; - Weight += MapI->second; - } - } - return (!FoundWeight) ? DEFAULT_WEIGHT : Weight; -} - -/// Set the edge weight for a given edge specified by PredBlock and an index -/// to the successors. -void BranchProbabilityInfo:: -setEdgeWeight(const BasicBlock *Src, unsigned IndexInSuccessors, - uint32_t Weight) { - Weights[std::make_pair(Src, IndexInSuccessors)] = Weight; - DEBUG(dbgs() << "set edge " << Src->getName() << " -> " - << IndexInSuccessors << " successor weight to " - << Weight << "\n"); -} - -/// Get an edge's probability, relative to other out-edges from Src. -BranchProbability BranchProbabilityInfo:: -getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const { - uint32_t N = getEdgeWeight(Src, IndexInSuccessors); - uint32_t D = getSumForBlock(Src); - - // It is possible that the edge weight on the only successor edge of Src is - // zero, in which case we return 100%. - if (N == 0 && D == 0) - return BranchProbability::getOne(); - - return BranchProbability(N, D); -} - -/// Get the probability of going from Src to Dst. It returns the sum of all -/// probabilities for edges from Src to Dst. -BranchProbability BranchProbabilityInfo:: -getEdgeProbability(const BasicBlock *Src, const BasicBlock *Dst) const { - - uint32_t N = getEdgeWeight(Src, Dst); - uint32_t D = getSumForBlock(Src); - - // It is possible that the edge weight on the only successor edge of Src is - // zero, in which case we return 100%. - if (N == 0 && D == 0) - return BranchProbability::getOne(); - - return BranchProbability(N, D); + return {1, + static_cast(std::distance(succ_begin(Src), succ_end(Src)))}; } BranchProbability @@ -663,6 +585,35 @@ BranchProbabilityInfo::getEdgeProbability(const BasicBlock *Src, return getEdgeProbability(Src, Dst.getSuccessorIndex()); } +/// Get the raw edge probability calculated for the block pair. This returns the +/// sum of all raw edge probabilities from Src to Dst. +BranchProbability +BranchProbabilityInfo::getEdgeProbability(const BasicBlock *Src, + const BasicBlock *Dst) const { + auto Prob = BranchProbability::getZero(); + bool FoundProb = false; + for (succ_const_iterator I = succ_begin(Src), E = succ_end(Src); I != E; ++I) + if (*I == Dst) { + auto MapI = Probs.find(std::make_pair(Src, I.getSuccessorIndex())); + if (MapI != Probs.end()) { + FoundProb = true; + Prob += MapI->second; + } + } + uint32_t succ_num = std::distance(succ_begin(Src), succ_end(Src)); + return FoundProb ? Prob : BranchProbability(1, succ_num); +} + +/// Set the edge probability for a given edge specified by PredBlock and an +/// index to the successors. +void BranchProbabilityInfo::setEdgeProbability(const BasicBlock *Src, + unsigned IndexInSuccessors, + BranchProbability Prob) { + Probs[std::make_pair(Src, IndexInSuccessors)] = Prob; + DEBUG(dbgs() << "set edge " << Src->getName() << " -> " << IndexInSuccessors + << " successor probability to " << Prob << "\n"); +} + raw_ostream & BranchProbabilityInfo::printEdgeProbability(raw_ostream &OS, const BasicBlock *Src, diff --git a/llvm/lib/CodeGen/StackProtector.cpp b/llvm/lib/CodeGen/StackProtector.cpp index ee1a20451903..db3fef524b30 100644 --- a/llvm/lib/CodeGen/StackProtector.cpp +++ b/llvm/lib/CodeGen/StackProtector.cpp @@ -453,12 +453,13 @@ bool StackProtector::InsertStackProtectors() { LoadInst *LI1 = B.CreateLoad(StackGuardVar); LoadInst *LI2 = B.CreateLoad(AI); Value *Cmp = B.CreateICmpEQ(LI1, LI2); - unsigned SuccessWeight = - BranchProbabilityInfo::getBranchWeightStackProtector(true); - unsigned FailureWeight = - BranchProbabilityInfo::getBranchWeightStackProtector(false); + auto SuccessProb = + BranchProbabilityInfo::getBranchProbStackProtector(true); + auto FailureProb = + BranchProbabilityInfo::getBranchProbStackProtector(false); MDNode *Weights = MDBuilder(F->getContext()) - .createBranchWeights(SuccessWeight, FailureWeight); + .createBranchWeights(SuccessProb.getNumerator(), + FailureProb.getNumerator()); B.CreateCondBr(Cmp, NewBB, FailBB, Weights); } } diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp index d57a070bcd52..a2d588c8c264 100644 --- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp @@ -414,8 +414,8 @@ static unsigned getBranchHint(unsigned PCC, FunctionLoweringInfo *FuncInfo, const BasicBlock *TBB = BBTerm->getSuccessor(0); const BasicBlock *FBB = BBTerm->getSuccessor(1); - uint32_t TWeight = FuncInfo->BPI->getEdgeWeight(BB, TBB); - uint32_t FWeight = FuncInfo->BPI->getEdgeWeight(BB, FBB); + auto TProb = FuncInfo->BPI->getEdgeProbability(BB, TBB); + auto FProb = FuncInfo->BPI->getEdgeProbability(BB, FBB); // We only want to handle cases which are easy to predict at static time, e.g. // C++ throw statement, that is very likely not taken, or calling never @@ -432,24 +432,22 @@ static unsigned getBranchHint(unsigned PCC, FunctionLoweringInfo *FuncInfo, // 5. PH/ZH/FPH 20:12 const uint32_t Threshold = 10000; - // Minimal weight should be at least 1 - if (std::max(TWeight, FWeight) / - std::max(1u, std::min(TWeight, FWeight)) < Threshold) + if (std::max(TProb, FProb) / Threshold < std::min(TProb, FProb)) return PPC::BR_NO_HINT; DEBUG(dbgs() << "Use branch hint for '" << FuncInfo->Fn->getName() << "::" << BB->getName() << "'\n" - << " -> " << TBB->getName() << ": " << TWeight << "\n" - << " -> " << FBB->getName() << ": " << FWeight << "\n"); + << " -> " << TBB->getName() << ": " << TProb << "\n" + << " -> " << FBB->getName() << ": " << FProb << "\n"); const BasicBlockSDNode *BBDN = cast(DestMBB); - // If Dest BasicBlock is False-BasicBlock (FBB), swap branch weight, - // because we want 'TWeight' stands for 'branch weight' to Dest BasicBlock + // If Dest BasicBlock is False-BasicBlock (FBB), swap branch probabilities, + // because we want 'TProb' stands for 'branch probability' to Dest BasicBlock if (BBDN->getBasicBlock()->getBasicBlock() != TBB) - std::swap(TWeight, FWeight); + std::swap(TProb, FProb); - return (TWeight > FWeight) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT; + return (TProb > FProb) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT; } // isOpcWithIntImmediate - This method tests to see if the node is a specific diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp index 6ab00ee2b19f..d30c336be212 100644 --- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp +++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp @@ -1636,7 +1636,7 @@ void JumpThreading::UpdateBlockFreqAndEdgeWeight(BasicBlock *PredBB, BFI->setBlockFreq(BB, BBNewFreq.getFrequency()); // Collect updated outgoing edges' frequencies from BB and use them to update - // edge weights. + // edge probabilities. SmallVector BBSuccFreq; for (auto I = succ_begin(BB), E = succ_end(BB); I != E; ++I) { auto SuccFreq = (*I == SuccBB) @@ -1645,18 +1645,26 @@ void JumpThreading::UpdateBlockFreqAndEdgeWeight(BasicBlock *PredBB, BBSuccFreq.push_back(SuccFreq.getFrequency()); } - // Normalize edge weights in Weights64 so that the sum of them can fit in - BranchProbability::normalizeEdgeWeights(BBSuccFreq.begin(), BBSuccFreq.end()); + uint64_t MaxBBSuccFreq = + *std::max_element(BBSuccFreq.begin(), BBSuccFreq.end()); + SmallVector BBSuccProbs; + for (uint64_t Freq : BBSuccFreq) + BBSuccProbs.push_back( + BranchProbability::getBranchProbability(Freq, MaxBBSuccFreq)); - SmallVector Weights; - for (auto Freq : BBSuccFreq) - Weights.push_back(static_cast(Freq)); + // Normalize edge probabilities so that they sum up to one. + BranchProbability::normalizeProbabilities(BBSuccProbs.begin(), + BBSuccProbs.end()); - // Update edge weights in BPI. - for (int I = 0, E = Weights.size(); I < E; I++) - BPI->setEdgeWeight(BB, I, Weights[I]); + // Update edge probabilities in BPI. + for (int I = 0, E = BBSuccProbs.size(); I < E; I++) + BPI->setEdgeProbability(BB, I, BBSuccProbs[I]); + + if (BBSuccProbs.size() >= 2) { + SmallVector Weights; + for (auto Prob : BBSuccProbs) + Weights.push_back(Prob.getNumerator()); - if (Weights.size() >= 2) { auto TI = BB->getTerminator(); TI->setMetadata( LLVMContext::MD_prof, diff --git a/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll b/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll index 513818d8da6a..7098c2f7b8cc 100644 --- a/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll @@ -26,11 +26,11 @@ entry: i32 2, label %case_b i32 3, label %case_c i32 4, label %case_d] -; CHECK: edge entry -> exit probability is 0x7fffe000 / 0x80000000 = 100.00% [HOT edge] -; CHECK: edge entry -> case_a probability is 0x00000800 / 0x80000000 = 0.00% -; CHECK: edge entry -> case_b probability is 0x00000800 / 0x80000000 = 0.00% -; CHECK: edge entry -> case_c probability is 0x00000800 / 0x80000000 = 0.00% -; CHECK: edge entry -> case_d probability is 0x00000800 / 0x80000000 = 0.00% +; CHECK: edge entry -> exit probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge] +; CHECK: edge entry -> case_a probability is 0x00000200 / 0x80000000 = 0.00% +; CHECK: edge entry -> case_b probability is 0x00000200 / 0x80000000 = 0.00% +; CHECK: edge entry -> case_c probability is 0x00000200 / 0x80000000 = 0.00% +; CHECK: edge entry -> case_d probability is 0x00000200 / 0x80000000 = 0.00% case_a: br label %case_b diff --git a/llvm/test/Transforms/JumpThreading/update-edge-weight.ll b/llvm/test/Transforms/JumpThreading/update-edge-weight.ll index b5c5d01a3c61..58cd71861d8a 100644 --- a/llvm/test/Transforms/JumpThreading/update-edge-weight.ll +++ b/llvm/test/Transforms/JumpThreading/update-edge-weight.ll @@ -2,7 +2,7 @@ ; Test if edge weights are properly updated after jump threading. -; CHECK: !2 = !{!"branch_weights", i32 22, i32 7} +; CHECK: !2 = !{!"branch_weights", i32 1629125526, i32 518358122} define void @foo(i32 %n) !prof !0 { entry: