maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Let SelectionDAG start to use probability-based interface to add successors. 

The patch in http://reviews.llvm.org/D13745 is broken into four parts:

1. New interfaces without functional changes.
2. Use new interfaces in SelectionDAG, while in other passes treat probabilities
as weights.
3. Use new interfaces in all other passes.
4. Remove old interfaces.

This the second patch above. In this patch SelectionDAG starts to use
probability-based interfaces in MBB to add successors but other MC passes are
still using weight-based interfaces. Therefore, we need to maintain correct
weight list in MBB even when probability-based interfaces are used. This is
done by updating weight list in probability-based interfaces by treating the
numerator of probabilities as weights. This change affects many test cases
that check successor weight values. I will update those test cases once this
patch looks good to you.


Differential revision: http://reviews.llvm.org/D14361

llvm-svn: 253965
This commit is contained in:
Cong Hou 2015-11-24 08:51:23 +00:00
parent 5712d46114
commit 1938f2eb98
23 changed files with 334 additions and 276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
llvm/include/llvm/Analysis/BranchProbabilityInfo.h
View File

@ -112,6 +112,11 @@ public:
return IsLikely ? (1u << 20) - 1 : 1; return IsLikely ? (1u << 20) - 1 : 1;
} }
static BranchProbability getBranchProbStackProtector(bool IsLikely) {
static const BranchProbability LikelyProb((1u << 20) - 1, 1u << 20);
return IsLikely ? LikelyProb : LikelyProb.getCompl();
}
void calculate(Function &F, const LoopInfo& LI); void calculate(Function &F, const LoopInfo& LI);
private: private:

31
llvm/include/llvm/Support/BranchProbability.h
View File

@ -41,7 +41,7 @@ class BranchProbability {
explicit BranchProbability(uint32_t n) : N(n) {} explicit BranchProbability(uint32_t n) : N(n) {}
public: public:
BranchProbability() : N(0) {} BranchProbability() : N(UnknownN) {}
BranchProbability(uint32_t Numerator, uint32_t Denominator); BranchProbability(uint32_t Numerator, uint32_t Denominator);
bool isZero() const { return N == 0; } bool isZero() const { return N == 0; }
@ -92,18 +92,24 @@ public:
uint64_t scaleByInverse(uint64_t Num) const; uint64_t scaleByInverse(uint64_t Num) const;
BranchProbability &operator+=(BranchProbability RHS) { BranchProbability &operator+=(BranchProbability RHS) {
assert(N != UnknownN && RHS.N != UnknownN &&
"Unknown probability cannot participate in arithmetics.");
// Saturate the result in case of overflow. // Saturate the result in case of overflow.
N = (uint64_t(N) + RHS.N > D) ? D : N + RHS.N; N = (uint64_t(N) + RHS.N > D) ? D : N + RHS.N;
return *this; return *this;
} }
BranchProbability &operator-=(BranchProbability RHS) { BranchProbability &operator-=(BranchProbability RHS) {
assert(N != UnknownN && RHS.N != UnknownN &&
"Unknown probability cannot participate in arithmetics.");
// Saturate the result in case of underflow. // Saturate the result in case of underflow.
N = N < RHS.N ? 0 : N - RHS.N; N = N < RHS.N ? 0 : N - RHS.N;
return *this; return *this;
} }
BranchProbability &operator*=(BranchProbability RHS) { BranchProbability &operator*=(BranchProbability RHS) {
assert(N != UnknownN && RHS.N != UnknownN &&
"Unknown probability cannot participate in arithmetics.");
N = (static_cast<uint64_t>(N) * RHS.N + D / 2) / D; N = (static_cast<uint64_t>(N) * RHS.N + D / 2) / D;
return *this; return *this;
} }
@ -136,6 +142,8 @@ inline raw_ostream &operator<<(raw_ostream &OS, BranchProbability Prob) {
} }
inline BranchProbability operator/(BranchProbability LHS, uint32_t RHS) { inline BranchProbability operator/(BranchProbability LHS, uint32_t RHS) {
assert(LHS != BranchProbability::getUnknown() &&
"Unknown probability cannot participate in arithmetics.");
return BranchProbability::getRaw(LHS.getNumerator() / RHS); return BranchProbability::getRaw(LHS.getNumerator() / RHS);
} }
@ -145,10 +153,23 @@ void BranchProbability::normalizeProbabilities(ProbabilityIter Begin,
if (Begin == End) if (Begin == End)
return; return;
uint64_t Sum = 0; auto UnknownProbCount =
for (auto I = Begin; I != End; ++I) std::count(Begin, End, BranchProbability::getUnknown());
Sum += I->N; assert((UnknownProbCount == 0 ||
assert(Sum > 0); UnknownProbCount == std::distance(Begin, End)) &&
"Cannot normalize probabilities with known and unknown ones.");
(void)UnknownProbCount;
uint64_t Sum = std::accumulate(
Begin, End, uint64_t(0),
[](uint64_t S, const BranchProbability &BP) { return S + BP.N; });
if (Sum == 0) {
BranchProbability BP(1, std::distance(Begin, End));
std::fill(Begin, End, BP);
return;
}
for (auto I = Begin; I != End; ++I) for (auto I = Begin; I != End; ++I)
I->N = (I->N * uint64_t(D) + Sum / 2) / Sum; I->N = (I->N * uint64_t(D) + Sum / 2) / Sum;
} }

25
llvm/lib/CodeGen/MachineBasicBlock.cpp
View File

@ -528,8 +528,13 @@ void MachineBasicBlock::addSuccessor(MachineBasicBlock *Succ,
BranchProbability Prob) { BranchProbability Prob) {
// Probability list is either empty (if successor list isn't empty, this means // Probability list is either empty (if successor list isn't empty, this means
// disabled optimization) or has the same size as successor list. // disabled optimization) or has the same size as successor list.
if (!(Probs.empty() && !Successors.empty())) if (!(Probs.empty() && !Successors.empty())) {
Probs.push_back(Prob); Probs.push_back(Prob);
// FIXME: Temporarily use the numerator of the probability to represent edge
// weight. This will be removed once all weight-version interfaces in MBB
// are replaced with probability-version interfaces.
Weights.push_back(Prob.getNumerator());
}
Successors.push_back(Succ); Successors.push_back(Succ);
Succ->addPredecessor(this); Succ->addPredecessor(this);
} }
@ -539,6 +544,7 @@ void MachineBasicBlock::addSuccessorWithoutProb(MachineBasicBlock *Succ) {
// of successor list. When this function is called, we can safely delete all // of successor list. When this function is called, we can safely delete all
// probability in the list. // probability in the list.
Probs.clear(); Probs.clear();
Weights.clear();
Successors.push_back(Succ); Successors.push_back(Succ);
Succ->addPredecessor(this); Succ->addPredecessor(this);
} }
@ -558,12 +564,17 @@ MachineBasicBlock::removeSuccessor(succ_iterator I) {
Weights.erase(WI); Weights.erase(WI);
} }
// FIXME: Temporarily comment the following code as probabilities are now only
// used during instruction lowering, but this interface is called in later
// passes. Uncomment it once all edge weights are replaced with probabilities.
#if 0
// If probability list is empty it means we don't use it (disabled // If probability list is empty it means we don't use it (disabled
// optimization). // optimization).
if (!Probs.empty()) { if (!Probs.empty()) {
probability_iterator WI = getProbabilityIterator(I); probability_iterator WI = getProbabilityIterator(I);
Probs.erase(WI); Probs.erase(WI);
} }
#endif
(*I)->removePredecessor(this); (*I)->removePredecessor(this);
return Successors.erase(I); return Successors.erase(I);
@ -603,10 +614,14 @@ void MachineBasicBlock::replaceSuccessor(MachineBasicBlock *Old,
// Update its weight instead of adding a duplicate edge. // Update its weight instead of adding a duplicate edge.
if (!Weights.empty()) if (!Weights.empty())
*getWeightIterator(NewI) += *getWeightIterator(OldI); *getWeightIterator(NewI) += *getWeightIterator(OldI);
// FIXME: Temporarily comment the following code as probabilities are now only
// used during instruction lowering, but this interface is called in later
// passes. Uncomment it once all edge weights are replaced with probabilities.
#if 0
// Update its probability instead of adding a duplicate edge. // Update its probability instead of adding a duplicate edge.
if (!Probs.empty()) if (!Probs.empty())
*getProbabilityIterator(NewI) += *getProbabilityIterator(OldI); *getProbabilityIterator(NewI) += *getProbabilityIterator(OldI);
#endif
removeSuccessor(OldI); removeSuccessor(OldI);
} }
@ -1165,9 +1180,13 @@ void MachineBasicBlock::setSuccWeight(succ_iterator I, uint32_t Weight) {
void MachineBasicBlock::setSuccProbability(succ_iterator I, void MachineBasicBlock::setSuccProbability(succ_iterator I,
BranchProbability Prob) { BranchProbability Prob) {
assert(!Prob.isUnknown()); assert(!Prob.isUnknown());
if (Probs.empty()) if (Probs.empty() || Weights.empty())
return; return;
*getProbabilityIterator(I) = Prob; *getProbabilityIterator(I) = Prob;
// FIXME: Temporarily use the numerator of the probability to represent edge
// weight. This will be removed once all weight-version interfaces in MBB
// are replaces with probability-version interfaces.
*getWeightIterator(I) = Prob.getNumerator();
} }
/// Return wight iterator corresonding to the I successor iterator. /// Return wight iterator corresonding to the I successor iterator.

14
llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
View File

@ -1395,11 +1395,11 @@ void FastISel::fastEmitBranch(MachineBasicBlock *MSucc, DebugLoc DbgLoc) {
SmallVector<MachineOperand, 0>(), DbgLoc); SmallVector<MachineOperand, 0>(), DbgLoc);
} }
if (FuncInfo.BPI) { if (FuncInfo.BPI) {
uint32_t BranchWeight = FuncInfo.BPI->getEdgeWeight( auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
FuncInfo.MBB->getBasicBlock(), MSucc->getBasicBlock()); FuncInfo.MBB->getBasicBlock(), MSucc->getBasicBlock());
FuncInfo.MBB->addSuccessor(MSucc, BranchWeight); FuncInfo.MBB->addSuccessor(MSucc, BranchProbability);
} else } else
FuncInfo.MBB->addSuccessorWithoutWeight(MSucc); FuncInfo.MBB->addSuccessorWithoutProb(MSucc);
} }
void FastISel::finishCondBranch(const BasicBlock *BranchBB, void FastISel::finishCondBranch(const BasicBlock *BranchBB,
@ -1410,11 +1410,11 @@ void FastISel::finishCondBranch(const BasicBlock *BranchBB,
// successor/predecessor lists. // successor/predecessor lists.
if (TrueMBB != FalseMBB) { if (TrueMBB != FalseMBB) {
if (FuncInfo.BPI) { if (FuncInfo.BPI) {
uint32_t BranchWeight = auto BranchProbability =
FuncInfo.BPI->getEdgeWeight(BranchBB, TrueMBB->getBasicBlock()); FuncInfo.BPI->getEdgeProbability(BranchBB, TrueMBB->getBasicBlock());
FuncInfo.MBB->addSuccessor(TrueMBB, BranchWeight); FuncInfo.MBB->addSuccessor(TrueMBB, BranchProbability);
} else } else
FuncInfo.MBB->addSuccessorWithoutWeight(TrueMBB); FuncInfo.MBB->addSuccessorWithoutProb(TrueMBB);
} }
fastEmitBranch(FalseMBB, DbgLoc); fastEmitBranch(FalseMBB, DbgLoc);

343
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
View File

@ -1251,11 +1251,13 @@ void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
/// This function skips over imaginary basic blocks that hold catchpad, /// This function skips over imaginary basic blocks that hold catchpad,
/// terminatepad, or catchendpad instructions, and finds all the "real" machine /// terminatepad, or catchendpad instructions, and finds all the "real" machine
/// basic block destinations. As those destinations may not be successors of /// basic block destinations. As those destinations may not be successors of
/// EHPadBB, here we also calculate the edge weight to those destinations. The /// EHPadBB, here we also calculate the edge probability to those destinations.
/// passed-in Weight is the edge weight to EHPadBB. /// The passed-in Prob is the edge probability to EHPadBB.
static void findUnwindDestinations( static void findUnwindDestinations(
FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, uint32_t Weight, FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
SmallVectorImpl<std::pair<MachineBasicBlock *, uint32_t>> &UnwindDests) { BranchProbability Prob,
SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
&UnwindDests) {
EHPersonality Personality = EHPersonality Personality =
classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX; bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
@ -1266,17 +1268,17 @@ static void findUnwindDestinations(
BasicBlock *NewEHPadBB = nullptr; BasicBlock *NewEHPadBB = nullptr;
if (isa<LandingPadInst>(Pad)) { if (isa<LandingPadInst>(Pad)) {
// Stop on landingpads. They are not funclets. // Stop on landingpads. They are not funclets.
UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Weight); UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
break; break;
} else if (isa<CleanupPadInst>(Pad)) { } else if (isa<CleanupPadInst>(Pad)) {
// Stop on cleanup pads. Cleanups are always funclet entries for all known // Stop on cleanup pads. Cleanups are always funclet entries for all known
// personalities. // personalities.
UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Weight); UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
UnwindDests.back().first->setIsEHFuncletEntry(); UnwindDests.back().first->setIsEHFuncletEntry();
break; break;
} else if (const auto *CPI = dyn_cast<CatchPadInst>(Pad)) { } else if (const auto *CPI = dyn_cast<CatchPadInst>(Pad)) {
// Add the catchpad handler to the possible destinations. // Add the catchpad handler to the possible destinations.
UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Weight); UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
// In MSVC C++, catchblocks are funclets and need prologues. // In MSVC C++, catchblocks are funclets and need prologues.
if (IsMSVCCXX || IsCoreCLR) if (IsMSVCCXX || IsCoreCLR)
UnwindDests.back().first->setIsEHFuncletEntry(); UnwindDests.back().first->setIsEHFuncletEntry();
@ -1289,28 +1291,27 @@ static void findUnwindDestinations(
continue; continue;
BranchProbabilityInfo *BPI = FuncInfo.BPI; BranchProbabilityInfo *BPI = FuncInfo.BPI;
if (BPI && NewEHPadBB) { if (BPI && NewEHPadBB)
// When BPI is available, the calculated weight cannot be zero as zero Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
// will be turned to a default weight in MachineBlockFrequencyInfo.
Weight = std::max<uint32_t>(
BPI->getEdgeProbability(EHPadBB, NewEHPadBB).scale(Weight), 1);
}
EHPadBB = NewEHPadBB; EHPadBB = NewEHPadBB;
} }
} }
void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) { void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
// Update successor info. // Update successor info.
SmallVector<std::pair<MachineBasicBlock *, uint32_t>, 1> UnwindDests; SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
auto UnwindDest = I.getUnwindDest(); auto UnwindDest = I.getUnwindDest();
BranchProbabilityInfo *BPI = FuncInfo.BPI; BranchProbabilityInfo *BPI = FuncInfo.BPI;
uint32_t UnwindDestWeight = BranchProbability UnwindDestProb =
BPI ? BPI->getEdgeWeight(FuncInfo.MBB->getBasicBlock(), UnwindDest) : 0; (BPI && UnwindDest)
findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestWeight, UnwindDests); ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
: BranchProbability::getZero();
findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
for (auto &UnwindDest : UnwindDests) { for (auto &UnwindDest : UnwindDests) {
UnwindDest.first->setIsEHPad(); UnwindDest.first->setIsEHPad();
addSuccessorWithWeight(FuncInfo.MBB, UnwindDest.first, UnwindDest.second); addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
} }
FuncInfo.MBB->normalizeSuccProbs();
// Create the terminator node. // Create the terminator node.
SDValue Ret = SDValue Ret =
@ -1493,29 +1494,34 @@ bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
} }
/// Return branch probability calculated by BranchProbabilityInfo for IR blocks. /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
uint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src, BranchProbability
const MachineBasicBlock *Dst) const { SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
const MachineBasicBlock *Dst) const {
BranchProbabilityInfo *BPI = FuncInfo.BPI; BranchProbabilityInfo *BPI = FuncInfo.BPI;
if (!BPI)
return 0;
const BasicBlock *SrcBB = Src->getBasicBlock(); const BasicBlock *SrcBB = Src->getBasicBlock();
const BasicBlock *DstBB = Dst->getBasicBlock(); const BasicBlock *DstBB = Dst->getBasicBlock();
return BPI->getEdgeWeight(SrcBB, DstBB); if (!BPI) {
// If BPI is not available, set the default probability as 1 / N, where N is
// the number of successors.
auto SuccSize = std::max<uint32_t>(
std::distance(succ_begin(SrcBB), succ_end(SrcBB)), 1);
return BranchProbability(1, SuccSize);
}
return BPI->getEdgeProbability(SrcBB, DstBB);
} }
void SelectionDAGBuilder:: void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst, MachineBasicBlock *Dst,
uint32_t Weight /* = 0 */) { BranchProbability Prob) {
if (!FuncInfo.BPI) if (!FuncInfo.BPI)
Src->addSuccessorWithoutWeight(Dst); Src->addSuccessorWithoutProb(Dst);
else { else {
if (!Weight) if (Prob.isUnknown())
Weight = getEdgeWeight(Src, Dst); Prob = getEdgeProbability(Src, Dst);
Src->addSuccessor(Dst, Weight); Src->addSuccessor(Dst, Prob);
} }
} }
static bool InBlock(const Value *V, const BasicBlock *BB) { static bool InBlock(const Value *V, const BasicBlock *BB) {
if (const Instruction *I = dyn_cast<Instruction>(V)) if (const Instruction *I = dyn_cast<Instruction>(V))
return I->getParent() == BB; return I->getParent() == BB;
@ -1532,8 +1538,8 @@ SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
MachineBasicBlock *FBB, MachineBasicBlock *FBB,
MachineBasicBlock *CurBB, MachineBasicBlock *CurBB,
MachineBasicBlock *SwitchBB, MachineBasicBlock *SwitchBB,
uint32_t TWeight, BranchProbability TProb,
uint32_t FWeight) { BranchProbability FProb) {
const BasicBlock *BB = CurBB->getBasicBlock(); const BasicBlock *BB = CurBB->getBasicBlock();
// If the leaf of the tree is a comparison, merge the condition into // If the leaf of the tree is a comparison, merge the condition into
@ -1556,7 +1562,7 @@ SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
} }
CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr, CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
TBB, FBB, CurBB, TWeight, FWeight); TBB, FBB, CurBB, TProb, FProb);
SwitchCases.push_back(CB); SwitchCases.push_back(CB);
return; return;
} }
@ -1564,18 +1570,10 @@ SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
// Create a CaseBlock record representing this branch. // Create a CaseBlock record representing this branch.
CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()), CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()),
nullptr, TBB, FBB, CurBB, TWeight, FWeight); nullptr, TBB, FBB, CurBB, TProb, FProb);
SwitchCases.push_back(CB); SwitchCases.push_back(CB);
} }
/// Scale down both weights to fit into uint32_t.
static void ScaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
uint32_t Scale = (NewMax / UINT32_MAX) + 1;
NewTrue = NewTrue / Scale;
NewFalse = NewFalse / Scale;
}
/// FindMergedConditions - If Cond is an expression like /// FindMergedConditions - If Cond is an expression like
void SelectionDAGBuilder::FindMergedConditions(const Value *Cond, void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
MachineBasicBlock *TBB, MachineBasicBlock *TBB,
@ -1583,8 +1581,8 @@ void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
MachineBasicBlock *CurBB, MachineBasicBlock *CurBB,
MachineBasicBlock *SwitchBB, MachineBasicBlock *SwitchBB,
Instruction::BinaryOps Opc, Instruction::BinaryOps Opc,
uint32_t TWeight, BranchProbability TProb,
uint32_t FWeight) { BranchProbability FProb) {
// If this node is not part of the or/and tree, emit it as a branch. // If this node is not part of the or/and tree, emit it as a branch.
const Instruction *BOp = dyn_cast<Instruction>(Cond); const Instruction *BOp = dyn_cast<Instruction>(Cond);
if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) || if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
@ -1593,7 +1591,7 @@ void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
!InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) || !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
!InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) { !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
TWeight, FWeight); TProb, FProb);
return; return;
} }
@ -1617,26 +1615,25 @@ void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
// The requirement is that // The requirement is that
// TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB) // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
// = TrueProb for original BB. // = TrueProb for original BB.
// Assuming the original weights are A and B, one choice is to set BB1's // Assuming the original probabilities are A and B, one choice is to set
// weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
// assumes that // A/(1+B) and 2B/(1+B). This choice assumes that
// TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB. // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
// Another choice is to assume TrueProb for BB1 equals to TrueProb for // Another choice is to assume TrueProb for BB1 equals to TrueProb for
// TmpBB, but the math is more complicated. // TmpBB, but the math is more complicated.
uint64_t NewTrueWeight = TWeight; auto NewTrueProb = TProb / 2;
uint64_t NewFalseWeight = (uint64_t)TWeight + 2 * (uint64_t)FWeight; auto NewFalseProb = TProb / 2 + FProb;
ScaleWeights(NewTrueWeight, NewFalseWeight);
// Emit the LHS condition. // Emit the LHS condition.
FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc, FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
NewTrueWeight, NewFalseWeight); NewTrueProb, NewFalseProb);
NewTrueWeight = TWeight; // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
NewFalseWeight = 2 * (uint64_t)FWeight; SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
ScaleWeights(NewTrueWeight, NewFalseWeight); BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
// Emit the RHS condition into TmpBB. // Emit the RHS condition into TmpBB.
FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc, FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
NewTrueWeight, NewFalseWeight); Probs[0], Probs[1]);
} else { } else {
assert(Opc == Instruction::And && "Unknown merge op!"); assert(Opc == Instruction::And && "Unknown merge op!");
// Codegen X & Y as: // Codegen X & Y as:
@ -1653,24 +1650,23 @@ void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
// The requirement is that // The requirement is that
// FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB) // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
// = FalseProb for original BB. // = FalseProb for original BB.
// Assuming the original weights are A and B, one choice is to set BB1's // Assuming the original probabilities are A and B, one choice is to set
// weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
// assumes that // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
// FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB. // TrueProb for BB1 * FalseProb for TmpBB.
uint64_t NewTrueWeight = 2 * (uint64_t)TWeight + (uint64_t)FWeight; auto NewTrueProb = TProb + FProb / 2;
uint64_t NewFalseWeight = FWeight; auto NewFalseProb = FProb / 2;
ScaleWeights(NewTrueWeight, NewFalseWeight);
// Emit the LHS condition. // Emit the LHS condition.
FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc, FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
NewTrueWeight, NewFalseWeight); NewTrueProb, NewFalseProb);
NewTrueWeight = 2 * (uint64_t)TWeight; // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
NewFalseWeight = FWeight; SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
ScaleWeights(NewTrueWeight, NewFalseWeight); BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
// Emit the RHS condition into TmpBB. // Emit the RHS condition into TmpBB.
FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc, FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
NewTrueWeight, NewFalseWeight); Probs[0], Probs[1]);
} }
} }
@ -1752,8 +1748,9 @@ void SelectionDAGBuilder::visitBr(const BranchInst &I) {
!I.getMetadata(LLVMContext::MD_unpredictable) && !I.getMetadata(LLVMContext::MD_unpredictable) &&
(Opcode == Instruction::And || Opcode == Instruction::Or)) { (Opcode == Instruction::And || Opcode == Instruction::Or)) {
FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
Opcode, getEdgeWeight(BrMBB, Succ0MBB), Opcode,
getEdgeWeight(BrMBB, Succ1MBB)); getEdgeProbability(BrMBB, Succ0MBB),
getEdgeProbability(BrMBB, Succ1MBB));
// If the compares in later blocks need to use values not currently // If the compares in later blocks need to use values not currently
// exported from this block, export them now. This block should always // exported from this block, export them now. This block should always
// be the first entry. // be the first entry.
@ -1832,11 +1829,12 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
} }
// Update successor info // Update successor info
addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight); addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
// TrueBB and FalseBB are always different unless the incoming IR is // TrueBB and FalseBB are always different unless the incoming IR is
// degenerate. This only happens when running llc on weird IR. // degenerate. This only happens when running llc on weird IR.
if (CB.TrueBB != CB.FalseBB) if (CB.TrueBB != CB.FalseBB)
addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight); addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
SwitchBB->normalizeSuccProbs();
// If the lhs block is the next block, invert the condition so that we can // If the lhs block is the next block, invert the condition so that we can
// fall through to the lhs instead of the rhs block. // fall through to the lhs instead of the rhs block.
@ -2047,8 +2045,9 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
MachineBasicBlock* MBB = B.Cases[0].ThisBB; MachineBasicBlock* MBB = B.Cases[0].ThisBB;
addSuccessorWithWeight(SwitchBB, B.Default, B.DefaultWeight); addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
addSuccessorWithWeight(SwitchBB, MBB, B.Weight); addSuccessorWithProb(SwitchBB, MBB, B.Prob);
SwitchBB->normalizeSuccProbs();
SDValue BrRange = DAG.getNode(ISD::BRCOND, dl, SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
MVT::Other, CopyTo, RangeCmp, MVT::Other, CopyTo, RangeCmp,
@ -2065,7 +2064,7 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
/// visitBitTestCase - this function produces one "bit test" /// visitBitTestCase - this function produces one "bit test"
void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB, void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
MachineBasicBlock* NextMBB, MachineBasicBlock* NextMBB,
uint32_t BranchWeightToNext, BranchProbability BranchProbToNext,
unsigned Reg, unsigned Reg,
BitTestCase &B, BitTestCase &B,
MachineBasicBlock *SwitchBB) { MachineBasicBlock *SwitchBB) {
@ -2101,10 +2100,14 @@ void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE); AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
} }
// The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight. // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight); addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
// The branch weight from SwitchBB to NextMBB is BranchWeightToNext. // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext); addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
// It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
// one as they are relative probabilities (and thus work more like weights),
// and hence we need to normalize them to let the sum of them become one.
SwitchBB->normalizeSuccProbs();
SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl, SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
MVT::Other, getControlRoot(), MVT::Other, getControlRoot(),
@ -2156,18 +2159,20 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
CopyToExportRegsIfNeeded(&I); CopyToExportRegsIfNeeded(&I);
} }
SmallVector<std::pair<MachineBasicBlock *, uint32_t>, 1> UnwindDests; SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
BranchProbabilityInfo *BPI = FuncInfo.BPI; BranchProbabilityInfo *BPI = FuncInfo.BPI;
uint32_t EHPadBBWeight = BranchProbability EHPadBBProb =
BPI ? BPI->getEdgeWeight(InvokeMBB->getBasicBlock(), EHPadBB) : 0; BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBWeight, UnwindDests); : BranchProbability::getZero();
findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
// Update successor info. // Update successor info.
addSuccessorWithWeight(InvokeMBB, Return); addSuccessorWithProb(InvokeMBB, Return);
for (auto &UnwindDest : UnwindDests) { for (auto &UnwindDest : UnwindDests) {
UnwindDest.first->setIsEHPad(); UnwindDest.first->setIsEHPad();
addSuccessorWithWeight(InvokeMBB, UnwindDest.first, UnwindDest.second); addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
} }
InvokeMBB->normalizeSuccProbs();
// Drop into normal successor. // Drop into normal successor.
DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
@ -2248,8 +2253,7 @@ void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
// If this case has the same successor and is a neighbour, merge it into // If this case has the same successor and is a neighbour, merge it into
// the previous cluster. // the previous cluster.
Clusters[DstIndex - 1].High = CaseVal; Clusters[DstIndex - 1].High = CaseVal;
Clusters[DstIndex - 1].Weight += CC.Weight; Clusters[DstIndex - 1].Prob += CC.Prob;
assert(Clusters[DstIndex - 1].Weight >= CC.Weight && "Weight overflow!");
} else { } else {
std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex], std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
sizeof(Clusters[SrcIndex])); sizeof(Clusters[SrcIndex]));
@ -2283,7 +2287,7 @@ void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
continue; continue;
MachineBasicBlock *Succ = FuncInfo.MBBMap[BB]; MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
addSuccessorWithWeight(IndirectBrMBB, Succ); addSuccessorWithProb(IndirectBrMBB, Succ);
} }
DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(), DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
@ -7642,7 +7646,7 @@ AddSuccessorMBB(const BasicBlock *BB,
} }
// Add it as a successor of ParentMBB. // Add it as a successor of ParentMBB.
ParentMBB->addSuccessor( ParentMBB->addSuccessor(
SuccMBB, BranchProbabilityInfo::getBranchWeightStackProtector(IsLikely)); SuccMBB, BranchProbabilityInfo::getBranchProbStackProtector(IsLikely));
return SuccMBB; return SuccMBB;
} }
@ -7704,14 +7708,18 @@ bool SelectionDAGBuilder::buildJumpTable(CaseClusterVector &Clusters,
CaseCluster &JTCluster) { CaseCluster &JTCluster) {
assert(First <= Last); assert(First <= Last);
uint32_t Weight = 0; auto Prob = BranchProbability::getZero();
unsigned NumCmps = 0; unsigned NumCmps = 0;
std::vector<MachineBasicBlock*> Table; std::vector<MachineBasicBlock*> Table;
DenseMap<MachineBasicBlock*, uint32_t> JTWeights; DenseMap<MachineBasicBlock*, BranchProbability> JTProbs;
// Initialize probabilities in JTProbs.
for (unsigned I = First; I <= Last; ++I)
JTProbs[Clusters[I].MBB] = BranchProbability::getZero();
for (unsigned I = First; I <= Last; ++I) { for (unsigned I = First; I <= Last; ++I) {
assert(Clusters[I].Kind == CC_Range); assert(Clusters[I].Kind == CC_Range);
Weight += Clusters[I].Weight; Prob += Clusters[I].Prob;
assert(Weight >= Clusters[I].Weight && "Weight overflow!");
APInt Low = Clusters[I].Low->getValue(); APInt Low = Clusters[I].Low->getValue();
APInt High = Clusters[I].High->getValue(); APInt High = Clusters[I].High->getValue();
NumCmps += (Low == High) ? 1 : 2; NumCmps += (Low == High) ? 1 : 2;
@ -7726,10 +7734,10 @@ bool SelectionDAGBuilder::buildJumpTable(CaseClusterVector &Clusters,
uint64_t ClusterSize = (High - Low).getLimitedValue() + 1; uint64_t ClusterSize = (High - Low).getLimitedValue() + 1;
for (uint64_t J = 0; J < ClusterSize; ++J) for (uint64_t J = 0; J < ClusterSize; ++J)
Table.push_back(Clusters[I].MBB); Table.push_back(Clusters[I].MBB);
JTWeights[Clusters[I].MBB] += Clusters[I].Weight; JTProbs[Clusters[I].MBB] += Clusters[I].Prob;
} }
unsigned NumDests = JTWeights.size(); unsigned NumDests = JTProbs.size();
if (isSuitableForBitTests(NumDests, NumCmps, if (isSuitableForBitTests(NumDests, NumCmps,
Clusters[First].Low->getValue(), Clusters[First].Low->getValue(),
Clusters[Last].High->getValue())) { Clusters[Last].High->getValue())) {
@ -7748,9 +7756,10 @@ bool SelectionDAGBuilder::buildJumpTable(CaseClusterVector &Clusters,
for (MachineBasicBlock *Succ : Table) { for (MachineBasicBlock *Succ : Table) {
if (Done.count(Succ)) if (Done.count(Succ))
continue; continue;
addSuccessorWithWeight(JumpTableMBB, Succ, JTWeights[Succ]); addSuccessorWithProb(JumpTableMBB, Succ, JTProbs[Succ]);
Done.insert(Succ); Done.insert(Succ);
} }
JumpTableMBB->normalizeSuccProbs();
const TargetLowering &TLI = DAG.getTargetLoweringInfo(); const TargetLowering &TLI = DAG.getTargetLoweringInfo();
unsigned JTI = CurMF->getOrCreateJumpTableInfo(TLI.getJumpTableEncoding()) unsigned JTI = CurMF->getOrCreateJumpTableInfo(TLI.getJumpTableEncoding())
@ -7764,7 +7773,7 @@ bool SelectionDAGBuilder::buildJumpTable(CaseClusterVector &Clusters,
JTCases.emplace_back(std::move(JTH), std::move(JT)); JTCases.emplace_back(std::move(JTH), std::move(JT));
JTCluster = CaseCluster::jumpTable(Clusters[First].Low, Clusters[Last].High, JTCluster = CaseCluster::jumpTable(Clusters[First].Low, Clusters[Last].High,
JTCases.size() - 1, Weight); JTCases.size() - 1, Prob);
return true; return true;
} }
@ -7964,7 +7973,7 @@ bool SelectionDAGBuilder::buildBitTests(CaseClusterVector &Clusters,
} }
CaseBitsVector CBV; CaseBitsVector CBV;
uint32_t TotalWeight = 0; auto TotalProb = BranchProbability::getZero();
for (unsigned i = First; i <= Last; ++i) { for (unsigned i = First; i <= Last; ++i) {
// Find the CaseBits for this destination. // Find the CaseBits for this destination.
unsigned j; unsigned j;
@ -7972,40 +7981,40 @@ bool SelectionDAGBuilder::buildBitTests(CaseClusterVector &Clusters,
if (CBV[j].BB == Clusters[i].MBB) if (CBV[j].BB == Clusters[i].MBB)
break; break;
if (j == CBV.size()) if (j == CBV.size())
CBV.push_back(CaseBits(0, Clusters[i].MBB, 0, 0)); CBV.push_back(
CaseBits(0, Clusters[i].MBB, 0, BranchProbability::getZero()));
CaseBits *CB = &CBV[j]; CaseBits *CB = &CBV[j];
// Update Mask, Bits and ExtraWeight. // Update Mask, Bits and ExtraProb.
uint64_t Lo = (Clusters[i].Low->getValue() - LowBound).getZExtValue(); uint64_t Lo = (Clusters[i].Low->getValue() - LowBound).getZExtValue();
uint64_t Hi = (Clusters[i].High->getValue() - LowBound).getZExtValue(); uint64_t Hi = (Clusters[i].High->getValue() - LowBound).getZExtValue();
assert(Hi >= Lo && Hi < 64 && "Invalid bit case!"); assert(Hi >= Lo && Hi < 64 && "Invalid bit case!");
CB->Mask |= (-1ULL >> (63 - (Hi - Lo))) << Lo; CB->Mask |= (-1ULL >> (63 - (Hi - Lo))) << Lo;
CB->Bits += Hi - Lo + 1; CB->Bits += Hi - Lo + 1;
CB->ExtraWeight += Clusters[i].Weight; CB->ExtraProb += Clusters[i].Prob;
TotalWeight += Clusters[i].Weight; TotalProb += Clusters[i].Prob;
assert(TotalWeight >= Clusters[i].Weight && "Weight overflow!");
} }
BitTestInfo BTI; BitTestInfo BTI;
std::sort(CBV.begin(), CBV.end(), [](const CaseBits &a, const CaseBits &b) { std::sort(CBV.begin(), CBV.end(), [](const CaseBits &a, const CaseBits &b) {
// Sort by weight first, number of bits second. // Sort by probability first, number of bits second.
if (a.ExtraWeight != b.ExtraWeight) if (a.ExtraProb != b.ExtraProb)
return a.ExtraWeight > b.ExtraWeight; return a.ExtraProb > b.ExtraProb;
return a.Bits > b.Bits; return a.Bits > b.Bits;
}); });
for (auto &CB : CBV) { for (auto &CB : CBV) {
MachineBasicBlock *BitTestBB = MachineBasicBlock *BitTestBB =
FuncInfo.MF->CreateMachineBasicBlock(SI->getParent()); FuncInfo.MF->CreateMachineBasicBlock(SI->getParent());
BTI.push_back(BitTestCase(CB.Mask, BitTestBB, CB.BB, CB.ExtraWeight)); BTI.push_back(BitTestCase(CB.Mask, BitTestBB, CB.BB, CB.ExtraProb));
} }
BitTestCases.emplace_back(std::move(LowBound), std::move(CmpRange), BitTestCases.emplace_back(std::move(LowBound), std::move(CmpRange),
SI->getCondition(), -1U, MVT::Other, false, SI->getCondition(), -1U, MVT::Other, false,
ContiguousRange, nullptr, nullptr, std::move(BTI), ContiguousRange, nullptr, nullptr, std::move(BTI),
TotalWeight); TotalProb);
BTCluster = CaseCluster::bitTests(Clusters[First].Low, Clusters[Last].High, BTCluster = CaseCluster::bitTests(Clusters[First].Low, Clusters[Last].High,
BitTestCases.size() - 1, TotalWeight); BitTestCases.size() - 1, TotalProb);
return true; return true;
} }
@ -8152,13 +8161,16 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
ISD::SETEQ); ISD::SETEQ);
// Update successor info. // Update successor info.
// Both Small and Big will jump to Small.BB, so we sum up the weights. // Both Small and Big will jump to Small.BB, so we sum up the
addSuccessorWithWeight(SwitchMBB, Small.MBB, Small.Weight + Big.Weight); // probabilities.
addSuccessorWithWeight( addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
SwitchMBB, DefaultMBB, if (BPI)
// The default destination is the first successor in IR. addSuccessorWithProb(
BPI ? BPI->getEdgeWeight(SwitchMBB->getBasicBlock(), (unsigned)0) SwitchMBB, DefaultMBB,
: 0); // The default destination is the first successor in IR.
BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
else
addSuccessorWithProb(SwitchMBB, DefaultMBB);
// Insert the true branch. // Insert the true branch.
SDValue BrCond = SDValue BrCond =
@ -8175,17 +8187,17 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
} }
if (TM.getOptLevel() != CodeGenOpt::None) { if (TM.getOptLevel() != CodeGenOpt::None) {
// Order cases by weight so the most likely case will be checked first. // Order cases by probability so the most likely case will be checked first.
std::sort(W.FirstCluster, W.LastCluster + 1, std::sort(W.FirstCluster, W.LastCluster + 1,
[](const CaseCluster &a, const CaseCluster &b) { [](const CaseCluster &a, const CaseCluster &b) {
return a.Weight > b.Weight; return a.Prob > b.Prob;
}); });
// Rearrange the case blocks so that the last one falls through if possible // Rearrange the case blocks so that the last one falls through if possible
// without without changing the order of weights. // without without changing the order of probabilities.
for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) { for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
--I; --I;
if (I->Weight > W.LastCluster->Weight) if (I->Prob > W.LastCluster->Prob)
break; break;
if (I->Kind == CC_Range && I->MBB == NextMBB) { if (I->Kind == CC_Range && I->MBB == NextMBB) {
std::swap(*I, *W.LastCluster); std::swap(*I, *W.LastCluster);
@ -8194,13 +8206,11 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
} }
} }
// Compute total weight. // Compute total probability.
uint32_t DefaultWeight = W.DefaultWeight; BranchProbability DefaultProb = W.DefaultProb;
uint32_t UnhandledWeights = DefaultWeight; BranchProbability UnhandledProbs = DefaultProb;
for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I) { for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
UnhandledWeights += I->Weight; UnhandledProbs += I->Prob;
assert(UnhandledWeights >= I->Weight && "Weight overflow!");
}
MachineBasicBlock *CurMBB = W.MBB; MachineBasicBlock *CurMBB = W.MBB;
for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) { for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
@ -8214,7 +8224,7 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
// Put Cond in a virtual register to make it available from the new blocks. // Put Cond in a virtual register to make it available from the new blocks.
ExportFromCurrentBlock(Cond); ExportFromCurrentBlock(Cond);
} }
UnhandledWeights -= I->Weight; UnhandledProbs -= I->Prob;
switch (I->Kind) { switch (I->Kind) {
case CC_JumpTable: { case CC_JumpTable: {
@ -8226,25 +8236,25 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
MachineBasicBlock *JumpMBB = JT->MBB; MachineBasicBlock *JumpMBB = JT->MBB;
CurMF->insert(BBI, JumpMBB); CurMF->insert(BBI, JumpMBB);
uint32_t JumpWeight = I->Weight; auto JumpProb = I->Prob;
uint32_t FallthroughWeight = UnhandledWeights; auto FallthroughProb = UnhandledProbs;
// If the default statement is a target of the jump table, we evenly // If the default statement is a target of the jump table, we evenly
// distribute the default weight to successors of CurMBB. Also update // distribute the default probability to successors of CurMBB. Also
// the weight on the edge from JumpMBB to Fallthrough. // update the probability on the edge from JumpMBB to Fallthrough.
for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(), for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
SE = JumpMBB->succ_end(); SE = JumpMBB->succ_end();
SI != SE; ++SI) { SI != SE; ++SI) {
if (*SI == DefaultMBB) { if (*SI == DefaultMBB) {
JumpWeight += DefaultWeight / 2; JumpProb += DefaultProb / 2;
FallthroughWeight -= DefaultWeight / 2; FallthroughProb -= DefaultProb / 2;
JumpMBB->setSuccWeight(SI, DefaultWeight / 2); JumpMBB->setSuccProbability(SI, DefaultProb / 2);
break; break;
} }
} }
addSuccessorWithWeight(CurMBB, Fallthrough, FallthroughWeight); addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
addSuccessorWithWeight(CurMBB, JumpMBB, JumpWeight); addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
// The jump table header will be inserted in our current block, do the // The jump table header will be inserted in our current block, do the
// range check, and fall through to our fallthrough block. // range check, and fall through to our fallthrough block.
@ -8270,13 +8280,13 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
BTB->Parent = CurMBB; BTB->Parent = CurMBB;
BTB->Default = Fallthrough; BTB->Default = Fallthrough;
BTB->DefaultWeight = UnhandledWeights; BTB->DefaultProb = UnhandledProbs;
// If the cases in bit test don't form a contiguous range, we evenly // If the cases in bit test don't form a contiguous range, we evenly
// distribute the weight on the edge to Fallthrough to two successors // distribute the probability on the edge to Fallthrough to two
// of CurMBB. // successors of CurMBB.
if (!BTB->ContiguousRange) { if (!BTB->ContiguousRange) {
BTB->Weight += DefaultWeight / 2; BTB->Prob += DefaultProb / 2;
BTB->DefaultWeight -= DefaultWeight / 2; BTB->DefaultProb -= DefaultProb / 2;
} }
// If we're in the right place, emit the bit test header right now. // If we're in the right place, emit the bit test header right now.
@ -8303,9 +8313,9 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
RHS = I->High; RHS = I->High;
} }
// The false weight is the sum of all unhandled cases. // The false probability is the sum of all unhandled cases.
CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB, I->Weight, CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB, I->Prob,
UnhandledWeights); UnhandledProbs);
if (CurMBB == SwitchMBB) if (CurMBB == SwitchMBB)
visitSwitchCase(CB, SwitchMBB); visitSwitchCase(CB, SwitchMBB);
@ -8323,8 +8333,8 @@ unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC,
CaseClusterIt First, CaseClusterIt First,
CaseClusterIt Last) { CaseClusterIt Last) {
return std::count_if(First, Last + 1, [&](const CaseCluster &X) { return std::count_if(First, Last + 1, [&](const CaseCluster &X) {
if (X.Weight != CC.Weight) if (X.Prob != CC.Prob)
return X.Weight > CC.Weight; return X.Prob > CC.Prob;
// Ties are broken by comparing the case value. // Ties are broken by comparing the case value.
return X.Low->getValue().slt(CC.Low->getValue()); return X.Low->getValue().slt(CC.Low->getValue());
@ -8340,24 +8350,24 @@ void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!"); assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
// Balance the tree based on branch weights to create a near-optimal (in terms // Balance the tree based on branch probabilities to create a near-optimal (in
// of search time given key frequency) binary search tree. See e.g. Kurt // terms of search time given key frequency) binary search tree. See e.g. Kurt
// Mehlhorn "Nearly Optimal Binary Search Trees" (1975). // Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
CaseClusterIt LastLeft = W.FirstCluster; CaseClusterIt LastLeft = W.FirstCluster;
CaseClusterIt FirstRight = W.LastCluster; CaseClusterIt FirstRight = W.LastCluster;
uint32_t LeftWeight = LastLeft->Weight + W.DefaultWeight / 2; auto LeftProb = LastLeft->Prob + W.DefaultProb / 2;
uint32_t RightWeight = FirstRight->Weight + W.DefaultWeight / 2; auto RightProb = FirstRight->Prob + W.DefaultProb / 2;
// Move LastLeft and FirstRight towards each other from opposite directions to // Move LastLeft and FirstRight towards each other from opposite directions to
// find a partitioning of the clusters which balances the weight on both // find a partitioning of the clusters which balances the probability on both
// sides. If LeftWeight and RightWeight are equal, alternate which side is // sides. If LeftProb and RightProb are equal, alternate which side is
// taken to ensure 0-weight nodes are distributed evenly. // taken to ensure 0-probability nodes are distributed evenly.
unsigned I = 0; unsigned I = 0;
while (LastLeft + 1 < FirstRight) { while (LastLeft + 1 < FirstRight) {
if (LeftWeight < RightWeight || (LeftWeight == RightWeight && (I & 1))) if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1)))
LeftWeight += (++LastLeft)->Weight; LeftProb += (++LastLeft)->Prob;
else else
RightWeight += (--FirstRight)->Weight; RightProb += (--FirstRight)->Prob;
I++; I++;
} }
@ -8433,7 +8443,7 @@ void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock()); LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
FuncInfo.MF->insert(BBI, LeftMBB); FuncInfo.MF->insert(BBI, LeftMBB);
WorkList.push_back( WorkList.push_back(
{LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultWeight / 2}); {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
// Put Cond in a virtual register to make it available from the new blocks. // Put Cond in a virtual register to make it available from the new blocks.
ExportFromCurrentBlock(Cond); ExportFromCurrentBlock(Cond);
} }
@ -8449,14 +8459,14 @@ void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock()); RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
FuncInfo.MF->insert(BBI, RightMBB); FuncInfo.MF->insert(BBI, RightMBB);
WorkList.push_back( WorkList.push_back(
{RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultWeight / 2}); {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
// Put Cond in a virtual register to make it available from the new blocks. // Put Cond in a virtual register to make it available from the new blocks.
ExportFromCurrentBlock(Cond); ExportFromCurrentBlock(Cond);
} }
// Create the CaseBlock record that will be used to lower the branch. // Create the CaseBlock record that will be used to lower the branch.
CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB, CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
LeftWeight, RightWeight); LeftProb, RightProb);
if (W.MBB == SwitchMBB) if (W.MBB == SwitchMBB)
visitSwitchCase(CB, SwitchMBB); visitSwitchCase(CB, SwitchMBB);
@ -8472,9 +8482,10 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
for (auto I : SI.cases()) { for (auto I : SI.cases()) {
MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()]; MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
const ConstantInt *CaseVal = I.getCaseValue(); const ConstantInt *CaseVal = I.getCaseValue();
uint32_t Weight = BranchProbability Prob =
BPI ? BPI->getEdgeWeight(SI.getParent(), I.getSuccessorIndex()) : 0; BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Weight)); : BranchProbability(1, SI.getNumCases() + 1);
Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
} }
MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()]; MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
@ -8550,8 +8561,8 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
SwitchWorkList WorkList; SwitchWorkList WorkList;
CaseClusterIt First = Clusters.begin(); CaseClusterIt First = Clusters.begin();
CaseClusterIt Last = Clusters.end() - 1; CaseClusterIt Last = Clusters.end() - 1;
uint32_t DefaultWeight = getEdgeWeight(SwitchMBB, DefaultMBB); auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultWeight}); WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
while (!WorkList.empty()) { while (!WorkList.empty()) {
SwitchWorkListItem W = WorkList.back(); SwitchWorkListItem W = WorkList.back();

74
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
View File

@ -154,39 +154,39 @@ private:
unsigned JTCasesIndex; unsigned JTCasesIndex;
unsigned BTCasesIndex; unsigned BTCasesIndex;
}; };
uint32_t Weight; BranchProbability Prob;
static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, static CaseCluster range(const ConstantInt *Low, const ConstantInt *High,
MachineBasicBlock *MBB, uint32_t Weight) { MachineBasicBlock *MBB, BranchProbability Prob) {
CaseCluster C; CaseCluster C;
C.Kind = CC_Range; C.Kind = CC_Range;
C.Low = Low; C.Low = Low;
C.High = High; C.High = High;
C.MBB = MBB; C.MBB = MBB;
C.Weight = Weight; C.Prob = Prob;
return C; return C;
} }
static CaseCluster jumpTable(const ConstantInt *Low, static CaseCluster jumpTable(const ConstantInt *Low,
const ConstantInt *High, unsigned JTCasesIndex, const ConstantInt *High, unsigned JTCasesIndex,
uint32_t Weight) { BranchProbability Prob) {
CaseCluster C; CaseCluster C;
C.Kind = CC_JumpTable; C.Kind = CC_JumpTable;
C.Low = Low; C.Low = Low;
C.High = High; C.High = High;
C.JTCasesIndex = JTCasesIndex; C.JTCasesIndex = JTCasesIndex;
C.Weight = Weight; C.Prob = Prob;
return C; return C;
} }
static CaseCluster bitTests(const ConstantInt *Low, const ConstantInt *High, static CaseCluster bitTests(const ConstantInt *Low, const ConstantInt *High,
unsigned BTCasesIndex, uint32_t Weight) { unsigned BTCasesIndex, BranchProbability Prob) {
CaseCluster C; CaseCluster C;
C.Kind = CC_BitTests; C.Kind = CC_BitTests;
C.Low = Low; C.Low = Low;
C.High = High; C.High = High;
C.BTCasesIndex = BTCasesIndex; C.BTCasesIndex = BTCasesIndex;
C.Weight = Weight; C.Prob = Prob;
return C; return C;
} }
}; };
@ -198,13 +198,13 @@ private:
uint64_t Mask; uint64_t Mask;
MachineBasicBlock* BB; MachineBasicBlock* BB;
unsigned Bits; unsigned Bits;
uint32_t ExtraWeight; BranchProbability ExtraProb;
CaseBits(uint64_t mask, MachineBasicBlock* bb, unsigned bits, CaseBits(uint64_t mask, MachineBasicBlock* bb, unsigned bits,
uint32_t Weight): BranchProbability Prob):
Mask(mask), BB(bb), Bits(bits), ExtraWeight(Weight) { } Mask(mask), BB(bb), Bits(bits), ExtraProb(Prob) { }
CaseBits() : Mask(0), BB(nullptr), Bits(0), ExtraWeight(0) {} CaseBits() : Mask(0), BB(nullptr), Bits(0) {}
}; };
typedef std::vector<CaseBits> CaseBitsVector; typedef std::vector<CaseBits> CaseBitsVector;
@ -217,13 +217,13 @@ private:
/// blocks needed by multi-case switch statements. /// blocks needed by multi-case switch statements.
struct CaseBlock { struct CaseBlock {
CaseBlock(ISD::CondCode cc, const Value *cmplhs, const Value *cmprhs, CaseBlock(ISD::CondCode cc, const Value *cmplhs, const Value *cmprhs,
const Value *cmpmiddle, const Value *cmpmiddle, MachineBasicBlock *truebb,
MachineBasicBlock *truebb, MachineBasicBlock *falsebb, MachineBasicBlock *falsebb, MachineBasicBlock *me,
MachineBasicBlock *me, BranchProbability trueprob = BranchProbability::getUnknown(),
uint32_t trueweight = 0, uint32_t falseweight = 0) BranchProbability falseprob = BranchProbability::getUnknown())
: CC(cc), CmpLHS(cmplhs), CmpMHS(cmpmiddle), CmpRHS(cmprhs), : CC(cc), CmpLHS(cmplhs), CmpMHS(cmpmiddle), CmpRHS(cmprhs),
TrueBB(truebb), FalseBB(falsebb), ThisBB(me), TrueBB(truebb), FalseBB(falsebb), ThisBB(me), TrueProb(trueprob),
TrueWeight(trueweight), FalseWeight(falseweight) { } FalseProb(falseprob) {}
// CC - the condition code to use for the case block's setcc node // CC - the condition code to use for the case block's setcc node
ISD::CondCode CC; ISD::CondCode CC;
@ -239,8 +239,8 @@ private:
// ThisBB - the block into which to emit the code for the setcc and branches // ThisBB - the block into which to emit the code for the setcc and branches
MachineBasicBlock *ThisBB; MachineBasicBlock *ThisBB;
// TrueWeight/FalseWeight - branch weights. // TrueProb/FalseProb - branch weights.
uint32_t TrueWeight, FalseWeight; BranchProbability TrueProb, FalseProb;
}; };
struct JumpTable { struct JumpTable {
@ -272,12 +272,12 @@ private:
struct BitTestCase { struct BitTestCase {
BitTestCase(uint64_t M, MachineBasicBlock* T, MachineBasicBlock* Tr, BitTestCase(uint64_t M, MachineBasicBlock* T, MachineBasicBlock* Tr,
uint32_t Weight): BranchProbability Prob):
Mask(M), ThisBB(T), TargetBB(Tr), ExtraWeight(Weight) { } Mask(M), ThisBB(T), TargetBB(Tr), ExtraProb(Prob) { }
uint64_t Mask; uint64_t Mask;
MachineBasicBlock *ThisBB; MachineBasicBlock *ThisBB;
MachineBasicBlock *TargetBB; MachineBasicBlock *TargetBB;
uint32_t ExtraWeight; BranchProbability ExtraProb;
}; };
typedef SmallVector<BitTestCase, 3> BitTestInfo; typedef SmallVector<BitTestCase, 3> BitTestInfo;
@ -285,10 +285,10 @@ private:
struct BitTestBlock { struct BitTestBlock {
BitTestBlock(APInt F, APInt R, const Value *SV, unsigned Rg, MVT RgVT, BitTestBlock(APInt F, APInt R, const Value *SV, unsigned Rg, MVT RgVT,
bool E, bool CR, MachineBasicBlock *P, MachineBasicBlock *D, bool E, bool CR, MachineBasicBlock *P, MachineBasicBlock *D,
BitTestInfo C, uint32_t W) BitTestInfo C, BranchProbability Pr)
: First(F), Range(R), SValue(SV), Reg(Rg), RegVT(RgVT), Emitted(E), : First(F), Range(R), SValue(SV), Reg(Rg), RegVT(RgVT), Emitted(E),
ContiguousRange(CR), Parent(P), Default(D), Cases(std::move(C)), ContiguousRange(CR), Parent(P), Default(D), Cases(std::move(C)),
Weight(W), DefaultWeight(0) {} Prob(Pr) {}
APInt First; APInt First;
APInt Range; APInt Range;
const Value *SValue; const Value *SValue;
@ -299,8 +299,8 @@ private:
MachineBasicBlock *Parent; MachineBasicBlock *Parent;
MachineBasicBlock *Default; MachineBasicBlock *Default;
BitTestInfo Cases; BitTestInfo Cases;
uint32_t Weight; BranchProbability Prob;
uint32_t DefaultWeight; BranchProbability DefaultProb;
}; };
/// Minimum jump table density, in percent. /// Minimum jump table density, in percent.
@ -342,7 +342,7 @@ private:
CaseClusterIt LastCluster; CaseClusterIt LastCluster;
const ConstantInt *GE; const ConstantInt *GE;
const ConstantInt *LT; const ConstantInt *LT;
uint32_t DefaultWeight; BranchProbability DefaultProb;
}; };
typedef SmallVector<SwitchWorkListItem, 4> SwitchWorkList; typedef SmallVector<SwitchWorkListItem, 4> SwitchWorkList;
@ -694,13 +694,13 @@ public:
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB,
MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB,
MachineBasicBlock *SwitchBB, MachineBasicBlock *SwitchBB,
Instruction::BinaryOps Opc, Instruction::BinaryOps Opc, BranchProbability TW,
uint32_t TW, uint32_t FW); BranchProbability FW);
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB,
MachineBasicBlock *FBB, MachineBasicBlock *FBB,
MachineBasicBlock *CurBB, MachineBasicBlock *CurBB,
MachineBasicBlock *SwitchBB, MachineBasicBlock *SwitchBB,
uint32_t TW, uint32_t FW); BranchProbability TW, BranchProbability FW);
bool ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases); bool ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases);
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB); bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB);
void CopyToExportRegsIfNeeded(const Value *V); void CopyToExportRegsIfNeeded(const Value *V);
@ -744,10 +744,12 @@ private:
void visitTerminatePad(const TerminatePadInst &TPI); void visitTerminatePad(const TerminatePadInst &TPI);
void visitCleanupPad(const CleanupPadInst &CPI); void visitCleanupPad(const CleanupPadInst &CPI);
uint32_t getEdgeWeight(const MachineBasicBlock *Src, BranchProbability getEdgeProbability(const MachineBasicBlock *Src,
const MachineBasicBlock *Dst) const; const MachineBasicBlock *Dst) const;
void addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst, void addSuccessorWithProb(
uint32_t Weight = 0); MachineBasicBlock *Src, MachineBasicBlock *Dst,
BranchProbability Prob = BranchProbability::getUnknown());
public: public:
void visitSwitchCase(CaseBlock &CB, void visitSwitchCase(CaseBlock &CB,
MachineBasicBlock *SwitchBB); MachineBasicBlock *SwitchBB);
@ -757,7 +759,7 @@ public:
void visitBitTestHeader(BitTestBlock &B, MachineBasicBlock *SwitchBB); void visitBitTestHeader(BitTestBlock &B, MachineBasicBlock *SwitchBB);
void visitBitTestCase(BitTestBlock &BB, void visitBitTestCase(BitTestBlock &BB,
MachineBasicBlock* NextMBB, MachineBasicBlock* NextMBB,
uint32_t BranchWeightToNext, BranchProbability BranchProbToNext,
unsigned Reg, unsigned Reg,
BitTestCase &B, BitTestCase &B,
MachineBasicBlock *SwitchBB); MachineBasicBlock *SwitchBB);

7
llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
View File

@ -1486,10 +1486,9 @@ SelectionDAGISel::FinishBasicBlock() {
CodeGenAndEmitDAG(); CodeGenAndEmitDAG();
} }
uint32_t UnhandledWeight = SDB->BitTestCases[i].Weight; BranchProbability UnhandledProb = SDB->BitTestCases[i].Prob;
for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j) { for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j) {
UnhandledWeight -= SDB->BitTestCases[i].Cases[j].ExtraWeight; UnhandledProb -= SDB->BitTestCases[i].Cases[j].ExtraProb;
// Set the current basic block to the mbb we wish to insert the code into // Set the current basic block to the mbb we wish to insert the code into
FuncInfo->MBB = SDB->BitTestCases[i].Cases[j].ThisBB; FuncInfo->MBB = SDB->BitTestCases[i].Cases[j].ThisBB;
FuncInfo->InsertPt = FuncInfo->MBB->end(); FuncInfo->InsertPt = FuncInfo->MBB->end();
@ -1509,7 +1508,7 @@ SelectionDAGISel::FinishBasicBlock() {
SDB->visitBitTestCase(SDB->BitTestCases[i], SDB->visitBitTestCase(SDB->BitTestCases[i],
NextMBB, NextMBB,
UnhandledWeight, UnhandledProb,
SDB->BitTestCases[i].Reg, SDB->BitTestCases[i].Reg,
SDB->BitTestCases[i].Cases[j], SDB->BitTestCases[i].Cases[j],
FuncInfo->MBB); FuncInfo->MBB);

10
llvm/lib/Target/AArch64/AArch64FastISel.cpp
View File

@ -2375,13 +2375,13 @@ bool AArch64FastISel::selectBranch(const Instruction *I) {
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::B)) BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::B))
.addMBB(Target); .addMBB(Target);
// Obtain the branch weight and add the target to the successor list. // Obtain the branch probability and add the target to the successor list.
if (FuncInfo.BPI) { if (FuncInfo.BPI) {
uint32_t BranchWeight = auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
FuncInfo.BPI->getEdgeWeight(BI->getParent(), Target->getBasicBlock()); BI->getParent(), Target->getBasicBlock());
FuncInfo.MBB->addSuccessor(Target, BranchWeight); FuncInfo.MBB->addSuccessor(Target, BranchProbability);
} else } else
FuncInfo.MBB->addSuccessorWithoutWeight(Target); FuncInfo.MBB->addSuccessorWithoutProb(Target);
return true; return true;
} else if (foldXALUIntrinsic(CC, I, BI->getCondition())) { } else if (foldXALUIntrinsic(CC, I, BI->getCondition())) {
// Fake request the condition, otherwise the intrinsic might be completely // Fake request the condition, otherwise the intrinsic might be completely

11
llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp
View File

@ -801,11 +801,12 @@ MachineBasicBlock *Filler::selectSuccBB(MachineBasicBlock &B) const {
// Select the successor with the larget edge weight. // Select the successor with the larget edge weight.
auto &Prob = getAnalysis<MachineBranchProbabilityInfo>(); auto &Prob = getAnalysis<MachineBranchProbabilityInfo>();
MachineBasicBlock *S = *std::max_element(B.succ_begin(), B.succ_end(), MachineBasicBlock *S = *std::max_element(
[&](const MachineBasicBlock *Dst0, B.succ_begin(), B.succ_end(),
const MachineBasicBlock *Dst1) { [&](const MachineBasicBlock *Dst0, const MachineBasicBlock *Dst1) {
return Prob.getEdgeWeight(&B, Dst0) < Prob.getEdgeWeight(&B, Dst1); return Prob.getEdgeProbability(&B, Dst0) <
}); Prob.getEdgeProbability(&B, Dst1);
});
return S->isEHPad() ? nullptr : S; return S->isEHPad() ? nullptr : S;
} }

4
llvm/lib/Target/PowerPC/PPCISelLowering.cpp
View File

@ -8413,8 +8413,8 @@ PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr *MI,
.addMBB(mainMBB); .addMBB(mainMBB);
MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::B)).addMBB(sinkMBB); MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::B)).addMBB(sinkMBB);
thisMBB->addSuccessor(mainMBB, /* weight */ 0); thisMBB->addSuccessor(mainMBB, BranchProbability::getZero());
thisMBB->addSuccessor(sinkMBB, /* weight */ 1); thisMBB->addSuccessor(sinkMBB, BranchProbability::getOne());
// mainMBB: // mainMBB:
// mainDstReg = 0 // mainDstReg = 0

8
llvm/lib/Target/X86/X86FrameLowering.cpp
View File

@ -2376,10 +2376,10 @@ void X86FrameLowering::adjustForHiPEPrologue(
.addReg(ScratchReg), PReg, false, SPLimitOffset); .addReg(ScratchReg), PReg, false, SPLimitOffset);
BuildMI(incStackMBB, DL, TII.get(X86::JLE_1)).addMBB(incStackMBB); BuildMI(incStackMBB, DL, TII.get(X86::JLE_1)).addMBB(incStackMBB);
stackCheckMBB->addSuccessor(&PrologueMBB, 99); stackCheckMBB->addSuccessor(&PrologueMBB, {99, 100});
stackCheckMBB->addSuccessor(incStackMBB, 1); stackCheckMBB->addSuccessor(incStackMBB, {1, 100});
incStackMBB->addSuccessor(&PrologueMBB, 99); incStackMBB->addSuccessor(&PrologueMBB, {99, 100});
incStackMBB->addSuccessor(incStackMBB, 1); incStackMBB->addSuccessor(incStackMBB, {1, 100});
} }
#ifdef XDEBUG #ifdef XDEBUG
MF.verify(); MF.verify();

2
llvm/test/CodeGen/ARM/ifcvt-branch-weight-bug.ll
View File

@ -22,7 +22,7 @@ entry:
; for.body -> for.cond.backedge (130023362) ; for.body -> for.cond.backedge (130023362)
; -> cond.false.i (62) ; -> cond.false.i (62)
; CHECK: BB#1: derived from LLVM BB %for.body ; CHECK: BB#1: derived from LLVM BB %for.body
; CHECK: Successors according to CFG: BB#2(130023362) BB#4(62) ; CHECK: Successors according to CFG: BB#2(4294967291) BB#4(2048)
for.body: for.body:
br i1 undef, label %for.cond.backedge, label %lor.lhs.false.i, !prof !1 br i1 undef, label %for.cond.backedge, label %lor.lhs.false.i, !prof !1

2
llvm/test/CodeGen/ARM/ifcvt-branch-weight.ll
View File

@ -19,7 +19,7 @@ bb:
br i1 %9, label %return, label %bb2 br i1 %9, label %return, label %bb2
; CHECK: BB#2: derived from LLVM BB %bb2 ; CHECK: BB#2: derived from LLVM BB %bb2
; CHECK: Successors according to CFG: BB#3(192) BB#4(192) ; CHECK: Successors according to CFG: BB#3(4294967289) BB#4(4294967287)
bb2: bb2:
%v10 = icmp eq i32 %3, 16 %v10 = icmp eq i32 %3, 16

4
llvm/test/CodeGen/ARM/ifcvt-iter-indbr.ll
View File

@ -30,9 +30,9 @@ declare i8* @bar(i32, i8*, i8*)
; CHECK-NEXT: blx _foo ; CHECK-NEXT: blx _foo
; ;
; CHECK-WEIGHT: BB#0: ; CHECK-WEIGHT: BB#0:
; CHECK-WEIGHT: Successors according to CFG: BB#1(16) BB#2(8) BB#4(8) ; CHECK-WEIGHT: Successors according to CFG: BB#1(1073741824) BB#2(536870912) BB#4(536870912)
; CHECK-WEIGHT: BB#1: ; CHECK-WEIGHT: BB#1:
; CHECK-WEIGHT: Successors according to CFG: BB#2(24) BB#4(8) ; CHECK-WEIGHT: Successors according to CFG: BB#2(1610612736) BB#4(536870912)
define i32 @test(i32 %a, i32 %a2, i32* %p, i32* %p2) { define i32 @test(i32 %a, i32 %a2, i32* %p, i32* %p2) {
entry: entry:

4
llvm/test/CodeGen/ARM/taildup-branch-weight.ll
View File

@ -3,7 +3,7 @@
; RUN: | FileCheck %s ; RUN: | FileCheck %s
; CHECK: Machine code for function test0: ; CHECK: Machine code for function test0:
; CHECK: Successors according to CFG: BB#1(4) BB#2(124) ; CHECK: Successors according to CFG: BB#1(67108864) BB#2(2080374784)
define void @test0(i32 %a, i32 %b, i32* %c, i32* %d) { define void @test0(i32 %a, i32 %b, i32* %c, i32* %d) {
entry: entry:
@ -30,7 +30,7 @@ B4:
!0 = !{!"branch_weights", i32 4, i32 124} !0 = !{!"branch_weights", i32 4, i32 124}
; CHECK: Machine code for function test1: ; CHECK: Machine code for function test1:
; CHECK: Successors according to CFG: BB#1(8) BB#2(248) ; CHECK: Successors according to CFG: BB#1(67108864) BB#2(2080374784)
@g0 = common global i32 0, align 4 @g0 = common global i32 0, align 4

8
llvm/test/CodeGen/Generic/MachineBranchProb.ll
View File

@ -16,11 +16,11 @@ entry:
i64 5, label %sw.bb1 i64 5, label %sw.bb1
], !prof !0 ], !prof !0
; CHECK: BB#0: derived from LLVM BB %entry ; CHECK: BB#0: derived from LLVM BB %entry
; CHECK: Successors according to CFG: BB#2(64) BB#4(21) ; CHECK: Successors according to CFG: BB#2(1616928864) BB#4(530554784)
; CHECK: BB#4: derived from LLVM BB %entry ; CHECK: BB#4: derived from LLVM BB %entry
; CHECK: Successors according to CFG: BB#1(10) BB#5(11) ; CHECK: Successors according to CFG: BB#1(252645135) BB#5(277909649)
; CHECK: BB#5: derived from LLVM BB %entry ; CHECK: BB#5: derived from LLVM BB %entry
; CHECK: Successors according to CFG: BB#1(4) BB#3(7) ; CHECK: Successors according to CFG: BB#1(101058054) BB#3(176851595)
sw.bb: sw.bb:
br label %return br label %return
@ -62,7 +62,7 @@ return: ret void
; CHECK-LABEL: Machine code for function left_leaning_weight_balanced_tree: ; CHECK-LABEL: Machine code for function left_leaning_weight_balanced_tree:
; CHECK: BB#0: derived from LLVM BB %entry ; CHECK: BB#0: derived from LLVM BB %entry
; CHECK-NOT: Successors ; CHECK-NOT: Successors
; CHECK: Successors according to CFG: BB#8(13) BB#9(20) ; CHECK: Successors according to CFG: BB#8(852677332) BB#9(1294806318)
} }
!1 = !{!"branch_weights", !1 = !{!"branch_weights",

2
llvm/test/CodeGen/Hexagon/ifcvt-edge-weight.ll
View File

@ -2,7 +2,7 @@
; Check that the edge weights are updated correctly after if-conversion. ; Check that the edge weights are updated correctly after if-conversion.
; CHECK: BB#3: ; CHECK: BB#3:
; CHECK: Successors according to CFG: BB#2(10) BB#1(90) ; CHECK: Successors according to CFG: BB#2(214748365) BB#1(1932735283)
@a = external global i32 @a = external global i32
@d = external global i32 @d = external global i32

20
llvm/test/CodeGen/PowerPC/sjlj.ll
View File

@ -74,24 +74,24 @@ return: ; preds = %if.end, %if.then
; CHECK-DAG: std [[REGA]], [[OFF:[0-9]+]](31) # 8-byte Folded Spill ; CHECK-DAG: std [[REGA]], [[OFF:[0-9]+]](31) # 8-byte Folded Spill
; CHECK-DAG: std 1, 16([[REGA]]) ; CHECK-DAG: std 1, 16([[REGA]])
; CHECK-DAG: std 2, 24([[REGA]]) ; CHECK-DAG: std 2, 24([[REGA]])
; CHECK: bcl 20, 31, .LBB1_1 ; CHECK: bcl 20, 31, .LBB1_5
; CHECK: li 3, 1 ; CHECK: li 3, 1
; CHECK: #EH_SjLj_Setup .LBB1_1 ; CHECK: #EH_SjLj_Setup .LBB1_5
; CHECK: b .LBB1_2 ; CHECK: b .LBB1_1
; CHECK: .LBB1_1: ; CHECK: .LBB1_4:
; CHECK: mflr [[REGL:[0-9]+]]
; CHECK: ld [[REG2:[0-9]+]], [[OFF]](31) # 8-byte Folded Reload
; CHECK: std [[REGL]], 8([[REG2]])
; CHECK: li 3, 0
; CHECK: .LBB1_2:
; CHECK: lfd ; CHECK: lfd
; CHECK: lvx ; CHECK: lvx
; CHECK: ld ; CHECK: ld
; CHECK: blr ; CHECK: blr
; CHECK: .LBB1_5:
; CHECK: mflr [[REGL:[0-9]+]]
; CHECK: ld [[REG2:[0-9]+]], [[OFF]](31) # 8-byte Folded Reload
; CHECK: std [[REGL]], 8([[REG2]])
; CHECK: li 3, 0
; CHECK-NOAV: @main ; CHECK-NOAV: @main
; CHECK-NOAV-NOT: stvx ; CHECK-NOAV-NOT: stvx
; CHECK-NOAV: bcl ; CHECK-NOAV: bcl

4
llvm/test/CodeGen/X86/MachineBranchProb.ll
View File

@ -18,9 +18,9 @@ for.cond2: ; preds = %for.inc, %for.cond
%or.cond = or i1 %tobool, %cmp4 %or.cond = or i1 %tobool, %cmp4
br i1 %or.cond, label %for.inc20, label %for.inc, !prof !0 br i1 %or.cond, label %for.inc20, label %for.inc, !prof !0
; CHECK: BB#1: derived from LLVM BB %for.cond2 ; CHECK: BB#1: derived from LLVM BB %for.cond2
; CHECK: Successors according to CFG: BB#3(56008718) BB#4(3615818718) ; CHECK: Successors according to CFG: BB#3(32756933) BB#4(2114726715)
; CHECK: BB#4: derived from LLVM BB %for.cond2 ; CHECK: BB#4: derived from LLVM BB %for.cond2
; CHECK: Successors according to CFG: BB#3(56008718) BB#2(3559810000) ; CHECK: Successors according to CFG: BB#3(33264335) BB#2(2114219313)
for.inc: ; preds = %for.cond2 for.inc: ; preds = %for.cond2
%shl = shl i32 %bit.0, 1 %shl = shl i32 %bit.0, 1

2
llvm/test/CodeGen/X86/catchpad-weight.ll
View File

@ -2,7 +2,7 @@
; Check if the edge weight to the catchpad is calculated correctly. ; Check if the edge weight to the catchpad is calculated correctly.
; CHECK: Successors according to CFG: BB#3(1048575) BB#1(1) BB#4(1) BB#6(1) BB#8(1) ; CHECK: Successors according to CFG: BB#3(2147481600) BB#1(2048) BB#4(1024) BB#6(512) BB#8(256)
target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64--windows-msvc18.0.0" target triple = "x86_64--windows-msvc18.0.0"

4
llvm/test/CodeGen/X86/stack-protector-weight.ll
View File

@ -2,13 +2,13 @@
; RUN: llc -mtriple=x86_64-apple-darwin -print-machineinstrs=expand-isel-pseudos -enable-selectiondag-sp=false %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=IR ; RUN: llc -mtriple=x86_64-apple-darwin -print-machineinstrs=expand-isel-pseudos -enable-selectiondag-sp=false %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=IR
; SELDAG: # Machine code for function test_branch_weights: ; SELDAG: # Machine code for function test_branch_weights:
; SELDAG: Successors according to CFG: BB#[[SUCCESS:[0-9]+]](1048575) BB#[[FAILURE:[0-9]+]](1) ; SELDAG: Successors according to CFG: BB#[[SUCCESS:[0-9]+]](2147481600) BB#[[FAILURE:[0-9]+]](2048)
; SELDAG: BB#[[FAILURE]]: ; SELDAG: BB#[[FAILURE]]:
; SELDAG: CALL64pcrel32 <es:__stack_chk_fail> ; SELDAG: CALL64pcrel32 <es:__stack_chk_fail>
; SELDAG: BB#[[SUCCESS]]: ; SELDAG: BB#[[SUCCESS]]:
; IR: # Machine code for function test_branch_weights: ; IR: # Machine code for function test_branch_weights:
; IR: Successors according to CFG: BB#[[SUCCESS:[0-9]+]](1048575) BB#[[FAILURE:[0-9]+]](1) ; IR: Successors according to CFG: BB#[[SUCCESS:[0-9]+]](2147481600) BB#[[FAILURE:[0-9]+]](2048)
; IR: BB#[[SUCCESS]]: ; IR: BB#[[SUCCESS]]:
; IR: BB#[[FAILURE]]: ; IR: BB#[[FAILURE]]:
; IR: CALL64pcrel32 <ga:@__stack_chk_fail> ; IR: CALL64pcrel32 <ga:@__stack_chk_fail>

22
llvm/test/CodeGen/X86/switch-edge-weight.ll
View File

@ -34,22 +34,22 @@ sw.epilog:
; CHECK: BB#0: ; CHECK: BB#0:
; BB#0 to BB#4: [0, 1133] (65 = 60 + 5) ; BB#0 to BB#4: [0, 1133] (65 = 60 + 5)
; BB#0 to BB#5: [1134, UINT32_MAX] (25 = 20 + 5) ; BB#0 to BB#5: [1134, UINT32_MAX] (25 = 20 + 5)
; CHECK: Successors according to CFG: BB#4(65) BB#5(25) ; CHECK: Successors according to CFG: BB#4(1550960411) BB#5(596523235)
; ;
; CHECK: BB#4: ; CHECK: BB#4:
; BB#4 to BB#1: [155, 159] (50) ; BB#4 to BB#1: [155, 159] (50)
; BB#4 to BB#5: [0, 1133] - [155, 159] (15 = 10 + 5) ; BB#4 to BB#5: [0, 1133] - [155, 159] (15 = 10 + 5)
; CHECK: Successors according to CFG: BB#1(50) BB#7(15) ; CHECK: Successors according to CFG: BB#1(1193046470) BB#7(357913941)
; ;
; CHECK: BB#5: ; CHECK: BB#5:
; BB#5 to BB#1: {1140} (10) ; BB#5 to BB#1: {1140} (10)
; BB#5 to BB#6: [1134, UINT32_MAX] - {1140} (15 = 10 + 5) ; BB#5 to BB#6: [1134, UINT32_MAX] - {1140} (15 = 10 + 5)
; CHECK: Successors according to CFG: BB#1(10) BB#6(15) ; CHECK: Successors according to CFG: BB#1(238609294) BB#6(357913941)
; ;
; CHECK: BB#6: ; CHECK: BB#6:
; BB#6 to BB#1: {1134} (10) ; BB#6 to BB#1: {1134} (10)
; BB#6 to BB#2: [1134, UINT32_MAX] - {1134, 1140} (5) ; BB#6 to BB#2: [1134, UINT32_MAX] - {1134, 1140} (5)
; CHECK: Successors according to CFG: BB#1(10) BB#2(5) ; CHECK: Successors according to CFG: BB#1(238609294) BB#2(119304647)
} }
; CHECK-LABEL: test2 ; CHECK-LABEL: test2
@ -102,7 +102,7 @@ sw.epilog:
; CHECK: BB#0: ; CHECK: BB#0:
; BB#0 to BB#6: {0} + [15, UINT32_MAX] (5) ; BB#0 to BB#6: {0} + [15, UINT32_MAX] (5)
; BB#0 to BB#8: [1, 14] (jump table) (65 = 60 + 5) ; BB#0 to BB#8: [1, 14] (jump table) (65 = 60 + 5)
; CHECK: Successors according to CFG: BB#6(5) BB#8(65) ; CHECK: Successors according to CFG: BB#6(153391689) BB#8(1994091957)
; ;
; CHECK: BB#8: ; CHECK: BB#8:
; BB#8 to BB#1: {1} (10) ; BB#8 to BB#1: {1} (10)
@ -111,7 +111,7 @@ sw.epilog:
; BB#8 to BB#3: {11} (10) ; BB#8 to BB#3: {11} (10)
; BB#8 to BB#4: {12} (10) ; BB#8 to BB#4: {12} (10)
; BB#8 to BB#5: {13, 14} (20) ; BB#8 to BB#5: {13, 14} (20)
; CHECK: Successors according to CFG: BB#1(10) BB#6(5) BB#2(10) BB#3(10) BB#4(10) BB#5(20) ; CHECK: Successors according to CFG: BB#1(306783378) BB#6(153391689) BB#2(306783378) BB#3(306783378) BB#4(306783378) BB#5(613566756)
} }
; CHECK-LABEL: test3 ; CHECK-LABEL: test3
@ -163,7 +163,7 @@ sw.epilog:
; CHECK: BB#0: ; CHECK: BB#0:
; BB#0 to BB#6: [0, 9] + [15, UINT32_MAX] {10} ; BB#0 to BB#6: [0, 9] + [15, UINT32_MAX] {10}
; BB#0 to BB#8: [10, 14] (jump table) (50) ; BB#0 to BB#8: [10, 14] (jump table) (50)
; CHECK: Successors according to CFG: BB#6(10) BB#8(50) ; CHECK: Successors according to CFG: BB#6(357913941) BB#8(1789569705)
; ;
; CHECK: BB#8: ; CHECK: BB#8:
; BB#8 to BB#1: {10} (10) ; BB#8 to BB#1: {10} (10)
@ -171,7 +171,7 @@ sw.epilog:
; BB#8 to BB#3: {12} (10) ; BB#8 to BB#3: {12} (10)
; BB#8 to BB#4: {13} (10) ; BB#8 to BB#4: {13} (10)
; BB#8 to BB#5: {14} (10) ; BB#8 to BB#5: {14} (10)
; CHECK: Successors according to CFG: BB#1(10) BB#2(10) BB#3(10) BB#4(10) BB#5(10) ; CHECK: Successors according to CFG: BB#1(357913941) BB#2(357913941) BB#3(357913941) BB#4(357913941) BB#5(357913941)
} }
; CHECK-LABEL: test4 ; CHECK-LABEL: test4
@ -216,12 +216,12 @@ sw.epilog:
; CHECK: BB#0: ; CHECK: BB#0:
; BB#0 to BB#6: [0, 110] + [116, UINT32_MAX] (20) ; BB#0 to BB#6: [0, 110] + [116, UINT32_MAX] (20)
; BB#0 to BB#7: [111, 115] (bit test) (50) ; BB#0 to BB#7: [111, 115] (bit test) (50)
; CHECK: Successors according to CFG: BB#6(20) BB#7(50) ; CHECK: Successors according to CFG: BB#6(613566756) BB#7(1533916890)
; ;
; CHECK: BB#7: ; CHECK: BB#7:
; BB#7 to BB#2: {111, 114, 115} (30) ; BB#7 to BB#2: {111, 114, 115} (30)
; BB#7 to BB#3: {112, 113} (20) ; BB#7 to BB#3: {112, 113} (20)
; CHECK: Successors according to CFG: BB#2(30) BB#3(20) ; CHECK: Successors according to CFG: BB#2(920350134) BB#3(613566756)
} }
; CHECK-LABEL: test5 ; CHECK-LABEL: test5
@ -273,7 +273,7 @@ sw.epilog:
; CHECK: BB#0: ; CHECK: BB#0:
; BB#0 to BB#6: [10, UINT32_MAX] (15) ; BB#0 to BB#6: [10, UINT32_MAX] (15)
; BB#0 to BB#8: [1, 5, 7, 9] (jump table) (45) ; BB#0 to BB#8: [1, 5, 7, 9] (jump table) (45)
; CHECK: Successors according to CFG: BB#8(15) BB#9(45) ; CHECK: Successors according to CFG: BB#8(536870912) BB#9(1610612734)
} }
!1 = !{!"branch_weights", i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10} !1 = !{!"branch_weights", i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10}

4
llvm/test/CodeGen/X86/switch-jump-table.ll
View File

@ -55,8 +55,8 @@ default:
define void @bar(i32 %x, i32* %to) { define void @bar(i32 %x, i32* %to) {
; CHECK-JT-WEIGHT-LABEL: bar: ; CHECK-JT-WEIGHT-LABEL: bar:
; CHECK-JT-WEIGHT: Successors according to CFG: BB#6(16) BB#8(96) ; CHECK-JT-WEIGHT: Successors according to CFG: BB#6(306783378) BB#8(1840700268)
; CHECK-JT-WEIGHT: Successors according to CFG: BB#1(16) BB#2(16) BB#3(16) BB#4(16) BB#5(32) ; CHECK-JT-WEIGHT: Successors according to CFG: BB#1(306783378) BB#2(306783378) BB#3(306783378) BB#4(306783378) BB#5(613566756)
entry: entry:
switch i32 %x, label %default [ switch i32 %x, label %default [