forked from OSchip/llvm-project
[clang][CodeGen] Lower Likelihood attributes to @llvm.expect intrin instead of branch weights
08196e0b2e exposed LowerExpectIntrinsic's
internal implementation detail in the form of
LikelyBranchWeight/UnlikelyBranchWeight options to the outside.
While this isn't incorrect from the results viewpoint,
this is suboptimal from the layering viewpoint,
and causes confusion - should transforms also use those weights,
or should they use something else, D98898?
So go back to status quo by making LikelyBranchWeight/UnlikelyBranchWeight
internal again, and fixing all the code that used it directly,
which currently is only clang codegen, thankfully,
to emit proper @llvm.expect intrinsics instead.This commit is contained in:
Roman Lebedev
parent
37d6be9052
commit
e3a4701627
|
|
@ -821,8 +821,11 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
|
|||
llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
|
||||
if (ConditionScope.requiresCleanups())
|
||||
ExitBlock = createBasicBlock("while.exit");
|
||||
llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
|
||||
S.getCond(), getProfileCount(S.getBody()), S.getBody());
|
||||
llvm::MDNode *Weights =
|
||||
createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
|
||||
if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
|
||||
BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
|
||||
BoolCondVal, Stmt::getLikelihood(S.getBody()));
|
||||
Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, Weights);
|
||||
|
||||
if (ExitBlock != LoopExit.getBlock()) {
|
||||
|
|
@ -1008,8 +1011,11 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
|
|||
// C99 6.8.5p2/p4: The first substatement is executed if the expression
|
||||
// compares unequal to 0. The condition must be a scalar type.
|
||||
llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
|
||||
llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
|
||||
S.getCond(), getProfileCount(S.getBody()), S.getBody());
|
||||
llvm::MDNode *Weights =
|
||||
createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
|
||||
if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
|
||||
BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
|
||||
BoolCondVal, Stmt::getLikelihood(S.getBody()));
|
||||
|
||||
if (llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal))
|
||||
if (C->isOne())
|
||||
|
|
@ -1094,8 +1100,11 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
|
|||
// The body is executed if the expression, contextually converted
|
||||
// to bool, is true.
|
||||
llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
|
||||
llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
|
||||
S.getCond(), getProfileCount(S.getBody()), S.getBody());
|
||||
llvm::MDNode *Weights =
|
||||
createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
|
||||
if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
|
||||
BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
|
||||
BoolCondVal, Stmt::getLikelihood(S.getBody()));
|
||||
Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights);
|
||||
|
||||
if (ExitBlock != LoopExit.getBlock()) {
|
||||
|
|
@ -1369,7 +1378,7 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S,
|
|||
// this case.
|
||||
(*SwitchWeights)[0] += ThisCount;
|
||||
} else if (SwitchLikelihood)
|
||||
Weights = createBranchWeights(LH);
|
||||
Cond = emitCondLikelihoodViaExpectIntrinsic(Cond, LH);
|
||||
|
||||
Builder.CreateCondBr(Cond, CaseDest, FalseDest, Weights);
|
||||
|
||||
|
|
|
|||
|
|
@ -1764,31 +1764,39 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
|
|||
return;
|
||||
}
|
||||
|
||||
// If the branch has a condition wrapped by __builtin_unpredictable,
|
||||
// create metadata that specifies that the branch is unpredictable.
|
||||
// Don't bother if not optimizing because that metadata would not be used.
|
||||
llvm::MDNode *Unpredictable = nullptr;
|
||||
auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts());
|
||||
if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
|
||||
auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
|
||||
if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
|
||||
llvm::MDBuilder MDHelper(getLLVMContext());
|
||||
Unpredictable = MDHelper.createUnpredictable();
|
||||
}
|
||||
}
|
||||
|
||||
llvm::MDNode *Weights = createBranchWeights(LH);
|
||||
if (!Weights) {
|
||||
uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
|
||||
Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
|
||||
}
|
||||
|
||||
// Emit the code with the fully general case.
|
||||
llvm::Value *CondV;
|
||||
{
|
||||
ApplyDebugLocation DL(*this, Cond);
|
||||
CondV = EvaluateExprAsBool(Cond);
|
||||
}
|
||||
|
||||
llvm::MDNode *Weights = nullptr;
|
||||
llvm::MDNode *Unpredictable = nullptr;
|
||||
|
||||
// If optimizing, lower unpredictability/probability knowledge about cond.
|
||||
if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
|
||||
// If the branch has a condition wrapped by __builtin_unpredictable,
|
||||
// create metadata that specifies that the branch is unpredictable.
|
||||
if (auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts())) {
|
||||
auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
|
||||
if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
|
||||
llvm::MDBuilder MDHelper(getLLVMContext());
|
||||
Unpredictable = MDHelper.createUnpredictable();
|
||||
}
|
||||
}
|
||||
|
||||
// If there is a Likelihood knowledge for the cond, lower it.
|
||||
llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(CondV, LH);
|
||||
if (CondV != NewCondV)
|
||||
CondV = NewCondV;
|
||||
else {
|
||||
// Otherwise, lower profile counts.
|
||||
uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
|
||||
Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
|
||||
}
|
||||
}
|
||||
|
||||
Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
|
||||
}
|
||||
|
||||
|
|
@ -2632,35 +2640,26 @@ llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
|
|||
return llvm::DebugLoc();
|
||||
}
|
||||
|
||||
static Optional<std::pair<uint32_t, uint32_t>>
|
||||
getLikelihoodWeights(Stmt::Likelihood LH) {
|
||||
llvm::Value *
|
||||
CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
|
||||
Stmt::Likelihood LH) {
|
||||
switch (LH) {
|
||||
case Stmt::LH_Unlikely:
|
||||
return std::pair<uint32_t, uint32_t>(llvm::UnlikelyBranchWeight,
|
||||
llvm::LikelyBranchWeight);
|
||||
case Stmt::LH_None:
|
||||
return None;
|
||||
return Cond;
|
||||
case Stmt::LH_Likely:
|
||||
return std::pair<uint32_t, uint32_t>(llvm::LikelyBranchWeight,
|
||||
llvm::UnlikelyBranchWeight);
|
||||
case Stmt::LH_Unlikely:
|
||||
// Don't generate llvm.expect on -O0 as the backend won't use it for
|
||||
// anything.
|
||||
if (CGM.getCodeGenOpts().OptimizationLevel == 0)
|
||||
return Cond;
|
||||
llvm::Type *CondTy = Cond->getType();
|
||||
assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
|
||||
llvm::Function *FnExpect =
|
||||
CGM.getIntrinsic(llvm::Intrinsic::expect, CondTy);
|
||||
llvm::Value *ExpectedValueOfCond =
|
||||
llvm::ConstantInt::getBool(CondTy, LH == Stmt::LH_Likely);
|
||||
return Builder.CreateCall(FnExpect, {Cond, ExpectedValueOfCond},
|
||||
Cond->getName() + ".expval");
|
||||
}
|
||||
llvm_unreachable("Unknown Likelihood");
|
||||
}
|
||||
|
||||
llvm::MDNode *CodeGenFunction::createBranchWeights(Stmt::Likelihood LH) const {
|
||||
Optional<std::pair<uint32_t, uint32_t>> LHW = getLikelihoodWeights(LH);
|
||||
if (!LHW)
|
||||
return nullptr;
|
||||
|
||||
llvm::MDBuilder MDHelper(CGM.getLLVMContext());
|
||||
return MDHelper.createBranchWeights(LHW->first, LHW->second);
|
||||
}
|
||||
|
||||
llvm::MDNode *CodeGenFunction::createProfileOrBranchWeightsForLoop(
|
||||
const Stmt *Cond, uint64_t LoopCount, const Stmt *Body) const {
|
||||
llvm::MDNode *Weights = createProfileWeightsForLoop(Cond, LoopCount);
|
||||
if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
|
||||
Weights = createBranchWeights(Stmt::getLikelihood(Body));
|
||||
|
||||
return Weights;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1445,8 +1445,9 @@ private:
|
|||
};
|
||||
OpenMPCancelExitStack OMPCancelStack;
|
||||
|
||||
/// Calculate branch weights for the likelihood attribute
|
||||
llvm::MDNode *createBranchWeights(Stmt::Likelihood LH) const;
|
||||
/// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.
|
||||
llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
|
||||
Stmt::Likelihood LH);
|
||||
|
||||
CodeGenPGO PGO;
|
||||
|
||||
|
|
@ -1457,13 +1458,6 @@ private:
|
|||
llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,
|
||||
uint64_t LoopCount) const;
|
||||
|
||||
/// Calculate the branch weight for PGO data or the likelihood attribute.
|
||||
/// The function tries to get the weight of \ref createProfileWeightsForLoop.
|
||||
/// If that fails it gets the weight of \ref createBranchWeights.
|
||||
llvm::MDNode *createProfileOrBranchWeightsForLoop(const Stmt *Cond,
|
||||
uint64_t LoopCount,
|
||||
const Stmt *Body) const;
|
||||
|
||||
public:
|
||||
/// Increment the profiler's counter for the given statement by \p StepV.
|
||||
/// If \p StepV is null, the default increment is 1.
|
||||
|
|
|
|||
|
|
@ -1,60 +1,263 @@
|
|||
// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
|
||||
// RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu -verify
|
||||
// RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu | FileCheck %s
|
||||
|
||||
// CHECK-LABEL: @_Z2wli(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4
|
||||
// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2:![0-9]+]]
|
||||
// CHECK-NEXT: br label [[WHILE_COND:%.*]]
|
||||
// CHECK: while.cond:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0
|
||||
// CHECK-NEXT: [[TOBOOL_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[TOBOOL]], i1 true)
|
||||
// CHECK-NEXT: br i1 [[TOBOOL_EXPVAL]], label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]]
|
||||
// CHECK: while.body:
|
||||
// CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
|
||||
// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: br label [[WHILE_COND]], !llvm.loop [[LOOP6:![0-9]+]]
|
||||
// CHECK: while.end:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void wl(int e){
|
||||
// CHECK-LABEL: define{{.*}}wl
|
||||
// CHECK: br {{.*}} !prof !6
|
||||
while(e) [[likely]] ++e;
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z2wui(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4
|
||||
// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: br label [[WHILE_COND:%.*]]
|
||||
// CHECK: while.cond:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0
|
||||
// CHECK-NEXT: [[TOBOOL_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[TOBOOL]], i1 false)
|
||||
// CHECK-NEXT: br i1 [[TOBOOL_EXPVAL]], label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]]
|
||||
// CHECK: while.body:
|
||||
// CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
|
||||
// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: br label [[WHILE_COND]], !llvm.loop [[LOOP9:![0-9]+]]
|
||||
// CHECK: while.end:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void wu(int e){
|
||||
// CHECK-LABEL: define{{.*}}wu
|
||||
// CHECK: br {{.*}} !prof !10
|
||||
while(e) [[unlikely]] ++e;
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z15w_branch_elidedj(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4
|
||||
// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: br label [[WHILE_BODY:%.*]]
|
||||
// CHECK: while.body:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[INC:%.*]] = add i32 [[TMP0]], 1
|
||||
// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: br label [[WHILE_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
|
||||
//
|
||||
void w_branch_elided(unsigned e){
|
||||
// CHECK-LABEL: define{{.*}}w_branch_elided
|
||||
// CHECK-NOT: br {{.*}} !prof
|
||||
// expected-warning@+2 {{attribute 'likely' has no effect when annotating an infinite loop}}
|
||||
// expected-note@+1 {{annotating the infinite loop here}}
|
||||
while(1) [[likely]] ++e;
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z2flj(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4
|
||||
// CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
|
||||
// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32* [[I]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP0]]) #[[ATTR4:[0-9]+]]
|
||||
// CHECK-NEXT: store i32 0, i32* [[I]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: br label [[FOR_COND:%.*]]
|
||||
// CHECK: for.cond:
|
||||
// CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[I]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[TMP1]], [[TMP2]]
|
||||
// CHECK-NEXT: [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 true)
|
||||
// CHECK-NEXT: br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
|
||||
// CHECK: for.cond.cleanup:
|
||||
// CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[I]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP3]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: br label [[FOR_END:%.*]]
|
||||
// CHECK: for.body:
|
||||
// CHECK-NEXT: br label [[FOR_INC:%.*]]
|
||||
// CHECK: for.inc:
|
||||
// CHECK-NEXT: [[TMP4:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[INC:%.*]] = add i32 [[TMP4]], 1
|
||||
// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
|
||||
// CHECK: for.end:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void fl(unsigned e)
|
||||
{
|
||||
// CHECK-LABEL: define{{.*}}fl
|
||||
// CHECK: br {{.*}} !prof !6
|
||||
for(int i = 0; i != e; ++e) [[likely]];
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z2fui(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4
|
||||
// CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
|
||||
// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32* [[I]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP0]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: store i32 0, i32* [[I]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: br label [[FOR_COND:%.*]]
|
||||
// CHECK: for.cond:
|
||||
// CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[I]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[TMP1]], [[TMP2]]
|
||||
// CHECK-NEXT: [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 false)
|
||||
// CHECK-NEXT: br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
|
||||
// CHECK: for.cond.cleanup:
|
||||
// CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[I]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP3]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: br label [[FOR_END:%.*]]
|
||||
// CHECK: for.body:
|
||||
// CHECK-NEXT: br label [[FOR_INC:%.*]]
|
||||
// CHECK: for.inc:
|
||||
// CHECK-NEXT: [[TMP4:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP4]], 1
|
||||
// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
|
||||
// CHECK: for.end:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void fu(int e)
|
||||
{
|
||||
// CHECK-LABEL: define{{.*}}fu
|
||||
// CHECK: br {{.*}} !prof !10
|
||||
for(int i = 0; i != e; ++e) [[unlikely]];
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z15f_branch_elidedv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: br label [[FOR_COND:%.*]]
|
||||
// CHECK: for.cond:
|
||||
// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP13:![0-9]+]]
|
||||
//
|
||||
void f_branch_elided()
|
||||
{
|
||||
// CHECK-LABEL: define{{.*}}f_branch_elided
|
||||
// CHECK-NOT: br {{.*}} !prof
|
||||
for(;;) [[likely]];
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z3frlOA4_i(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[E_ADDR:%.*]] = alloca [4 x i32]*, align 8
|
||||
// CHECK-NEXT: [[__RANGE1:%.*]] = alloca [4 x i32]*, align 8
|
||||
// CHECK-NEXT: [[__BEGIN1:%.*]] = alloca i32*, align 8
|
||||
// CHECK-NEXT: [[__END1:%.*]] = alloca i32*, align 8
|
||||
// CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
|
||||
// CHECK-NEXT: store [4 x i32]* [[E:%.*]], [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14:![0-9]+]]
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP0]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: [[TMP1:%.*]] = load [4 x i32]*, [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: store [4 x i32]* [[TMP1]], [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[TMP2:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP2]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: [[TMP3:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP3]], i64 0, i64 0
|
||||
// CHECK-NEXT: store i32* [[ARRAYDECAY]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[TMP4:%.*]] = bitcast i32** [[__END1]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP4]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: [[TMP5:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP5]], i64 0, i64 0
|
||||
// CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAYDECAY1]], i64 4
|
||||
// CHECK-NEXT: store i32* [[ADD_PTR]], i32** [[__END1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: br label [[FOR_COND:%.*]]
|
||||
// CHECK: for.cond:
|
||||
// CHECK-NEXT: [[TMP6:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[TMP7:%.*]] = load i32*, i32** [[__END1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[CMP:%.*]] = icmp ne i32* [[TMP6]], [[TMP7]]
|
||||
// CHECK-NEXT: [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 true)
|
||||
// CHECK-NEXT: br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
|
||||
// CHECK: for.cond.cleanup:
|
||||
// CHECK-NEXT: [[TMP8:%.*]] = bitcast i32** [[__END1]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP8]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: [[TMP9:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP9]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: [[TMP10:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP10]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: br label [[FOR_END:%.*]]
|
||||
// CHECK: for.body:
|
||||
// CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[I]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP11]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: [[TMP12:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: store i32 [[TMP13]], i32* [[I]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[TMP14:%.*]] = bitcast i32* [[I]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP14]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: br label [[FOR_INC:%.*]]
|
||||
// CHECK: for.inc:
|
||||
// CHECK-NEXT: [[TMP15:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[TMP15]], i32 1
|
||||
// CHECK-NEXT: store i32* [[INCDEC_PTR]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
|
||||
// CHECK: for.end:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void frl(int (&&e) [4])
|
||||
{
|
||||
// CHECK-LABEL: define{{.*}}frl
|
||||
// CHECK: br {{.*}} !prof !6
|
||||
for(int i : e) [[likely]];
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z3fruOA4_i(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[E_ADDR:%.*]] = alloca [4 x i32]*, align 8
|
||||
// CHECK-NEXT: [[__RANGE1:%.*]] = alloca [4 x i32]*, align 8
|
||||
// CHECK-NEXT: [[__BEGIN1:%.*]] = alloca i32*, align 8
|
||||
// CHECK-NEXT: [[__END1:%.*]] = alloca i32*, align 8
|
||||
// CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
|
||||
// CHECK-NEXT: store [4 x i32]* [[E:%.*]], [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP0]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: [[TMP1:%.*]] = load [4 x i32]*, [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: store [4 x i32]* [[TMP1]], [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[TMP2:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP2]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: [[TMP3:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP3]], i64 0, i64 0
|
||||
// CHECK-NEXT: store i32* [[ARRAYDECAY]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[TMP4:%.*]] = bitcast i32** [[__END1]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP4]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: [[TMP5:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP5]], i64 0, i64 0
|
||||
// CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAYDECAY1]], i64 4
|
||||
// CHECK-NEXT: store i32* [[ADD_PTR]], i32** [[__END1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: br label [[FOR_COND:%.*]]
|
||||
// CHECK: for.cond:
|
||||
// CHECK-NEXT: [[TMP6:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[TMP7:%.*]] = load i32*, i32** [[__END1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[CMP:%.*]] = icmp ne i32* [[TMP6]], [[TMP7]]
|
||||
// CHECK-NEXT: [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 false)
|
||||
// CHECK-NEXT: br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
|
||||
// CHECK: for.cond.cleanup:
|
||||
// CHECK-NEXT: [[TMP8:%.*]] = bitcast i32** [[__END1]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP8]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: [[TMP9:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP9]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: [[TMP10:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP10]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: br label [[FOR_END:%.*]]
|
||||
// CHECK: for.body:
|
||||
// CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[I]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP11]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: [[TMP12:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: store i32 [[TMP13]], i32* [[I]], align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[TMP14:%.*]] = bitcast i32* [[I]] to i8*
|
||||
// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP14]]) #[[ATTR4]]
|
||||
// CHECK-NEXT: br label [[FOR_INC:%.*]]
|
||||
// CHECK: for.inc:
|
||||
// CHECK-NEXT: [[TMP15:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[TMP15]], i32 1
|
||||
// CHECK-NEXT: store i32* [[INCDEC_PTR]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
|
||||
// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
|
||||
// CHECK: for.end:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void fru(int (&&e) [4])
|
||||
{
|
||||
// CHECK-LABEL: define{{.*}}fru
|
||||
// CHECK: br {{.*}} !prof !10
|
||||
for(int i : e) [[unlikely]];
|
||||
}
|
||||
|
||||
// CHECK: !6 = !{!"branch_weights", i32 2000, i32 1}
|
||||
// CHECK: !10 = !{!"branch_weights", i32 1, i32 2000}
|
||||
|
|
|
|||
|
|
@ -1,59 +1,111 @@
|
|||
// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
|
||||
// RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu | FileCheck %s
|
||||
|
||||
extern volatile int i;
|
||||
|
||||
// CHECK-LABEL: @_Z8OneCaseLv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2:![0-9]+]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
|
||||
// CHECK-NEXT: i32 1, label [[SW_EPILOG]]
|
||||
// CHECK-NEXT: ], !prof !6
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void OneCaseL() {
|
||||
// CHECK-LABEL: define{{.*}}OneCaseL
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !6
|
||||
switch (i) {
|
||||
[[likely]] case 1: break;
|
||||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z8OneCaseUv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
|
||||
// CHECK-NEXT: i32 1, label [[SW_BB:%.*]]
|
||||
// CHECK-NEXT: ], !prof !7
|
||||
// CHECK: sw.bb:
|
||||
// CHECK-NEXT: [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
|
||||
// CHECK-NEXT: store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: br label [[SW_EPILOG]]
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void OneCaseU() {
|
||||
// CHECK-LABEL: define{{.*}}OneCaseU
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !7
|
||||
switch (i) {
|
||||
[[unlikely]] case 1: ++i; break;
|
||||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z10TwoCasesLNv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
|
||||
// CHECK-NEXT: i32 1, label [[SW_EPILOG]]
|
||||
// CHECK-NEXT: i32 2, label [[SW_EPILOG]]
|
||||
// CHECK-NEXT: ], !prof !8
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void TwoCasesLN() {
|
||||
// CHECK-LABEL: define{{.*}}TwoCasesLN
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !8
|
||||
switch (i) {
|
||||
[[likely]] case 1: break;
|
||||
case 2: break;
|
||||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z10TwoCasesUNv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
|
||||
// CHECK-NEXT: i32 1, label [[SW_EPILOG]]
|
||||
// CHECK-NEXT: i32 2, label [[SW_EPILOG]]
|
||||
// CHECK-NEXT: ], !prof !9
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void TwoCasesUN() {
|
||||
// CHECK-LABEL: define{{.*}}TwoCasesUN
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !9
|
||||
switch (i) {
|
||||
[[unlikely]] case 1: break;
|
||||
case 2: break;
|
||||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z10TwoCasesLUv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
|
||||
// CHECK-NEXT: i32 1, label [[SW_EPILOG]]
|
||||
// CHECK-NEXT: i32 2, label [[SW_EPILOG]]
|
||||
// CHECK-NEXT: ], !prof !10
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void TwoCasesLU() {
|
||||
// CHECK-LABEL: define{{.*}}TwoCasesLU
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !10
|
||||
switch (i) {
|
||||
[[likely]] case 1: break;
|
||||
[[unlikely]] case 2: break;
|
||||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z20CasesFallthroughNNLNv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
|
||||
// CHECK-NEXT: i32 1, label [[SW_BB:%.*]]
|
||||
// CHECK-NEXT: i32 2, label [[SW_BB]]
|
||||
// CHECK-NEXT: i32 3, label [[SW_BB1:%.*]]
|
||||
// CHECK-NEXT: i32 4, label [[SW_BB1]]
|
||||
// CHECK-NEXT: ], !prof !11
|
||||
// CHECK: sw.bb:
|
||||
// CHECK-NEXT: br label [[SW_BB1]]
|
||||
// CHECK: sw.bb1:
|
||||
// CHECK-NEXT: br label [[SW_EPILOG]]
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void CasesFallthroughNNLN() {
|
||||
// CHECK-LABEL: define{{.*}}CasesFallthroughNNLN
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !11
|
||||
switch (i) {
|
||||
case 1:
|
||||
case 2:
|
||||
|
|
@ -62,10 +114,23 @@ void CasesFallthroughNNLN() {
|
|||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z20CasesFallthroughNNUNv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
|
||||
// CHECK-NEXT: i32 1, label [[SW_BB:%.*]]
|
||||
// CHECK-NEXT: i32 2, label [[SW_BB]]
|
||||
// CHECK-NEXT: i32 3, label [[SW_BB1:%.*]]
|
||||
// CHECK-NEXT: i32 4, label [[SW_BB1]]
|
||||
// CHECK-NEXT: ], !prof !12
|
||||
// CHECK: sw.bb:
|
||||
// CHECK-NEXT: br label [[SW_BB1]]
|
||||
// CHECK: sw.bb1:
|
||||
// CHECK-NEXT: br label [[SW_EPILOG]]
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void CasesFallthroughNNUN() {
|
||||
// CHECK-LABEL: define{{.*}}CasesFallthroughNNUN
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !12
|
||||
switch (i) {
|
||||
case 1:
|
||||
case 2:
|
||||
|
|
@ -74,10 +139,32 @@ void CasesFallthroughNNUN() {
|
|||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z28CasesFallthroughRangeSmallLNv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
|
||||
// CHECK-NEXT: i32 1, label [[SW_BB:%.*]]
|
||||
// CHECK-NEXT: i32 2, label [[SW_BB]]
|
||||
// CHECK-NEXT: i32 3, label [[SW_BB]]
|
||||
// CHECK-NEXT: i32 4, label [[SW_BB]]
|
||||
// CHECK-NEXT: i32 5, label [[SW_BB]]
|
||||
// CHECK-NEXT: i32 102, label [[SW_BB1:%.*]]
|
||||
// CHECK-NEXT: i32 103, label [[SW_BB2:%.*]]
|
||||
// CHECK-NEXT: i32 104, label [[SW_BB2]]
|
||||
// CHECK-NEXT: ], !prof !13
|
||||
// CHECK: sw.bb:
|
||||
// CHECK-NEXT: [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
|
||||
// CHECK-NEXT: store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: br label [[SW_BB1]]
|
||||
// CHECK: sw.bb1:
|
||||
// CHECK-NEXT: br label [[SW_BB2]]
|
||||
// CHECK: sw.bb2:
|
||||
// CHECK-NEXT: br label [[SW_EPILOG]]
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void CasesFallthroughRangeSmallLN() {
|
||||
// CHECK-LABEL: define{{.*}}CasesFallthroughRangeSmallLN
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !13
|
||||
switch (i) {
|
||||
case 1 ... 5: ++i;
|
||||
case 102:
|
||||
|
|
@ -86,10 +173,32 @@ void CasesFallthroughRangeSmallLN() {
|
|||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z28CasesFallthroughRangeSmallUNv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
|
||||
// CHECK-NEXT: i32 1, label [[SW_BB:%.*]]
|
||||
// CHECK-NEXT: i32 2, label [[SW_BB]]
|
||||
// CHECK-NEXT: i32 3, label [[SW_BB]]
|
||||
// CHECK-NEXT: i32 4, label [[SW_BB]]
|
||||
// CHECK-NEXT: i32 5, label [[SW_BB]]
|
||||
// CHECK-NEXT: i32 102, label [[SW_BB1:%.*]]
|
||||
// CHECK-NEXT: i32 103, label [[SW_BB2:%.*]]
|
||||
// CHECK-NEXT: i32 104, label [[SW_BB2]]
|
||||
// CHECK-NEXT: ], !prof !14
|
||||
// CHECK: sw.bb:
|
||||
// CHECK-NEXT: [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
|
||||
// CHECK-NEXT: store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: br label [[SW_BB1]]
|
||||
// CHECK: sw.bb1:
|
||||
// CHECK-NEXT: br label [[SW_BB2]]
|
||||
// CHECK: sw.bb2:
|
||||
// CHECK-NEXT: br label [[SW_EPILOG]]
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void CasesFallthroughRangeSmallUN() {
|
||||
// CHECK-LABEL: define{{.*}}CasesFallthroughRangeSmallUN
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !14
|
||||
switch (i) {
|
||||
case 1 ... 5: ++i;
|
||||
case 102:
|
||||
|
|
@ -98,12 +207,26 @@ void CasesFallthroughRangeSmallUN() {
|
|||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z29CasesFallthroughRangeLargeLLNv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_CASERANGE:%.*]] [
|
||||
// CHECK-NEXT: i32 1003, label [[SW_BB1:%.*]]
|
||||
// CHECK-NEXT: i32 104, label [[SW_BB1]]
|
||||
// CHECK-NEXT: ], !prof !8
|
||||
// CHECK: sw.bb:
|
||||
// CHECK-NEXT: br label [[SW_BB1]]
|
||||
// CHECK: sw.bb1:
|
||||
// CHECK-NEXT: br label [[SW_EPILOG:%.*]]
|
||||
// CHECK: sw.caserange:
|
||||
// CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], 0
|
||||
// CHECK-NEXT: [[INBOUNDS:%.*]] = icmp ule i32 [[TMP1]], 64
|
||||
// CHECK-NEXT: [[INBOUNDS_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[INBOUNDS]], i1 true)
|
||||
// CHECK-NEXT: br i1 [[INBOUNDS_EXPVAL]], label [[SW_BB:%.*]], label [[SW_EPILOG]]
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void CasesFallthroughRangeLargeLLN() {
|
||||
// CHECK-LABEL: define{{.*}}CasesFallthroughRangeLargeLLN
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !8
|
||||
// CHECK: caserange
|
||||
// CHECK: br{{.*}} !prof !15
|
||||
switch (i) {
|
||||
[[likely]] case 0 ... 64:
|
||||
[[likely]] case 1003:
|
||||
|
|
@ -111,12 +234,26 @@ void CasesFallthroughRangeLargeLLN() {
|
|||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z29CasesFallthroughRangeLargeUUNv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_CASERANGE:%.*]] [
|
||||
// CHECK-NEXT: i32 1003, label [[SW_BB1:%.*]]
|
||||
// CHECK-NEXT: i32 104, label [[SW_BB1]]
|
||||
// CHECK-NEXT: ], !prof !9
|
||||
// CHECK: sw.bb:
|
||||
// CHECK-NEXT: br label [[SW_BB1]]
|
||||
// CHECK: sw.bb1:
|
||||
// CHECK-NEXT: br label [[SW_EPILOG:%.*]]
|
||||
// CHECK: sw.caserange:
|
||||
// CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], 0
|
||||
// CHECK-NEXT: [[INBOUNDS:%.*]] = icmp ule i32 [[TMP1]], 64
|
||||
// CHECK-NEXT: [[INBOUNDS_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[INBOUNDS]], i1 false)
|
||||
// CHECK-NEXT: br i1 [[INBOUNDS_EXPVAL]], label [[SW_BB:%.*]], label [[SW_EPILOG]]
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void CasesFallthroughRangeLargeUUN() {
|
||||
// CHECK-LABEL: define{{.*}}CasesFallthroughRangeLargeUUN
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !9
|
||||
// CHECK: caserange
|
||||
// CHECK: br{{.*}} !prof !16
|
||||
switch (i) {
|
||||
[[unlikely]] case 0 ... 64:
|
||||
[[unlikely]] case 1003:
|
||||
|
|
@ -124,30 +261,55 @@ void CasesFallthroughRangeLargeUUN() {
|
|||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z15OneCaseDefaultLv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
|
||||
// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]]
|
||||
// CHECK-NEXT: ], !prof !15
|
||||
// CHECK: sw.default:
|
||||
// CHECK-NEXT: br label [[SW_EPILOG]]
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void OneCaseDefaultL() {
|
||||
// CHECK-LABEL: define{{.*}}OneCaseDefaultL
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !17
|
||||
switch (i) {
|
||||
case 1: break;
|
||||
[[likely]] default: break;
|
||||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z15OneCaseDefaultUv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
|
||||
// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]]
|
||||
// CHECK-NEXT: ], !prof !16
|
||||
// CHECK: sw.default:
|
||||
// CHECK-NEXT: br label [[SW_EPILOG]]
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void OneCaseDefaultU() {
|
||||
// CHECK-LABEL: define{{.*}}OneCaseDefaultU
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !18
|
||||
switch (i) {
|
||||
case 1: break;
|
||||
[[unlikely]] default: break;
|
||||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z18TwoCasesDefaultLNLv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
|
||||
// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]]
|
||||
// CHECK-NEXT: i32 2, label [[SW_EPILOG]]
|
||||
// CHECK-NEXT: ], !prof !17
|
||||
// CHECK: sw.default:
|
||||
// CHECK-NEXT: br label [[SW_EPILOG]]
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void TwoCasesDefaultLNL() {
|
||||
// CHECK-LABEL: define{{.*}}TwoCasesDefaultLNL
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !19
|
||||
switch (i) {
|
||||
[[likely]] case 1: break;
|
||||
case 2: break;
|
||||
|
|
@ -155,10 +317,19 @@ void TwoCasesDefaultLNL() {
|
|||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z18TwoCasesDefaultLNNv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
|
||||
// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]]
|
||||
// CHECK-NEXT: i32 2, label [[SW_EPILOG]]
|
||||
// CHECK-NEXT: ], !prof !8
|
||||
// CHECK: sw.default:
|
||||
// CHECK-NEXT: br label [[SW_EPILOG]]
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void TwoCasesDefaultLNN() {
|
||||
// CHECK-LABEL: define{{.*}}TwoCasesDefaultLNN
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !8
|
||||
switch (i) {
|
||||
[[likely]] case 1: break;
|
||||
case 2: break;
|
||||
|
|
@ -166,29 +337,22 @@ void TwoCasesDefaultLNN() {
|
|||
}
|
||||
}
|
||||
|
||||
// CHECK-LABEL: @_Z18TwoCasesDefaultLNUv(
|
||||
// CHECK-NEXT: entry:
|
||||
// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
|
||||
// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
|
||||
// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]]
|
||||
// CHECK-NEXT: i32 2, label [[SW_EPILOG]]
|
||||
// CHECK-NEXT: ], !prof !18
|
||||
// CHECK: sw.default:
|
||||
// CHECK-NEXT: br label [[SW_EPILOG]]
|
||||
// CHECK: sw.epilog:
|
||||
// CHECK-NEXT: ret void
|
||||
//
|
||||
void TwoCasesDefaultLNU() {
|
||||
// CHECK-LABEL: define{{.*}}TwoCasesDefaultLNU
|
||||
// CHECK: switch
|
||||
// CHECK: {{.*}} !prof !20
|
||||
switch (i) {
|
||||
[[likely]] case 1: break;
|
||||
case 2: break;
|
||||
[[unlikely]] default: break;
|
||||
}
|
||||
}
|
||||
|
||||
// CHECK: !6 = !{!"branch_weights", i32 357913942, i32 715827883}
|
||||
// CHECK: !7 = !{!"branch_weights", i32 536870912, i32 1}
|
||||
// CHECK: !8 = !{!"branch_weights", i32 238609295, i32 715827883, i32 238609295}
|
||||
// CHECK: !9 = !{!"branch_weights", i32 357913942, i32 1, i32 357913942}
|
||||
// CHECK: !10 = !{!"branch_weights", i32 357913942, i32 715827883, i32 1}
|
||||
// CHECK: !11 = !{!"branch_weights", i32 143165577, i32 143165577, i32 143165577, i32 715827883, i32 143165577}
|
||||
// CHECK: !12 = !{!"branch_weights", i32 214748365, i32 214748365, i32 214748365, i32 1, i32 214748365}
|
||||
// CHECK: !13 = !{!"branch_weights", i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 715827883, i32 79536432}
|
||||
// CHECK: !14 = !{!"branch_weights", i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 1, i32 119304648}
|
||||
// CHECK: !15 = !{!"branch_weights", i32 2000, i32 1}
|
||||
// CHECK: !16 = !{!"branch_weights", i32 1, i32 2000}
|
||||
// CHECK: !17 = !{!"branch_weights", i32 715827883, i32 357913942}
|
||||
// CHECK: !18 = !{!"branch_weights", i32 1, i32 536870912}
|
||||
// CHECK: !19 = !{!"branch_weights", i32 536870912, i32 536870912, i32 268435456}
|
||||
// CHECK: !20 = !{!"branch_weights", i32 1, i32 715827883, i32 357913942}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@
|
|||
|
||||
#include "llvm/IR/Function.h"
|
||||
#include "llvm/IR/PassManager.h"
|
||||
#include "llvm/Support/CommandLine.h"
|
||||
|
||||
namespace llvm {
|
||||
|
||||
|
|
@ -32,8 +31,6 @@ struct LowerExpectIntrinsicPass : PassInfoMixin<LowerExpectIntrinsicPass> {
|
|||
PreservedAnalyses run(Function &F, FunctionAnalysisManager &);
|
||||
};
|
||||
|
||||
extern cl::opt<uint32_t> LikelyBranchWeight;
|
||||
extern cl::opt<uint32_t> UnlikelyBranchWeight;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@
|
|||
#include "llvm/IR/Metadata.h"
|
||||
#include "llvm/InitializePasses.h"
|
||||
#include "llvm/Pass.h"
|
||||
#include "llvm/Support/CommandLine.h"
|
||||
#include "llvm/Support/Debug.h"
|
||||
#include "llvm/Transforms/Scalar.h"
|
||||
|
||||
|
|
@ -41,15 +42,15 @@ STATISTIC(ExpectIntrinsicsHandled,
|
|||
// only be used in extreme cases, we could make this ratio higher. As it stands,
|
||||
// programmers may be using __builtin_expect() / llvm.expect to annotate that a
|
||||
// branch is likely or unlikely to be taken.
|
||||
//
|
||||
// There is a known dependency on this ratio in CodeGenPrepare when transforming
|
||||
// 'select' instructions. It may be worthwhile to hoist these values to some
|
||||
// shared space, so they can be used directly by other passes.
|
||||
|
||||
cl::opt<uint32_t> llvm::LikelyBranchWeight(
|
||||
// WARNING: these values are internal implementation detail of the pass.
|
||||
// They should not be exposed to the outside of the pass, front-end codegen
|
||||
// should emit @llvm.expect intrinsics instead of using these weights directly.
|
||||
// Transforms should use TargetLowering getPredictableBranchThreshold() hook.
|
||||
static cl::opt<uint32_t> LikelyBranchWeight(
|
||||
"likely-branch-weight", cl::Hidden, cl::init(2000),
|
||||
cl::desc("Weight of the branch likely to be taken (default = 2000)"));
|
||||
cl::opt<uint32_t> llvm::UnlikelyBranchWeight(
|
||||
static cl::opt<uint32_t> UnlikelyBranchWeight(
|
||||
"unlikely-branch-weight", cl::Hidden, cl::init(1),
|
||||
cl::desc("Weight of the branch unlikely to be taken (default = 1)"));
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue