From e3a470162738871bba982416748ae5f5e3572947 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Sun, 21 Mar 2021 22:13:47 +0300 Subject: [PATCH] [clang][CodeGen] Lower Likelihood attributes to @llvm.expect intrin instead of branch weights 08196e0b2e1f8aaa8a854585335c17ba479114df exposed LowerExpectIntrinsic's internal implementation detail in the form of LikelyBranchWeight/UnlikelyBranchWeight options to the outside. While this isn't incorrect from the results viewpoint, this is suboptimal from the layering viewpoint, and causes confusion - should transforms also use those weights, or should they use something else, D98898? So go back to status quo by making LikelyBranchWeight/UnlikelyBranchWeight internal again, and fixing all the code that used it directly, which currently is only clang codegen, thankfully, to emit proper @llvm.expect intrinsics instead. --- clang/lib/CodeGen/CGStmt.cpp | 23 +- clang/lib/CodeGen/CodeGenFunction.cpp | 89 +++--- clang/lib/CodeGen/CodeGenFunction.h | 12 +- .../attr-likelihood-iteration-stmt.cpp | 241 ++++++++++++-- .../attr-likelihood-switch-branch-weights.cpp | 300 ++++++++++++++---- .../Transforms/Scalar/LowerExpectIntrinsic.h | 3 - .../Scalar/LowerExpectIntrinsic.cpp | 13 +- 7 files changed, 524 insertions(+), 157 deletions(-) diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp index 38f3aa941415..fb719efb1a35 100644 --- a/clang/lib/CodeGen/CGStmt.cpp +++ b/clang/lib/CodeGen/CGStmt.cpp @@ -821,8 +821,11 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); if (ConditionScope.requiresCleanups()) ExitBlock = createBasicBlock("while.exit"); - llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop( - S.getCond(), getProfileCount(S.getBody()), S.getBody()); + llvm::MDNode *Weights = + createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody())); + if (!Weights && CGM.getCodeGenOpts().OptimizationLevel) + BoolCondVal = emitCondLikelihoodViaExpectIntrinsic( + BoolCondVal, Stmt::getLikelihood(S.getBody())); Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, Weights); if (ExitBlock != LoopExit.getBlock()) { @@ -1008,8 +1011,11 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, // C99 6.8.5p2/p4: The first substatement is executed if the expression // compares unequal to 0. The condition must be a scalar type. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); - llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop( - S.getCond(), getProfileCount(S.getBody()), S.getBody()); + llvm::MDNode *Weights = + createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody())); + if (!Weights && CGM.getCodeGenOpts().OptimizationLevel) + BoolCondVal = emitCondLikelihoodViaExpectIntrinsic( + BoolCondVal, Stmt::getLikelihood(S.getBody())); if (llvm::ConstantInt *C = dyn_cast(BoolCondVal)) if (C->isOne()) @@ -1094,8 +1100,11 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, // The body is executed if the expression, contextually converted // to bool, is true. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); - llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop( - S.getCond(), getProfileCount(S.getBody()), S.getBody()); + llvm::MDNode *Weights = + createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody())); + if (!Weights && CGM.getCodeGenOpts().OptimizationLevel) + BoolCondVal = emitCondLikelihoodViaExpectIntrinsic( + BoolCondVal, Stmt::getLikelihood(S.getBody())); Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights); if (ExitBlock != LoopExit.getBlock()) { @@ -1369,7 +1378,7 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S, // this case. (*SwitchWeights)[0] += ThisCount; } else if (SwitchLikelihood) - Weights = createBranchWeights(LH); + Cond = emitCondLikelihoodViaExpectIntrinsic(Cond, LH); Builder.CreateCondBr(Cond, CaseDest, FalseDest, Weights); diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp index a00ae74fa165..1c538262737d 100644 --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -1764,31 +1764,39 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, return; } - // If the branch has a condition wrapped by __builtin_unpredictable, - // create metadata that specifies that the branch is unpredictable. - // Don't bother if not optimizing because that metadata would not be used. - llvm::MDNode *Unpredictable = nullptr; - auto *Call = dyn_cast(Cond->IgnoreImpCasts()); - if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) { - auto *FD = dyn_cast_or_null(Call->getCalleeDecl()); - if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) { - llvm::MDBuilder MDHelper(getLLVMContext()); - Unpredictable = MDHelper.createUnpredictable(); - } - } - - llvm::MDNode *Weights = createBranchWeights(LH); - if (!Weights) { - uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount); - Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount); - } - // Emit the code with the fully general case. llvm::Value *CondV; { ApplyDebugLocation DL(*this, Cond); CondV = EvaluateExprAsBool(Cond); } + + llvm::MDNode *Weights = nullptr; + llvm::MDNode *Unpredictable = nullptr; + + // If optimizing, lower unpredictability/probability knowledge about cond. + if (CGM.getCodeGenOpts().OptimizationLevel != 0) { + // If the branch has a condition wrapped by __builtin_unpredictable, + // create metadata that specifies that the branch is unpredictable. + if (auto *Call = dyn_cast(Cond->IgnoreImpCasts())) { + auto *FD = dyn_cast_or_null(Call->getCalleeDecl()); + if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) { + llvm::MDBuilder MDHelper(getLLVMContext()); + Unpredictable = MDHelper.createUnpredictable(); + } + } + + // If there is a Likelihood knowledge for the cond, lower it. + llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(CondV, LH); + if (CondV != NewCondV) + CondV = NewCondV; + else { + // Otherwise, lower profile counts. + uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount); + Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount); + } + } + Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable); } @@ -2632,35 +2640,26 @@ llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) { return llvm::DebugLoc(); } -static Optional> -getLikelihoodWeights(Stmt::Likelihood LH) { +llvm::Value * +CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond, + Stmt::Likelihood LH) { switch (LH) { - case Stmt::LH_Unlikely: - return std::pair(llvm::UnlikelyBranchWeight, - llvm::LikelyBranchWeight); case Stmt::LH_None: - return None; + return Cond; case Stmt::LH_Likely: - return std::pair(llvm::LikelyBranchWeight, - llvm::UnlikelyBranchWeight); + case Stmt::LH_Unlikely: + // Don't generate llvm.expect on -O0 as the backend won't use it for + // anything. + if (CGM.getCodeGenOpts().OptimizationLevel == 0) + return Cond; + llvm::Type *CondTy = Cond->getType(); + assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean"); + llvm::Function *FnExpect = + CGM.getIntrinsic(llvm::Intrinsic::expect, CondTy); + llvm::Value *ExpectedValueOfCond = + llvm::ConstantInt::getBool(CondTy, LH == Stmt::LH_Likely); + return Builder.CreateCall(FnExpect, {Cond, ExpectedValueOfCond}, + Cond->getName() + ".expval"); } llvm_unreachable("Unknown Likelihood"); } - -llvm::MDNode *CodeGenFunction::createBranchWeights(Stmt::Likelihood LH) const { - Optional> LHW = getLikelihoodWeights(LH); - if (!LHW) - return nullptr; - - llvm::MDBuilder MDHelper(CGM.getLLVMContext()); - return MDHelper.createBranchWeights(LHW->first, LHW->second); -} - -llvm::MDNode *CodeGenFunction::createProfileOrBranchWeightsForLoop( - const Stmt *Cond, uint64_t LoopCount, const Stmt *Body) const { - llvm::MDNode *Weights = createProfileWeightsForLoop(Cond, LoopCount); - if (!Weights && CGM.getCodeGenOpts().OptimizationLevel) - Weights = createBranchWeights(Stmt::getLikelihood(Body)); - - return Weights; -} diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index 4bca21a51f9c..11de9166335b 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -1445,8 +1445,9 @@ private: }; OpenMPCancelExitStack OMPCancelStack; - /// Calculate branch weights for the likelihood attribute - llvm::MDNode *createBranchWeights(Stmt::Likelihood LH) const; + /// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin. + llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond, + Stmt::Likelihood LH); CodeGenPGO PGO; @@ -1457,13 +1458,6 @@ private: llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond, uint64_t LoopCount) const; - /// Calculate the branch weight for PGO data or the likelihood attribute. - /// The function tries to get the weight of \ref createProfileWeightsForLoop. - /// If that fails it gets the weight of \ref createBranchWeights. - llvm::MDNode *createProfileOrBranchWeightsForLoop(const Stmt *Cond, - uint64_t LoopCount, - const Stmt *Body) const; - public: /// Increment the profiler's counter for the given statement by \p StepV. /// If \p StepV is null, the default increment is 1. diff --git a/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp b/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp index ec2ee37f97c0..27e8a4999c32 100644 --- a/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp +++ b/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp @@ -1,60 +1,263 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu -verify // RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu | FileCheck %s +// CHECK-LABEL: @_Z2wli( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2:![0-9]+]] +// CHECK-NEXT: br label [[WHILE_COND:%.*]] +// CHECK: while.cond: +// CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0 +// CHECK-NEXT: [[TOBOOL_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[TOBOOL]], i1 true) +// CHECK-NEXT: br i1 [[TOBOOL_EXPVAL]], label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]] +// CHECK: while.body: +// CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1 +// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: br label [[WHILE_COND]], !llvm.loop [[LOOP6:![0-9]+]] +// CHECK: while.end: +// CHECK-NEXT: ret void +// void wl(int e){ - // CHECK-LABEL: define{{.*}}wl - // CHECK: br {{.*}} !prof !6 while(e) [[likely]] ++e; } +// CHECK-LABEL: @_Z2wui( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: br label [[WHILE_COND:%.*]] +// CHECK: while.cond: +// CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0 +// CHECK-NEXT: [[TOBOOL_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[TOBOOL]], i1 false) +// CHECK-NEXT: br i1 [[TOBOOL_EXPVAL]], label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]] +// CHECK: while.body: +// CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1 +// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: br label [[WHILE_COND]], !llvm.loop [[LOOP9:![0-9]+]] +// CHECK: while.end: +// CHECK-NEXT: ret void +// void wu(int e){ - // CHECK-LABEL: define{{.*}}wu - // CHECK: br {{.*}} !prof !10 while(e) [[unlikely]] ++e; } +// CHECK-LABEL: @_Z15w_branch_elidedj( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: br label [[WHILE_BODY:%.*]] +// CHECK: while.body: +// CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[INC:%.*]] = add i32 [[TMP0]], 1 +// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: br label [[WHILE_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +// void w_branch_elided(unsigned e){ - // CHECK-LABEL: define{{.*}}w_branch_elided - // CHECK-NOT: br {{.*}} !prof // expected-warning@+2 {{attribute 'likely' has no effect when annotating an infinite loop}} // expected-note@+1 {{annotating the infinite loop here}} while(1) [[likely]] ++e; } +// CHECK-LABEL: @_Z2flj( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: [[I:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32* [[I]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP0]]) #[[ATTR4:[0-9]+]] +// CHECK-NEXT: store i32 0, i32* [[I]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: br label [[FOR_COND:%.*]] +// CHECK: for.cond: +// CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[I]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[TMP1]], [[TMP2]] +// CHECK-NEXT: [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 true) +// CHECK-NEXT: br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]] +// CHECK: for.cond.cleanup: +// CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[I]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP3]]) #[[ATTR4]] +// CHECK-NEXT: br label [[FOR_END:%.*]] +// CHECK: for.body: +// CHECK-NEXT: br label [[FOR_INC:%.*]] +// CHECK: for.inc: +// CHECK-NEXT: [[TMP4:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[INC:%.*]] = add i32 [[TMP4]], 1 +// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]] +// CHECK: for.end: +// CHECK-NEXT: ret void +// void fl(unsigned e) { - // CHECK-LABEL: define{{.*}}fl - // CHECK: br {{.*}} !prof !6 for(int i = 0; i != e; ++e) [[likely]]; } +// CHECK-LABEL: @_Z2fui( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: [[I:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32* [[I]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP0]]) #[[ATTR4]] +// CHECK-NEXT: store i32 0, i32* [[I]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: br label [[FOR_COND:%.*]] +// CHECK: for.cond: +// CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[I]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[TMP1]], [[TMP2]] +// CHECK-NEXT: [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 false) +// CHECK-NEXT: br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]] +// CHECK: for.cond.cleanup: +// CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[I]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP3]]) #[[ATTR4]] +// CHECK-NEXT: br label [[FOR_END:%.*]] +// CHECK: for.body: +// CHECK-NEXT: br label [[FOR_INC:%.*]] +// CHECK: for.inc: +// CHECK-NEXT: [[TMP4:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP4]], 1 +// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]] +// CHECK: for.end: +// CHECK-NEXT: ret void +// void fu(int e) { - // CHECK-LABEL: define{{.*}}fu - // CHECK: br {{.*}} !prof !10 for(int i = 0; i != e; ++e) [[unlikely]]; } +// CHECK-LABEL: @_Z15f_branch_elidedv( +// CHECK-NEXT: entry: +// CHECK-NEXT: br label [[FOR_COND:%.*]] +// CHECK: for.cond: +// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP13:![0-9]+]] +// void f_branch_elided() { - // CHECK-LABEL: define{{.*}}f_branch_elided - // CHECK-NOT: br {{.*}} !prof for(;;) [[likely]]; } +// CHECK-LABEL: @_Z3frlOA4_i( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[E_ADDR:%.*]] = alloca [4 x i32]*, align 8 +// CHECK-NEXT: [[__RANGE1:%.*]] = alloca [4 x i32]*, align 8 +// CHECK-NEXT: [[__BEGIN1:%.*]] = alloca i32*, align 8 +// CHECK-NEXT: [[__END1:%.*]] = alloca i32*, align 8 +// CHECK-NEXT: [[I:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store [4 x i32]* [[E:%.*]], [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14:![0-9]+]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP0]]) #[[ATTR4]] +// CHECK-NEXT: [[TMP1:%.*]] = load [4 x i32]*, [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: store [4 x i32]* [[TMP1]], [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[TMP2:%.*]] = bitcast i32** [[__BEGIN1]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP2]]) #[[ATTR4]] +// CHECK-NEXT: [[TMP3:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP3]], i64 0, i64 0 +// CHECK-NEXT: store i32* [[ARRAYDECAY]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[TMP4:%.*]] = bitcast i32** [[__END1]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP4]]) #[[ATTR4]] +// CHECK-NEXT: [[TMP5:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP5]], i64 0, i64 0 +// CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAYDECAY1]], i64 4 +// CHECK-NEXT: store i32* [[ADD_PTR]], i32** [[__END1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: br label [[FOR_COND:%.*]] +// CHECK: for.cond: +// CHECK-NEXT: [[TMP6:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[TMP7:%.*]] = load i32*, i32** [[__END1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[CMP:%.*]] = icmp ne i32* [[TMP6]], [[TMP7]] +// CHECK-NEXT: [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 true) +// CHECK-NEXT: br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]] +// CHECK: for.cond.cleanup: +// CHECK-NEXT: [[TMP8:%.*]] = bitcast i32** [[__END1]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP8]]) #[[ATTR4]] +// CHECK-NEXT: [[TMP9:%.*]] = bitcast i32** [[__BEGIN1]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP9]]) #[[ATTR4]] +// CHECK-NEXT: [[TMP10:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP10]]) #[[ATTR4]] +// CHECK-NEXT: br label [[FOR_END:%.*]] +// CHECK: for.body: +// CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[I]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP11]]) #[[ATTR4]] +// CHECK-NEXT: [[TMP12:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: store i32 [[TMP13]], i32* [[I]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[TMP14:%.*]] = bitcast i32* [[I]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP14]]) #[[ATTR4]] +// CHECK-NEXT: br label [[FOR_INC:%.*]] +// CHECK: for.inc: +// CHECK-NEXT: [[TMP15:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[TMP15]], i32 1 +// CHECK-NEXT: store i32* [[INCDEC_PTR]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]] +// CHECK: for.end: +// CHECK-NEXT: ret void +// void frl(int (&&e) [4]) { - // CHECK-LABEL: define{{.*}}frl - // CHECK: br {{.*}} !prof !6 for(int i : e) [[likely]]; } +// CHECK-LABEL: @_Z3fruOA4_i( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[E_ADDR:%.*]] = alloca [4 x i32]*, align 8 +// CHECK-NEXT: [[__RANGE1:%.*]] = alloca [4 x i32]*, align 8 +// CHECK-NEXT: [[__BEGIN1:%.*]] = alloca i32*, align 8 +// CHECK-NEXT: [[__END1:%.*]] = alloca i32*, align 8 +// CHECK-NEXT: [[I:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store [4 x i32]* [[E:%.*]], [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP0]]) #[[ATTR4]] +// CHECK-NEXT: [[TMP1:%.*]] = load [4 x i32]*, [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: store [4 x i32]* [[TMP1]], [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[TMP2:%.*]] = bitcast i32** [[__BEGIN1]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP2]]) #[[ATTR4]] +// CHECK-NEXT: [[TMP3:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP3]], i64 0, i64 0 +// CHECK-NEXT: store i32* [[ARRAYDECAY]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[TMP4:%.*]] = bitcast i32** [[__END1]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP4]]) #[[ATTR4]] +// CHECK-NEXT: [[TMP5:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP5]], i64 0, i64 0 +// CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAYDECAY1]], i64 4 +// CHECK-NEXT: store i32* [[ADD_PTR]], i32** [[__END1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: br label [[FOR_COND:%.*]] +// CHECK: for.cond: +// CHECK-NEXT: [[TMP6:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[TMP7:%.*]] = load i32*, i32** [[__END1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[CMP:%.*]] = icmp ne i32* [[TMP6]], [[TMP7]] +// CHECK-NEXT: [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 false) +// CHECK-NEXT: br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]] +// CHECK: for.cond.cleanup: +// CHECK-NEXT: [[TMP8:%.*]] = bitcast i32** [[__END1]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP8]]) #[[ATTR4]] +// CHECK-NEXT: [[TMP9:%.*]] = bitcast i32** [[__BEGIN1]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP9]]) #[[ATTR4]] +// CHECK-NEXT: [[TMP10:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP10]]) #[[ATTR4]] +// CHECK-NEXT: br label [[FOR_END:%.*]] +// CHECK: for.body: +// CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[I]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP11]]) #[[ATTR4]] +// CHECK-NEXT: [[TMP12:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: store i32 [[TMP13]], i32* [[I]], align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[TMP14:%.*]] = bitcast i32* [[I]] to i8* +// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP14]]) #[[ATTR4]] +// CHECK-NEXT: br label [[FOR_INC:%.*]] +// CHECK: for.inc: +// CHECK-NEXT: [[TMP15:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[TMP15]], i32 1 +// CHECK-NEXT: store i32* [[INCDEC_PTR]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]] +// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]] +// CHECK: for.end: +// CHECK-NEXT: ret void +// void fru(int (&&e) [4]) { - // CHECK-LABEL: define{{.*}}fru - // CHECK: br {{.*}} !prof !10 for(int i : e) [[unlikely]]; } - -// CHECK: !6 = !{!"branch_weights", i32 2000, i32 1} -// CHECK: !10 = !{!"branch_weights", i32 1, i32 2000} diff --git a/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp b/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp index 5fb7a67a7d9e..05729eb5a789 100644 --- a/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp +++ b/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp @@ -1,59 +1,111 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu | FileCheck %s extern volatile int i; +// CHECK-LABEL: @_Z8OneCaseLv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2:![0-9]+]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [ +// CHECK-NEXT: i32 1, label [[SW_EPILOG]] +// CHECK-NEXT: ], !prof !6 +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void OneCaseL() { - // CHECK-LABEL: define{{.*}}OneCaseL - // CHECK: switch - // CHECK: {{.*}} !prof !6 switch (i) { [[likely]] case 1: break; } } +// CHECK-LABEL: @_Z8OneCaseUv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [ +// CHECK-NEXT: i32 1, label [[SW_BB:%.*]] +// CHECK-NEXT: ], !prof !7 +// CHECK: sw.bb: +// CHECK-NEXT: [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1 +// CHECK-NEXT: store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: br label [[SW_EPILOG]] +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void OneCaseU() { - // CHECK-LABEL: define{{.*}}OneCaseU - // CHECK: switch - // CHECK: {{.*}} !prof !7 switch (i) { [[unlikely]] case 1: ++i; break; } } +// CHECK-LABEL: @_Z10TwoCasesLNv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [ +// CHECK-NEXT: i32 1, label [[SW_EPILOG]] +// CHECK-NEXT: i32 2, label [[SW_EPILOG]] +// CHECK-NEXT: ], !prof !8 +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void TwoCasesLN() { - // CHECK-LABEL: define{{.*}}TwoCasesLN - // CHECK: switch - // CHECK: {{.*}} !prof !8 switch (i) { [[likely]] case 1: break; case 2: break; } } +// CHECK-LABEL: @_Z10TwoCasesUNv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [ +// CHECK-NEXT: i32 1, label [[SW_EPILOG]] +// CHECK-NEXT: i32 2, label [[SW_EPILOG]] +// CHECK-NEXT: ], !prof !9 +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void TwoCasesUN() { - // CHECK-LABEL: define{{.*}}TwoCasesUN - // CHECK: switch - // CHECK: {{.*}} !prof !9 switch (i) { [[unlikely]] case 1: break; case 2: break; } } +// CHECK-LABEL: @_Z10TwoCasesLUv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [ +// CHECK-NEXT: i32 1, label [[SW_EPILOG]] +// CHECK-NEXT: i32 2, label [[SW_EPILOG]] +// CHECK-NEXT: ], !prof !10 +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void TwoCasesLU() { - // CHECK-LABEL: define{{.*}}TwoCasesLU - // CHECK: switch - // CHECK: {{.*}} !prof !10 switch (i) { [[likely]] case 1: break; [[unlikely]] case 2: break; } } +// CHECK-LABEL: @_Z20CasesFallthroughNNLNv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [ +// CHECK-NEXT: i32 1, label [[SW_BB:%.*]] +// CHECK-NEXT: i32 2, label [[SW_BB]] +// CHECK-NEXT: i32 3, label [[SW_BB1:%.*]] +// CHECK-NEXT: i32 4, label [[SW_BB1]] +// CHECK-NEXT: ], !prof !11 +// CHECK: sw.bb: +// CHECK-NEXT: br label [[SW_BB1]] +// CHECK: sw.bb1: +// CHECK-NEXT: br label [[SW_EPILOG]] +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void CasesFallthroughNNLN() { - // CHECK-LABEL: define{{.*}}CasesFallthroughNNLN - // CHECK: switch - // CHECK: {{.*}} !prof !11 switch (i) { case 1: case 2: @@ -62,10 +114,23 @@ void CasesFallthroughNNLN() { } } +// CHECK-LABEL: @_Z20CasesFallthroughNNUNv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [ +// CHECK-NEXT: i32 1, label [[SW_BB:%.*]] +// CHECK-NEXT: i32 2, label [[SW_BB]] +// CHECK-NEXT: i32 3, label [[SW_BB1:%.*]] +// CHECK-NEXT: i32 4, label [[SW_BB1]] +// CHECK-NEXT: ], !prof !12 +// CHECK: sw.bb: +// CHECK-NEXT: br label [[SW_BB1]] +// CHECK: sw.bb1: +// CHECK-NEXT: br label [[SW_EPILOG]] +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void CasesFallthroughNNUN() { - // CHECK-LABEL: define{{.*}}CasesFallthroughNNUN - // CHECK: switch - // CHECK: {{.*}} !prof !12 switch (i) { case 1: case 2: @@ -74,10 +139,32 @@ void CasesFallthroughNNUN() { } } +// CHECK-LABEL: @_Z28CasesFallthroughRangeSmallLNv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [ +// CHECK-NEXT: i32 1, label [[SW_BB:%.*]] +// CHECK-NEXT: i32 2, label [[SW_BB]] +// CHECK-NEXT: i32 3, label [[SW_BB]] +// CHECK-NEXT: i32 4, label [[SW_BB]] +// CHECK-NEXT: i32 5, label [[SW_BB]] +// CHECK-NEXT: i32 102, label [[SW_BB1:%.*]] +// CHECK-NEXT: i32 103, label [[SW_BB2:%.*]] +// CHECK-NEXT: i32 104, label [[SW_BB2]] +// CHECK-NEXT: ], !prof !13 +// CHECK: sw.bb: +// CHECK-NEXT: [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1 +// CHECK-NEXT: store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: br label [[SW_BB1]] +// CHECK: sw.bb1: +// CHECK-NEXT: br label [[SW_BB2]] +// CHECK: sw.bb2: +// CHECK-NEXT: br label [[SW_EPILOG]] +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void CasesFallthroughRangeSmallLN() { - // CHECK-LABEL: define{{.*}}CasesFallthroughRangeSmallLN - // CHECK: switch - // CHECK: {{.*}} !prof !13 switch (i) { case 1 ... 5: ++i; case 102: @@ -86,10 +173,32 @@ void CasesFallthroughRangeSmallLN() { } } +// CHECK-LABEL: @_Z28CasesFallthroughRangeSmallUNv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [ +// CHECK-NEXT: i32 1, label [[SW_BB:%.*]] +// CHECK-NEXT: i32 2, label [[SW_BB]] +// CHECK-NEXT: i32 3, label [[SW_BB]] +// CHECK-NEXT: i32 4, label [[SW_BB]] +// CHECK-NEXT: i32 5, label [[SW_BB]] +// CHECK-NEXT: i32 102, label [[SW_BB1:%.*]] +// CHECK-NEXT: i32 103, label [[SW_BB2:%.*]] +// CHECK-NEXT: i32 104, label [[SW_BB2]] +// CHECK-NEXT: ], !prof !14 +// CHECK: sw.bb: +// CHECK-NEXT: [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1 +// CHECK-NEXT: store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: br label [[SW_BB1]] +// CHECK: sw.bb1: +// CHECK-NEXT: br label [[SW_BB2]] +// CHECK: sw.bb2: +// CHECK-NEXT: br label [[SW_EPILOG]] +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void CasesFallthroughRangeSmallUN() { - // CHECK-LABEL: define{{.*}}CasesFallthroughRangeSmallUN - // CHECK: switch - // CHECK: {{.*}} !prof !14 switch (i) { case 1 ... 5: ++i; case 102: @@ -98,12 +207,26 @@ void CasesFallthroughRangeSmallUN() { } } +// CHECK-LABEL: @_Z29CasesFallthroughRangeLargeLLNv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_CASERANGE:%.*]] [ +// CHECK-NEXT: i32 1003, label [[SW_BB1:%.*]] +// CHECK-NEXT: i32 104, label [[SW_BB1]] +// CHECK-NEXT: ], !prof !8 +// CHECK: sw.bb: +// CHECK-NEXT: br label [[SW_BB1]] +// CHECK: sw.bb1: +// CHECK-NEXT: br label [[SW_EPILOG:%.*]] +// CHECK: sw.caserange: +// CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], 0 +// CHECK-NEXT: [[INBOUNDS:%.*]] = icmp ule i32 [[TMP1]], 64 +// CHECK-NEXT: [[INBOUNDS_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[INBOUNDS]], i1 true) +// CHECK-NEXT: br i1 [[INBOUNDS_EXPVAL]], label [[SW_BB:%.*]], label [[SW_EPILOG]] +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void CasesFallthroughRangeLargeLLN() { - // CHECK-LABEL: define{{.*}}CasesFallthroughRangeLargeLLN - // CHECK: switch - // CHECK: {{.*}} !prof !8 - // CHECK: caserange - // CHECK: br{{.*}} !prof !15 switch (i) { [[likely]] case 0 ... 64: [[likely]] case 1003: @@ -111,12 +234,26 @@ void CasesFallthroughRangeLargeLLN() { } } +// CHECK-LABEL: @_Z29CasesFallthroughRangeLargeUUNv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_CASERANGE:%.*]] [ +// CHECK-NEXT: i32 1003, label [[SW_BB1:%.*]] +// CHECK-NEXT: i32 104, label [[SW_BB1]] +// CHECK-NEXT: ], !prof !9 +// CHECK: sw.bb: +// CHECK-NEXT: br label [[SW_BB1]] +// CHECK: sw.bb1: +// CHECK-NEXT: br label [[SW_EPILOG:%.*]] +// CHECK: sw.caserange: +// CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], 0 +// CHECK-NEXT: [[INBOUNDS:%.*]] = icmp ule i32 [[TMP1]], 64 +// CHECK-NEXT: [[INBOUNDS_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[INBOUNDS]], i1 false) +// CHECK-NEXT: br i1 [[INBOUNDS_EXPVAL]], label [[SW_BB:%.*]], label [[SW_EPILOG]] +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void CasesFallthroughRangeLargeUUN() { - // CHECK-LABEL: define{{.*}}CasesFallthroughRangeLargeUUN - // CHECK: switch - // CHECK: {{.*}} !prof !9 - // CHECK: caserange - // CHECK: br{{.*}} !prof !16 switch (i) { [[unlikely]] case 0 ... 64: [[unlikely]] case 1003: @@ -124,30 +261,55 @@ void CasesFallthroughRangeLargeUUN() { } } +// CHECK-LABEL: @_Z15OneCaseDefaultLv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [ +// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]] +// CHECK-NEXT: ], !prof !15 +// CHECK: sw.default: +// CHECK-NEXT: br label [[SW_EPILOG]] +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void OneCaseDefaultL() { - // CHECK-LABEL: define{{.*}}OneCaseDefaultL - // CHECK: switch - // CHECK: {{.*}} !prof !17 switch (i) { case 1: break; [[likely]] default: break; } } +// CHECK-LABEL: @_Z15OneCaseDefaultUv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [ +// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]] +// CHECK-NEXT: ], !prof !16 +// CHECK: sw.default: +// CHECK-NEXT: br label [[SW_EPILOG]] +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void OneCaseDefaultU() { - // CHECK-LABEL: define{{.*}}OneCaseDefaultU - // CHECK: switch - // CHECK: {{.*}} !prof !18 switch (i) { case 1: break; [[unlikely]] default: break; } } +// CHECK-LABEL: @_Z18TwoCasesDefaultLNLv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [ +// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]] +// CHECK-NEXT: i32 2, label [[SW_EPILOG]] +// CHECK-NEXT: ], !prof !17 +// CHECK: sw.default: +// CHECK-NEXT: br label [[SW_EPILOG]] +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void TwoCasesDefaultLNL() { - // CHECK-LABEL: define{{.*}}TwoCasesDefaultLNL - // CHECK: switch - // CHECK: {{.*}} !prof !19 switch (i) { [[likely]] case 1: break; case 2: break; @@ -155,10 +317,19 @@ void TwoCasesDefaultLNL() { } } +// CHECK-LABEL: @_Z18TwoCasesDefaultLNNv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [ +// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]] +// CHECK-NEXT: i32 2, label [[SW_EPILOG]] +// CHECK-NEXT: ], !prof !8 +// CHECK: sw.default: +// CHECK-NEXT: br label [[SW_EPILOG]] +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void TwoCasesDefaultLNN() { - // CHECK-LABEL: define{{.*}}TwoCasesDefaultLNN - // CHECK: switch - // CHECK: {{.*}} !prof !8 switch (i) { [[likely]] case 1: break; case 2: break; @@ -166,29 +337,22 @@ void TwoCasesDefaultLNN() { } } +// CHECK-LABEL: @_Z18TwoCasesDefaultLNUv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]] +// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [ +// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]] +// CHECK-NEXT: i32 2, label [[SW_EPILOG]] +// CHECK-NEXT: ], !prof !18 +// CHECK: sw.default: +// CHECK-NEXT: br label [[SW_EPILOG]] +// CHECK: sw.epilog: +// CHECK-NEXT: ret void +// void TwoCasesDefaultLNU() { - // CHECK-LABEL: define{{.*}}TwoCasesDefaultLNU - // CHECK: switch - // CHECK: {{.*}} !prof !20 switch (i) { [[likely]] case 1: break; case 2: break; [[unlikely]] default: break; } } - -// CHECK: !6 = !{!"branch_weights", i32 357913942, i32 715827883} -// CHECK: !7 = !{!"branch_weights", i32 536870912, i32 1} -// CHECK: !8 = !{!"branch_weights", i32 238609295, i32 715827883, i32 238609295} -// CHECK: !9 = !{!"branch_weights", i32 357913942, i32 1, i32 357913942} -// CHECK: !10 = !{!"branch_weights", i32 357913942, i32 715827883, i32 1} -// CHECK: !11 = !{!"branch_weights", i32 143165577, i32 143165577, i32 143165577, i32 715827883, i32 143165577} -// CHECK: !12 = !{!"branch_weights", i32 214748365, i32 214748365, i32 214748365, i32 1, i32 214748365} -// CHECK: !13 = !{!"branch_weights", i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 715827883, i32 79536432} -// CHECK: !14 = !{!"branch_weights", i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 1, i32 119304648} -// CHECK: !15 = !{!"branch_weights", i32 2000, i32 1} -// CHECK: !16 = !{!"branch_weights", i32 1, i32 2000} -// CHECK: !17 = !{!"branch_weights", i32 715827883, i32 357913942} -// CHECK: !18 = !{!"branch_weights", i32 1, i32 536870912} -// CHECK: !19 = !{!"branch_weights", i32 536870912, i32 536870912, i32 268435456} -// CHECK: !20 = !{!"branch_weights", i32 1, i32 715827883, i32 357913942} diff --git a/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h b/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h index 22b2e649e4d4..4e47ff70d557 100644 --- a/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h +++ b/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h @@ -17,7 +17,6 @@ #include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" -#include "llvm/Support/CommandLine.h" namespace llvm { @@ -32,8 +31,6 @@ struct LowerExpectIntrinsicPass : PassInfoMixin { PreservedAnalyses run(Function &F, FunctionAnalysisManager &); }; -extern cl::opt LikelyBranchWeight; -extern cl::opt UnlikelyBranchWeight; } #endif diff --git a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp index da13075dfee2..7f3549aaaabf 100644 --- a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp +++ b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp @@ -24,6 +24,7 @@ #include "llvm/IR/Metadata.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Transforms/Scalar.h" @@ -41,15 +42,15 @@ STATISTIC(ExpectIntrinsicsHandled, // only be used in extreme cases, we could make this ratio higher. As it stands, // programmers may be using __builtin_expect() / llvm.expect to annotate that a // branch is likely or unlikely to be taken. -// -// There is a known dependency on this ratio in CodeGenPrepare when transforming -// 'select' instructions. It may be worthwhile to hoist these values to some -// shared space, so they can be used directly by other passes. -cl::opt llvm::LikelyBranchWeight( +// WARNING: these values are internal implementation detail of the pass. +// They should not be exposed to the outside of the pass, front-end codegen +// should emit @llvm.expect intrinsics instead of using these weights directly. +// Transforms should use TargetLowering getPredictableBranchThreshold() hook. +static cl::opt LikelyBranchWeight( "likely-branch-weight", cl::Hidden, cl::init(2000), cl::desc("Weight of the branch likely to be taken (default = 2000)")); -cl::opt llvm::UnlikelyBranchWeight( +static cl::opt UnlikelyBranchWeight( "unlikely-branch-weight", cl::Hidden, cl::init(1), cl::desc("Weight of the branch unlikely to be taken (default = 1)"));