maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[LoopVectorizer] Require no-signed-zeros-fp-math=true for fmin/fmax 

Currently, setting the `no-nans-fp-math` attribute to true will allow
loops with fmin/fmax to vectorize, though we should be requiring that
`no-signed-zeros-fp-math` is also set.

This patch adds the check for no-signed-zeros at the function level and includes
tests to make sure we don't vectorize functions with only one of the attributes
associated.

Reviewed By: spatel

Differential Revision: https://reviews.llvm.org/D96604
This commit is contained in:
Kerry McLaughlin 2021-02-15 10:36:40 +00:00
parent 50c19b4c11
commit 5fe1593438
5 changed files with 103 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
llvm/include/llvm/Analysis/IVDescriptors.h
View File

@ -114,7 +114,7 @@ public:
/// compare instruction to the select instruction and stores this pointer in
/// 'PatternLastInst' member of the returned struct.
static InstDesc isRecurrenceInstr(Instruction *I, RecurKind Kind,
InstDesc &Prev, bool HasFunNoNaNAttr);
InstDesc &Prev, FastMathFlags FMF);
/// Returns true if instruction I has multiple uses in Insts
static bool hasMultipleUsesOf(Instruction *I,
@ -146,7 +146,7 @@ public:
/// non-null, the minimal bit width needed to compute the reduction will be
/// computed.
static bool AddReductionVar(PHINode *Phi, RecurKind Kind, Loop *TheLoop,
bool HasFunNoNaNAttr,
FastMathFlags FMF,
RecurrenceDescriptor &RedDes,
DemandedBits *DB = nullptr,
AssumptionCache *AC = nullptr,

61
llvm/lib/Analysis/IVDescriptors.cpp
View File

@ -190,7 +190,7 @@ static void collectCastsToIgnore(Loop *TheLoop, Instruction *Exit,
}
bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind,
Loop *TheLoop, bool HasFunNoNaNAttr,
Loop *TheLoop, FastMathFlags FuncFMF,
RecurrenceDescriptor &RedDes,
DemandedBits *DB,
AssumptionCache *AC,
@ -298,7 +298,7 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind,
// the starting value (the Phi or an AND instruction if the Phi has been
// type-promoted).
if (Cur != Start) {
ReduxDesc = isRecurrenceInstr(Cur, Kind, ReduxDesc, HasFunNoNaNAttr);
ReduxDesc = isRecurrenceInstr(Cur, Kind, ReduxDesc, FuncFMF);
if (!ReduxDesc.isRecurrence())
return false;
// FIXME: FMF is allowed on phi, but propagation is not handled correctly.
@ -561,7 +561,7 @@ RecurrenceDescriptor::isConditionalRdxPattern(RecurKind Kind, Instruction *I) {
RecurrenceDescriptor::InstDesc
RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurKind Kind,
InstDesc &Prev, bool HasFunNoNaNAttr) {
InstDesc &Prev, FastMathFlags FMF) {
Instruction *UAI = Prev.getUnsafeAlgebraInst();
if (!UAI && isa<FPMathOperator>(I) && !I->hasAllowReassoc())
UAI = I; // Found an unsafe (unvectorizable) algebra instruction.
@ -594,10 +594,11 @@ RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurKind Kind,
LLVM_FALLTHROUGH;
case Instruction::FCmp:
case Instruction::ICmp:
if (!isIntMinMaxRecurrenceKind(Kind) &&
(!HasFunNoNaNAttr || !isFPMinMaxRecurrenceKind(Kind)))
return InstDesc(false, I);
return isMinMaxSelectCmpPattern(I, Prev);
if (isIntMinMaxRecurrenceKind(Kind) ||
(FMF.noNaNs() && FMF.noSignedZeros() &&
isFPMinMaxRecurrenceKind(Kind)))
return isMinMaxSelectCmpPattern(I, Prev);
return InstDesc(false, I);
}
}
@ -622,71 +623,61 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
BasicBlock *Header = TheLoop->getHeader();
Function &F = *Header->getParent();
bool HasFunNoNaNAttr =
F.getFnAttribute("no-nans-fp-math").getValueAsString() == "true";
FastMathFlags FMF;
FMF.setNoNaNs(
F.getFnAttribute("no-nans-fp-math").getValueAsString() == "true");
FMF.setNoSignedZeros(
F.getFnAttribute("no-signed-zeros-fp-math").getValueAsString() == "true");
if (AddReductionVar(Phi, RecurKind::Add, TheLoop, HasFunNoNaNAttr, RedDes, DB,
AC, DT)) {
if (AddReductionVar(Phi, RecurKind::Add, TheLoop, FMF, RedDes, DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found an ADD reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::Mul, TheLoop, HasFunNoNaNAttr, RedDes, DB,
AC, DT)) {
if (AddReductionVar(Phi, RecurKind::Mul, TheLoop, FMF, RedDes, DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found a MUL reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::Or, TheLoop, HasFunNoNaNAttr, RedDes, DB,
AC, DT)) {
if (AddReductionVar(Phi, RecurKind::Or, TheLoop, FMF, RedDes, DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found an OR reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::And, TheLoop, HasFunNoNaNAttr, RedDes, DB,
AC, DT)) {
if (AddReductionVar(Phi, RecurKind::And, TheLoop, FMF, RedDes, DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found an AND reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::Xor, TheLoop, HasFunNoNaNAttr, RedDes, DB,
AC, DT)) {
if (AddReductionVar(Phi, RecurKind::Xor, TheLoop, FMF, RedDes, DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found a XOR reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::SMax, TheLoop, HasFunNoNaNAttr, RedDes,
DB, AC, DT)) {
if (AddReductionVar(Phi, RecurKind::SMax, TheLoop, FMF, RedDes, DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found a SMAX reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::SMin, TheLoop, HasFunNoNaNAttr, RedDes,
DB, AC, DT)) {
if (AddReductionVar(Phi, RecurKind::SMin, TheLoop, FMF, RedDes, DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found a SMIN reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::UMax, TheLoop, HasFunNoNaNAttr, RedDes,
DB, AC, DT)) {
if (AddReductionVar(Phi, RecurKind::UMax, TheLoop, FMF, RedDes, DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found a UMAX reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::UMin, TheLoop, HasFunNoNaNAttr, RedDes,
DB, AC, DT)) {
if (AddReductionVar(Phi, RecurKind::UMin, TheLoop, FMF, RedDes, DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found a UMIN reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::FMul, TheLoop, HasFunNoNaNAttr, RedDes,
DB, AC, DT)) {
if (AddReductionVar(Phi, RecurKind::FMul, TheLoop, FMF, RedDes, DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found an FMult reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::FAdd, TheLoop, HasFunNoNaNAttr, RedDes,
DB, AC, DT)) {
if (AddReductionVar(Phi, RecurKind::FAdd, TheLoop, FMF, RedDes, DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found an FAdd reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::FMax, TheLoop, HasFunNoNaNAttr, RedDes,
DB, AC, DT)) {
if (AddReductionVar(Phi, RecurKind::FMax, TheLoop, FMF, RedDes, DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found a float MAX reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::FMin, TheLoop, HasFunNoNaNAttr, RedDes,
DB, AC, DT)) {
if (AddReductionVar(Phi, RecurKind::FMin, TheLoop, FMF, RedDes, DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found a float MIN reduction PHI." << *Phi << "\n");
return true;
}

16
llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll
View File

@ -292,17 +292,17 @@ define float @PR35538(float* nocapture readonly %a, i32 %N) #0 {
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, float* [[TMP2]], i32 4
; CHECK-NEXT: [[TMP7:%.*]] = bitcast float* [[TMP6]] to <4 x float>*
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, <4 x float>* [[TMP7]], align 4
; CHECK-NEXT: [[TMP8:%.*]] = fcmp nnan ninf oge <4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
; CHECK-NEXT: [[TMP9:%.*]] = fcmp nnan ninf oge <4 x float> [[WIDE_LOAD2]], [[VEC_PHI1]]
; CHECK-NEXT: [[TMP8:%.*]] = fcmp nnan ninf nsz oge <4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
; CHECK-NEXT: [[TMP9:%.*]] = fcmp nnan ninf nsz oge <4 x float> [[WIDE_LOAD2]], [[VEC_PHI1]]
; CHECK-NEXT: [[TMP10]] = select <4 x i1> [[TMP8]], <4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]]
; CHECK-NEXT: [[TMP11]] = select <4 x i1> [[TMP9]], <4 x float> [[WIDE_LOAD2]], <4 x float> [[VEC_PHI1]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP8:!llvm.loop !.*]]
; CHECK: middle.block:
; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp nnan ninf ogt <4 x float> [[TMP10]], [[TMP11]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select nnan ninf <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP10]], <4 x float> [[TMP11]]
; CHECK-NEXT: [[TMP13:%.*]] = call nnan ninf float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp nnan ninf nsz ogt <4 x float> [[TMP10]], [[TMP11]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select nnan ninf nsz <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP10]], <4 x float> [[TMP11]]
; CHECK-NEXT: [[TMP13:%.*]] = call nnan ninf nsz float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
@ -320,7 +320,7 @@ define float @PR35538(float* nocapture readonly %a, i32 %N) #0 {
; CHECK-NEXT: [[MAX_013:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MAX_0_]], [[FOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP14:%.*]] = load float, float* [[ARRAYIDX]], align 4
; CHECK-NEXT: [[CMP1_INV:%.*]] = fcmp nnan ninf oge float [[TMP14]], [[MAX_013]]
; CHECK-NEXT: [[CMP1_INV:%.*]] = fcmp nnan ninf nsz oge float [[TMP14]], [[MAX_013]]
; CHECK-NEXT: [[MAX_0_]] = select i1 [[CMP1_INV]], float [[TMP14]], float [[MAX_013]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
@ -343,7 +343,7 @@ for.body:
%max.013 = phi float [ -1.000000e+00, %for.body.lr.ph ], [ %max.0., %for.body ]
%arrayidx = getelementptr inbounds float, float* %a, i64 %indvars.iv
%0 = load float, float* %arrayidx, align 4
%cmp1.inv = fcmp nnan ninf oge float %0, %max.013
%cmp1.inv = fcmp nnan ninf nsz oge float %0, %max.013
%max.0. = select i1 %cmp1.inv, float %0, float %max.013
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
%exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
@ -437,4 +437,4 @@ for.body:
br i1 %exitcond, label %for.cond.cleanup, label %for.body
}
attributes #0 = { "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="false" "unsafe-fp-math"="false" }
attributes #0 = { "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "unsafe-fp-math"="false" }

43
llvm/test/Transforms/LoopVectorize/float-minmax-instruction-flag.ll
View File

@ -148,6 +148,46 @@ out: ; preds = %loop
ret float %t6
}
; This test is checking that we don't vectorize when only one of the required attributes is set.
; Note that this test should not vectorize even after switching to IR-level FMF.
define float @minloopmissingnsz(float* nocapture readonly %arg) #1 {
; CHECK-LABEL: @minloopmissingnsz(
; CHECK-NEXT: top:
; CHECK-NEXT: [[T:%.*]] = load float, float* [[ARG:%.*]], align 4
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[T1:%.*]] = phi i64 [ [[T7:%.*]], [[LOOP]] ], [ 1, [[TOP:%.*]] ]
; CHECK-NEXT: [[T2:%.*]] = phi float [ [[T6:%.*]], [[LOOP]] ], [ [[T]], [[TOP]] ]
; CHECK-NEXT: [[T3:%.*]] = getelementptr float, float* [[ARG]], i64 [[T1]]
; CHECK-NEXT: [[T4:%.*]] = load float, float* [[T3]], align 4
; CHECK-NEXT: [[T5:%.*]] = fcmp olt float [[T2]], [[T4]]
; CHECK-NEXT: [[T6]] = select i1 [[T5]], float [[T2]], float [[T4]]
; CHECK-NEXT: [[T7]] = add i64 [[T1]], 1
; CHECK-NEXT: [[T8:%.*]] = icmp eq i64 [[T7]], 65537
; CHECK-NEXT: br i1 [[T8]], label [[OUT:%.*]], label [[LOOP]]
; CHECK: out:
; CHECK-NEXT: [[T6_LCSSA:%.*]] = phi float [ [[T6]], [[LOOP]] ]
; CHECK-NEXT: ret float [[T6_LCSSA]]
;
top:
%t = load float, float* %arg
br label %loop
loop: ; preds = %loop, %top
%t1 = phi i64 [ %t7, %loop ], [ 1, %top ]
%t2 = phi float [ %t6, %loop ], [ %t, %top ]
%t3 = getelementptr float, float* %arg, i64 %t1
%t4 = load float, float* %t3, align 4
%t5 = fcmp olt float %t2, %t4
%t6 = select i1 %t5, float %t2, float %t4
%t7 = add i64 %t1, 1
%t8 = icmp eq i64 %t7, 65537
br i1 %t8, label %out, label %loop
out: ; preds = %loop
ret float %t6
}
; This would assert on FMF propagation.
define void @not_a_min_max() {
@ -177,4 +217,5 @@ end:
ret void
}
attributes #0 = { "no-nans-fp-math"="true" }
attributes #0 = { "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" }
attributes #1 = { "no-nans-fp-math"="true" }

26
llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
View File

@ -852,7 +852,31 @@ for.end:
ret float %max.red.0
}
; As above, with the no-signed-zeros-fp-math attribute missing
; CHECK-LABEL: @max_red_float_nsz(
; CHECK-NOT: <2 x float>
define float @max_red_float_nsz(float %max) #1 {
entry:
br label %for.body
for.body:
%indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
%max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
%arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
%0 = load float, float* %arrayidx, align 4
%cmp3 = fcmp fast ogt float %0, %max.red.08
%max.red.0 = select i1 %cmp3, float %0, float %max.red.08
%indvars.iv.next = add i64 %indvars.iv, 1
%exitcond = icmp eq i64 %indvars.iv.next, 1024
br i1 %exitcond, label %for.end, label %for.body
for.end:
ret float %max.red.0
}
; Make sure any check-not directives are not triggered by function declarations.
; CHECK: declare
attributes #0 = { "no-nans-fp-math"="true" }
attributes #0 = { "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" }
attributes #1 = { "no-nans-fp-math"="true" }