forked from OSchip/llvm-project
[LoopVectorize] Leverage speculation safety to avoid masked.loads
If we're vectorizing a load in a predicated block, check to see if the load can be speculated rather than predicated. This allows us to generate a normal vector load instead of a masked.load. To do so, we must prove that all bytes accessed on any iteration of the original loop are dereferenceable, and that all loads (across all iterations) are properly aligned. This is equivelent to proving that hoisting the load into the loop header in the original scalar loop is safe. Note: There are a couple of code motion todos in the code. My intention is to wait about a day - to be sure this sticks - and then perform the NFC motion without furthe review. Differential Revision: https://reviews.llvm.org/D66688 llvm-svn: 371452
This commit is contained in:
Philip Reames
parent
589273bebd
commit
7403569be7
|
|
@ -15,6 +15,8 @@
|
|||
//
|
||||
#include "llvm/Transforms/Vectorize/LoopVectorize.h"
|
||||
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
|
||||
#include "llvm/Analysis/Loads.h"
|
||||
#include "llvm/Analysis/ValueTracking.h"
|
||||
#include "llvm/Analysis/VectorUtils.h"
|
||||
#include "llvm/IR/IntrinsicInst.h"
|
||||
|
||||
|
|
@ -916,6 +918,72 @@ bool LoopVectorizationLegality::blockCanBePredicated(
|
|||
return true;
|
||||
}
|
||||
|
||||
/// Return true if we can prove that the given load would access only
|
||||
/// dereferenceable memory, and be properly aligned on every iteration.
|
||||
/// (i.e. does not require predication beyond that required by the the header
|
||||
/// itself) TODO: Move to Loads.h/cpp in a separate change
|
||||
static bool isDereferenceableAndAlignedInLoop(LoadInst *LI, Loop *L,
|
||||
ScalarEvolution &SE,
|
||||
DominatorTree &DT) {
|
||||
auto &DL = LI->getModule()->getDataLayout();
|
||||
Value *Ptr = LI->getPointerOperand();
|
||||
auto *AddRec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(Ptr));
|
||||
if (!AddRec || AddRec->getLoop() != L || !AddRec->isAffine())
|
||||
return false;
|
||||
auto* Step = dyn_cast<SCEVConstant>(AddRec->getStepRecurrence(SE));
|
||||
if (!Step)
|
||||
return false;
|
||||
APInt StepC = Step->getAPInt();
|
||||
APInt EltSize(DL.getIndexTypeSizeInBits(Ptr->getType()),
|
||||
DL.getTypeStoreSize(LI->getType()));
|
||||
// TODO: generalize to access patterns which have gaps
|
||||
// TODO: handle uniform addresses (if not already handled by LICM)
|
||||
if (StepC != EltSize)
|
||||
return false;
|
||||
|
||||
// TODO: If the symbolic trip count has a small bound (max count), we might
|
||||
// be able to prove safety.
|
||||
auto TC = SE.getSmallConstantTripCount(L);
|
||||
if (!TC)
|
||||
return false;
|
||||
|
||||
const APInt AccessSize = TC * EltSize;
|
||||
|
||||
auto *StartS = dyn_cast<SCEVUnknown>(AddRec->getStart());
|
||||
if (!StartS)
|
||||
return false;
|
||||
assert(SE.isLoopInvariant(StartS, L) && "implied by addrec definition");
|
||||
Value *Base = StartS->getValue();
|
||||
|
||||
Instruction *HeaderFirstNonPHI = L->getHeader()->getFirstNonPHI();
|
||||
|
||||
unsigned Align = LI->getAlignment();
|
||||
if (Align == 0)
|
||||
Align = DL.getABITypeAlignment(LI->getType());
|
||||
// For the moment, restrict ourselves to the case where the access size is a
|
||||
// multiple of the requested alignment and the base is aligned.
|
||||
// TODO: generalize if a case found which warrants
|
||||
if (EltSize.urem(Align) != 0)
|
||||
return false;
|
||||
return isDereferenceableAndAlignedPointer(Base, Align, AccessSize,
|
||||
DL, HeaderFirstNonPHI, &DT);
|
||||
}
|
||||
|
||||
/// Return true if speculation of the given load must be suppressed for
|
||||
/// correctness reasons. If not suppressed, dereferenceability and alignment
|
||||
/// must be proven.
|
||||
/// TODO: Move to ValueTracking.h/cpp in a separate change
|
||||
static bool mustSuppressSpeculation(const LoadInst &LI) {
|
||||
if (!LI.isUnordered())
|
||||
return true;
|
||||
const Function &F = *LI.getFunction();
|
||||
// Speculative load may create a race that did not exist in the source.
|
||||
return F.hasFnAttribute(Attribute::SanitizeThread) ||
|
||||
// Speculative load may load data from dirty regions.
|
||||
F.hasFnAttribute(Attribute::SanitizeAddress) ||
|
||||
F.hasFnAttribute(Attribute::SanitizeHWAddress);
|
||||
}
|
||||
|
||||
bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
|
||||
if (!EnableIfConversion) {
|
||||
reportVectorizationFailure("If-conversion is disabled",
|
||||
|
|
@ -936,12 +1004,25 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
|
|||
|
||||
// Collect safe addresses.
|
||||
for (BasicBlock *BB : TheLoop->blocks()) {
|
||||
if (blockNeedsPredication(BB))
|
||||
if (!blockNeedsPredication(BB)) {
|
||||
for (Instruction &I : *BB)
|
||||
if (auto *Ptr = getLoadStorePointerOperand(&I))
|
||||
SafePointes.insert(Ptr);
|
||||
continue;
|
||||
}
|
||||
|
||||
for (Instruction &I : *BB)
|
||||
if (auto *Ptr = getLoadStorePointerOperand(&I))
|
||||
SafePointes.insert(Ptr);
|
||||
// For a block which requires predication, a address may be safe to access
|
||||
// in the loop w/o predication if we can prove dereferenceability facts
|
||||
// sufficient to ensure it'll never fault within the loop. For the moment,
|
||||
// we restrict this to loads; stores are more complicated due to
|
||||
// concurrency restrictions.
|
||||
ScalarEvolution &SE = *PSE.getSE();
|
||||
for (Instruction &I : *BB) {
|
||||
LoadInst *LI = dyn_cast<LoadInst>(&I);
|
||||
if (LI && !mustSuppressSpeculation(*LI) &&
|
||||
isDereferenceableAndAlignedInLoop(LI, TheLoop, SE, *DT))
|
||||
SafePointes.insert(LI->getPointerOperand());
|
||||
}
|
||||
}
|
||||
|
||||
// Collect the blocks that need predication.
|
||||
|
|
|
|||
|
|
@ -67,24 +67,24 @@ define i32 @test_explicit_pred(i64 %len) {
|
|||
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP12]]
|
||||
; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[TMP20]], i32 0
|
||||
; CHECK-NEXT: [[TMP25:%.*]] = bitcast i32* [[TMP24]] to <4 x i32>*
|
||||
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP25]], i32 4, <4 x i1> [[TMP16]], <4 x i32> undef)
|
||||
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP25]], align 4
|
||||
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[TMP20]], i32 4
|
||||
; CHECK-NEXT: [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
|
||||
; CHECK-NEXT: [[WIDE_MASKED_LOAD13:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP27]], i32 4, <4 x i1> [[TMP17]], <4 x i32> undef)
|
||||
; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP27]], align 4
|
||||
; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[TMP20]], i32 8
|
||||
; CHECK-NEXT: [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
|
||||
; CHECK-NEXT: [[WIDE_MASKED_LOAD14:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP29]], i32 4, <4 x i1> [[TMP18]], <4 x i32> undef)
|
||||
; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
|
||||
; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TMP20]], i32 12
|
||||
; CHECK-NEXT: [[TMP31:%.*]] = bitcast i32* [[TMP30]] to <4 x i32>*
|
||||
; CHECK-NEXT: [[WIDE_MASKED_LOAD15:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP31]], i32 4, <4 x i1> [[TMP19]], <4 x i32> undef)
|
||||
; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP31]], align 4
|
||||
; CHECK-NEXT: [[TMP32:%.*]] = xor <4 x i1> [[TMP16]], <i1 true, i1 true, i1 true, i1 true>
|
||||
; CHECK-NEXT: [[TMP33:%.*]] = xor <4 x i1> [[TMP17]], <i1 true, i1 true, i1 true, i1 true>
|
||||
; CHECK-NEXT: [[TMP34:%.*]] = xor <4 x i1> [[TMP18]], <i1 true, i1 true, i1 true, i1 true>
|
||||
; CHECK-NEXT: [[TMP35:%.*]] = xor <4 x i1> [[TMP19]], <i1 true, i1 true, i1 true, i1 true>
|
||||
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[PREDPHI16:%.*]] = select <4 x i1> [[TMP17]], <4 x i32> [[WIDE_MASKED_LOAD13]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[PREDPHI17:%.*]] = select <4 x i1> [[TMP18]], <4 x i32> [[WIDE_MASKED_LOAD14]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[PREDPHI18:%.*]] = select <4 x i1> [[TMP19]], <4 x i32> [[WIDE_MASKED_LOAD15]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i32> [[WIDE_LOAD]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[PREDPHI16:%.*]] = select <4 x i1> [[TMP17]], <4 x i32> [[WIDE_LOAD13]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[PREDPHI17:%.*]] = select <4 x i1> [[TMP18]], <4 x i32> [[WIDE_LOAD14]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[PREDPHI18:%.*]] = select <4 x i1> [[TMP19]], <4 x i32> [[WIDE_LOAD15]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[TMP36]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
|
||||
; CHECK-NEXT: [[TMP37]] = add <4 x i32> [[VEC_PHI4]], [[PREDPHI16]]
|
||||
; CHECK-NEXT: [[TMP38]] = add <4 x i32> [[VEC_PHI5]], [[PREDPHI17]]
|
||||
|
|
@ -244,24 +244,24 @@ define i32 @test_explicit_pred_generic(i64 %len, i1* %test_base) {
|
|||
; CHECK-NEXT: [[TMP67:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP12]]
|
||||
; CHECK-NEXT: [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 0
|
||||
; CHECK-NEXT: [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
|
||||
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP69]], i32 4, <4 x i1> [[TMP39]], <4 x i32> undef)
|
||||
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
|
||||
; CHECK-NEXT: [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 4
|
||||
; CHECK-NEXT: [[TMP71:%.*]] = bitcast i32* [[TMP70]] to <4 x i32>*
|
||||
; CHECK-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP71]], i32 4, <4 x i1> [[TMP47]], <4 x i32> undef)
|
||||
; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP71]], align 4
|
||||
; CHECK-NEXT: [[TMP72:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 8
|
||||
; CHECK-NEXT: [[TMP73:%.*]] = bitcast i32* [[TMP72]] to <4 x i32>*
|
||||
; CHECK-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP73]], i32 4, <4 x i1> [[TMP55]], <4 x i32> undef)
|
||||
; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP73]], align 4
|
||||
; CHECK-NEXT: [[TMP74:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 12
|
||||
; CHECK-NEXT: [[TMP75:%.*]] = bitcast i32* [[TMP74]] to <4 x i32>*
|
||||
; CHECK-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP75]], i32 4, <4 x i1> [[TMP63]], <4 x i32> undef)
|
||||
; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP75]], align 4
|
||||
; CHECK-NEXT: [[TMP76:%.*]] = xor <4 x i1> [[TMP39]], <i1 true, i1 true, i1 true, i1 true>
|
||||
; CHECK-NEXT: [[TMP77:%.*]] = xor <4 x i1> [[TMP47]], <i1 true, i1 true, i1 true, i1 true>
|
||||
; CHECK-NEXT: [[TMP78:%.*]] = xor <4 x i1> [[TMP55]], <i1 true, i1 true, i1 true, i1 true>
|
||||
; CHECK-NEXT: [[TMP79:%.*]] = xor <4 x i1> [[TMP63]], <i1 true, i1 true, i1 true, i1 true>
|
||||
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[PREDPHI10:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[WIDE_MASKED_LOAD7]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[PREDPHI11:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[WIDE_MASKED_LOAD8]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[PREDPHI12:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[WIDE_MASKED_LOAD9]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[WIDE_LOAD]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[PREDPHI10:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[WIDE_LOAD7]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[PREDPHI11:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[WIDE_LOAD8]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[PREDPHI12:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[WIDE_LOAD9]], <4 x i32> zeroinitializer
|
||||
; CHECK-NEXT: [[TMP80]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
|
||||
; CHECK-NEXT: [[TMP81]] = add <4 x i32> [[VEC_PHI4]], [[PREDPHI10]]
|
||||
; CHECK-NEXT: [[TMP82]] = add <4 x i32> [[VEC_PHI5]], [[PREDPHI11]]
|
||||
|
|
|
|||
|
|
@ -42,12 +42,12 @@ for.end:
|
|||
; CHECK: load <2 x float>
|
||||
; CHECK-NOT: load <2 x float>
|
||||
|
||||
define void @dont_hoist_cond_load() {
|
||||
define void @dont_hoist_cond_load([1024 x float]* %a) {
|
||||
entry:
|
||||
br label %for.body
|
||||
for.body:
|
||||
%indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %if.end9 ]
|
||||
%arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @A, i64 0, i64 %indvars.iv
|
||||
%arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* %a, i64 0, i64 %indvars.iv
|
||||
%arrayidx2 = getelementptr inbounds [1024 x float], [1024 x float]* @B, i64 0, i64 %indvars.iv
|
||||
%0 = load float, float* %arrayidx2, align 4
|
||||
%cmp3 = fcmp oeq float %0, 0.000000e+00
|
||||
|
|
|
|||
Loading…
Reference in New Issue