maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[NFC][TTI] Add Alignment for isLegalMasked[Load/Store] 

Add an extra parameter so the backend can take the alignment into
consideration.

Differential Revision: https://reviews.llvm.org/D68400

llvm-svn: 374763
This commit is contained in:
Sam Parker 2019-10-14 10:00:21 +00:00
parent 2a3f527cf8
commit 527a35e155
9 changed files with 53 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
llvm/include/llvm/Analysis/TargetTransformInfo.h
View File

@ -574,10 +574,10 @@ public:
/// modes that operate across loop iterations. /// modes that operate across loop iterations.
bool shouldFavorBackedgeIndex(const Loop *L) const; bool shouldFavorBackedgeIndex(const Loop *L) const;
/// Return true if the target supports masked load.
bool isLegalMaskedStore(Type *DataType) const;
/// Return true if the target supports masked store. /// Return true if the target supports masked store.
bool isLegalMaskedLoad(Type *DataType) const; bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment) const;
/// Return true if the target supports masked load.
bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) const;
/// Return true if the target supports nontemporal store. /// Return true if the target supports nontemporal store.
bool isLegalNTStore(Type *DataType, Align Alignment) const; bool isLegalNTStore(Type *DataType, Align Alignment) const;
@ -1209,8 +1209,8 @@ public:
TargetLibraryInfo *LibInfo) = 0; TargetLibraryInfo *LibInfo) = 0;
virtual bool shouldFavorPostInc() const = 0; virtual bool shouldFavorPostInc() const = 0;
virtual bool shouldFavorBackedgeIndex(const Loop *L) const = 0; virtual bool shouldFavorBackedgeIndex(const Loop *L) const = 0;
virtual bool isLegalMaskedStore(Type *DataType) = 0; virtual bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment) = 0;
virtual bool isLegalMaskedLoad(Type *DataType) = 0; virtual bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) = 0;
virtual bool isLegalNTStore(Type *DataType, Align Alignment) = 0; virtual bool isLegalNTStore(Type *DataType, Align Alignment) = 0;
virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0; virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0;
virtual bool isLegalMaskedScatter(Type *DataType) = 0; virtual bool isLegalMaskedScatter(Type *DataType) = 0;
@ -1496,11 +1496,11 @@ public:
bool shouldFavorBackedgeIndex(const Loop *L) const override { bool shouldFavorBackedgeIndex(const Loop *L) const override {
return Impl.shouldFavorBackedgeIndex(L); return Impl.shouldFavorBackedgeIndex(L);
} }
bool isLegalMaskedStore(Type *DataType) override { bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment) override {
return Impl.isLegalMaskedStore(DataType); return Impl.isLegalMaskedStore(DataType, Alignment);
} }
bool isLegalMaskedLoad(Type *DataType) override { bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) override {
return Impl.isLegalMaskedLoad(DataType); return Impl.isLegalMaskedLoad(DataType, Alignment);
} }
bool isLegalNTStore(Type *DataType, Align Alignment) override { bool isLegalNTStore(Type *DataType, Align Alignment) override {
return Impl.isLegalNTStore(DataType, Alignment); return Impl.isLegalNTStore(DataType, Alignment);

4
llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
View File

@ -243,9 +243,9 @@ public:
bool shouldFavorBackedgeIndex(const Loop *L) const { return false; } bool shouldFavorBackedgeIndex(const Loop *L) const { return false; }
bool isLegalMaskedStore(Type *DataType) { return false; } bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment) { return false; }
bool isLegalMaskedLoad(Type *DataType) { return false; } bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) { return false; }
bool isLegalNTStore(Type *DataType, Align Alignment) { bool isLegalNTStore(Type *DataType, Align Alignment) {
// By default, assume nontemporal memory stores are available for stores // By default, assume nontemporal memory stores are available for stores

10
llvm/lib/Analysis/TargetTransformInfo.cpp
View File

@ -288,12 +288,14 @@ bool TargetTransformInfo::shouldFavorBackedgeIndex(const Loop *L) const {
return TTIImpl->shouldFavorBackedgeIndex(L); return TTIImpl->shouldFavorBackedgeIndex(L);
} }
bool TargetTransformInfo::isLegalMaskedStore(Type *DataType) const { bool TargetTransformInfo::isLegalMaskedStore(Type *DataType,
return TTIImpl->isLegalMaskedStore(DataType); MaybeAlign Alignment) const {
return TTIImpl->isLegalMaskedStore(DataType, Alignment);
} }
bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType) const { bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType,
return TTIImpl->isLegalMaskedLoad(DataType); MaybeAlign Alignment) const {
return TTIImpl->isLegalMaskedLoad(DataType, Alignment);
} }
bool TargetTransformInfo::isLegalNTStore(Type *DataType, bool TargetTransformInfo::isLegalNTStore(Type *DataType,

15
llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
View File

@ -851,17 +851,24 @@ bool ScalarizeMaskedMemIntrin::optimizeCallInst(CallInst *CI,
switch (II->getIntrinsicID()) { switch (II->getIntrinsicID()) {
default: default:
break; break;
case Intrinsic::masked_load: case Intrinsic::masked_load: {
// Scalarize unsupported vector masked load // Scalarize unsupported vector masked load
if (TTI->isLegalMaskedLoad(CI->getType())) unsigned Alignment =
cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
if (TTI->isLegalMaskedLoad(CI->getType(), MaybeAlign(Alignment)))
return false; return false;
scalarizeMaskedLoad(CI, ModifiedDT); scalarizeMaskedLoad(CI, ModifiedDT);
return true; return true;
case Intrinsic::masked_store: }
if (TTI->isLegalMaskedStore(CI->getArgOperand(0)->getType())) case Intrinsic::masked_store: {
unsigned Alignment =
cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
if (TTI->isLegalMaskedStore(CI->getArgOperand(0)->getType(),
MaybeAlign(Alignment)))
return false; return false;
scalarizeMaskedStore(CI, ModifiedDT); scalarizeMaskedStore(CI, ModifiedDT);
return true; return true;
}
case Intrinsic::masked_gather: case Intrinsic::masked_gather:
if (TTI->isLegalMaskedGather(CI->getType())) if (TTI->isLegalMaskedGather(CI->getType()))
return false; return false;

2
llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
View File

@ -491,7 +491,7 @@ int ARMTTIImpl::getAddressComputationCost(Type *Ty, ScalarEvolution *SE,
return BaseT::getAddressComputationCost(Ty, SE, Ptr); return BaseT::getAddressComputationCost(Ty, SE, Ptr);
} }
bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy) { bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, MaybeAlign Alignment) {
if (!EnableMaskedLoadStores || !ST->hasMVEIntegerOps()) if (!EnableMaskedLoadStores || !ST->hasMVEIntegerOps())
return false; return false;

6
llvm/lib/Target/ARM/ARMTargetTransformInfo.h
View File

@ -153,8 +153,10 @@ public:
return ST->getMaxInterleaveFactor(); return ST->getMaxInterleaveFactor();
} }
bool isLegalMaskedLoad(Type *DataTy); bool isLegalMaskedLoad(Type *DataTy, MaybeAlign Alignment);
bool isLegalMaskedStore(Type *DataTy) { return isLegalMaskedLoad(DataTy); } bool isLegalMaskedStore(Type *DataTy, MaybeAlign Alignment) {
return isLegalMaskedLoad(DataTy, Alignment);
}
int getMemcpyCost(const Instruction *I); int getMemcpyCost(const Instruction *I);

11
llvm/lib/Target/X86/X86TargetTransformInfo.cpp
View File

@ -2417,8 +2417,9 @@ int X86TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *SrcTy,
unsigned NumElem = SrcVTy->getVectorNumElements(); unsigned NumElem = SrcVTy->getVectorNumElements();
VectorType *MaskTy = VectorType *MaskTy =
VectorType::get(Type::getInt8Ty(SrcVTy->getContext()), NumElem); VectorType::get(Type::getInt8Ty(SrcVTy->getContext()), NumElem);
if ((IsLoad && !isLegalMaskedLoad(SrcVTy)) || if ((IsLoad && !isLegalMaskedLoad(SrcVTy, MaybeAlign(Alignment))) ||
(IsStore && !isLegalMaskedStore(SrcVTy)) || !isPowerOf2_32(NumElem)) { (IsStore && !isLegalMaskedStore(SrcVTy, MaybeAlign(Alignment))) ||
!isPowerOf2_32(NumElem)) {
// Scalarization // Scalarization
int MaskSplitCost = getScalarizationOverhead(MaskTy, false, true); int MaskSplitCost = getScalarizationOverhead(MaskTy, false, true);
int ScalarCompareCost = getCmpSelInstrCost( int ScalarCompareCost = getCmpSelInstrCost(
@ -3213,7 +3214,7 @@ bool X86TTIImpl::canMacroFuseCmp() {
return ST->hasMacroFusion() || ST->hasBranchFusion(); return ST->hasMacroFusion() || ST->hasBranchFusion();
} }
bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy) { bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, MaybeAlign Alignment) {
if (!ST->hasAVX()) if (!ST->hasAVX())
return false; return false;
@ -3236,8 +3237,8 @@ bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy) {
((IntWidth == 8 || IntWidth == 16) && ST->hasBWI()); ((IntWidth == 8 || IntWidth == 16) && ST->hasBWI());
} }
bool X86TTIImpl::isLegalMaskedStore(Type *DataType) { bool X86TTIImpl::isLegalMaskedStore(Type *DataType, MaybeAlign Alignment) {
return isLegalMaskedLoad(DataType); return isLegalMaskedLoad(DataType, Alignment);
} }
bool X86TTIImpl::isLegalNTLoad(Type *DataType, Align Alignment) { bool X86TTIImpl::isLegalNTLoad(Type *DataType, Align Alignment) {

4
llvm/lib/Target/X86/X86TargetTransformInfo.h
View File

@ -185,8 +185,8 @@ public:
bool isLSRCostLess(TargetTransformInfo::LSRCost &C1, bool isLSRCostLess(TargetTransformInfo::LSRCost &C1,
TargetTransformInfo::LSRCost &C2); TargetTransformInfo::LSRCost &C2);
bool canMacroFuseCmp(); bool canMacroFuseCmp();
bool isLegalMaskedLoad(Type *DataType); bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment);
bool isLegalMaskedStore(Type *DataType); bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment);
bool isLegalNTLoad(Type *DataType, Align Alignment); bool isLegalNTLoad(Type *DataType, Align Alignment);
bool isLegalNTStore(Type *DataType, Align Alignment); bool isLegalNTStore(Type *DataType, Align Alignment);
bool isLegalMaskedGather(Type *DataType); bool isLegalMaskedGather(Type *DataType);

20
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -1190,14 +1190,16 @@ public:
/// Returns true if the target machine supports masked store operation /// Returns true if the target machine supports masked store operation
/// for the given \p DataType and kind of access to \p Ptr. /// for the given \p DataType and kind of access to \p Ptr.
bool isLegalMaskedStore(Type *DataType, Value *Ptr) { bool isLegalMaskedStore(Type *DataType, Value *Ptr, unsigned Alignment) {
return Legal->isConsecutivePtr(Ptr) && TTI.isLegalMaskedStore(DataType); return Legal->isConsecutivePtr(Ptr) &&
TTI.isLegalMaskedStore(DataType, MaybeAlign(Alignment));
} }
/// Returns true if the target machine supports masked load operation /// Returns true if the target machine supports masked load operation
/// for the given \p DataType and kind of access to \p Ptr. /// for the given \p DataType and kind of access to \p Ptr.
bool isLegalMaskedLoad(Type *DataType, Value *Ptr) { bool isLegalMaskedLoad(Type *DataType, Value *Ptr, unsigned Alignment) {
return Legal->isConsecutivePtr(Ptr) && TTI.isLegalMaskedLoad(DataType); return Legal->isConsecutivePtr(Ptr) &&
TTI.isLegalMaskedLoad(DataType, MaybeAlign(Alignment));
} }
/// Returns true if the target machine supports masked scatter operation /// Returns true if the target machine supports masked scatter operation
@ -4551,6 +4553,7 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I, unsigne
return false; return false;
auto *Ptr = getLoadStorePointerOperand(I); auto *Ptr = getLoadStorePointerOperand(I);
auto *Ty = getMemInstValueType(I); auto *Ty = getMemInstValueType(I);
unsigned Alignment = getLoadStoreAlignment(I);
// We have already decided how to vectorize this instruction, get that // We have already decided how to vectorize this instruction, get that
// result. // result.
if (VF > 1) { if (VF > 1) {
@ -4560,8 +4563,8 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I, unsigne
return WideningDecision == CM_Scalarize; return WideningDecision == CM_Scalarize;
} }
return isa<LoadInst>(I) ? return isa<LoadInst>(I) ?
!(isLegalMaskedLoad(Ty, Ptr) || isLegalMaskedGather(Ty)) !(isLegalMaskedLoad(Ty, Ptr, Alignment) || isLegalMaskedGather(Ty))
: !(isLegalMaskedStore(Ty, Ptr) || isLegalMaskedScatter(Ty)); : !(isLegalMaskedStore(Ty, Ptr, Alignment) || isLegalMaskedScatter(Ty));
} }
case Instruction::UDiv: case Instruction::UDiv:
case Instruction::SDiv: case Instruction::SDiv:
@ -4604,8 +4607,9 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(Instruction *I,
"Masked interleave-groups for predicated accesses are not enabled."); "Masked interleave-groups for predicated accesses are not enabled.");
auto *Ty = getMemInstValueType(I); auto *Ty = getMemInstValueType(I);
return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty) unsigned Alignment = getLoadStoreAlignment(I);
: TTI.isLegalMaskedStore(Ty); return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, MaybeAlign(Alignment))
: TTI.isLegalMaskedStore(Ty, MaybeAlign(Alignment));
} }
bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I, bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,