maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[NFC][TTI] Add Alignment for isLegalMasked[Gather/Scatter] 

Add an extra parameter so alignment can be taken under
consideration in gather/scatter legalization.

Differential Revision: https://reviews.llvm.org/D71610
This commit is contained in:
Anna Welker 2019-12-18 08:42:53 +00:00
parent a6be12dbb1
commit 7cd1cfdd6b
8 changed files with 52 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
llvm/include/llvm/Analysis/TargetTransformInfo.h
View File

@ -597,9 +597,9 @@ public:
bool isLegalNTLoad(Type *DataType, Align Alignment) const;
/// Return true if the target supports masked scatter.
bool isLegalMaskedScatter(Type *DataType) const;
bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) const;
/// Return true if the target supports masked gather.
bool isLegalMaskedGather(Type *DataType) const;
bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) const;
/// Return true if the target supports masked compress store.
bool isLegalMaskedCompressStore(Type *DataType) const;
@ -1237,8 +1237,8 @@ public:
virtual bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) = 0;
virtual bool isLegalNTStore(Type *DataType, Align Alignment) = 0;
virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0;
virtual bool isLegalMaskedScatter(Type *DataType) = 0;
virtual bool isLegalMaskedGather(Type *DataType) = 0;
virtual bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) = 0;
virtual bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) = 0;
virtual bool isLegalMaskedCompressStore(Type *DataType) = 0;
virtual bool isLegalMaskedExpandLoad(Type *DataType) = 0;
virtual bool hasDivRemOp(Type *DataType, bool IsSigned) = 0;
@ -1537,11 +1537,11 @@ public:
bool isLegalNTLoad(Type *DataType, Align Alignment) override {
return Impl.isLegalNTLoad(DataType, Alignment);
}
bool isLegalMaskedScatter(Type *DataType) override {
return Impl.isLegalMaskedScatter(DataType);
bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) override {
return Impl.isLegalMaskedScatter(DataType, Alignment);
}
bool isLegalMaskedGather(Type *DataType) override {
return Impl.isLegalMaskedGather(DataType);
bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) override {
return Impl.isLegalMaskedGather(DataType, Alignment);
}
bool isLegalMaskedCompressStore(Type *DataType) override {
return Impl.isLegalMaskedCompressStore(DataType);

8
llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
View File

@ -272,9 +272,13 @@ public:
return Alignment >= DataSize && isPowerOf2_32(DataSize);
}
bool isLegalMaskedScatter(Type *DataType) { return false; }
bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) {
return false;
}
bool isLegalMaskedGather(Type *DataType) { return false; }
bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) {
return false;
}
bool isLegalMaskedCompressStore(Type *DataType) { return false; }

10
llvm/lib/Analysis/TargetTransformInfo.cpp
View File

@ -315,12 +315,14 @@ bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const {
return TTIImpl->isLegalNTLoad(DataType, Alignment);
}
bool TargetTransformInfo::isLegalMaskedGather(Type *DataType) const {
return TTIImpl->isLegalMaskedGather(DataType);
bool TargetTransformInfo::isLegalMaskedGather(Type *DataType,
MaybeAlign Alignment) const {
return TTIImpl->isLegalMaskedGather(DataType, Alignment);
}
bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType) const {
return TTIImpl->isLegalMaskedScatter(DataType);
bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType,
MaybeAlign Alignment) const {
return TTIImpl->isLegalMaskedScatter(DataType, Alignment);
}
bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType) const {

14
llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
View File

@ -849,21 +849,20 @@ bool ScalarizeMaskedMemIntrin::optimizeCallInst(CallInst *CI,
bool &ModifiedDT) {
IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI);
if (II) {
unsigned Alignment;
switch (II->getIntrinsicID()) {
default:
break;
case Intrinsic::masked_load: {
// Scalarize unsupported vector masked load
unsigned Alignment =
cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
Alignment = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
if (TTI->isLegalMaskedLoad(CI->getType(), MaybeAlign(Alignment)))
return false;
scalarizeMaskedLoad(CI, ModifiedDT);
return true;
}
case Intrinsic::masked_store: {
unsigned Alignment =
cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
Alignment = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
if (TTI->isLegalMaskedStore(CI->getArgOperand(0)->getType(),
MaybeAlign(Alignment)))
return false;
@ -871,12 +870,15 @@ bool ScalarizeMaskedMemIntrin::optimizeCallInst(CallInst *CI,
return true;
}
case Intrinsic::masked_gather:
if (TTI->isLegalMaskedGather(CI->getType()))
Alignment = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
if (TTI->isLegalMaskedGather(CI->getType(), MaybeAlign(Alignment)))
return false;
scalarizeMaskedGather(CI, ModifiedDT);
return true;
case Intrinsic::masked_scatter:
if (TTI->isLegalMaskedScatter(CI->getArgOperand(0)->getType()))
Alignment = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
if (TTI->isLegalMaskedScatter(CI->getArgOperand(0)->getType(),
MaybeAlign(Alignment)))
return false;
scalarizeMaskedScatter(CI, ModifiedDT);
return true;

4
llvm/lib/Target/ARM/ARMTargetTransformInfo.h
View File

@ -159,6 +159,10 @@ public:
return isLegalMaskedLoad(DataTy, Alignment);
}
bool isLegalMaskedGather(Type *Ty, MaybeAlign Alignment) { return false; }
bool isLegalMaskedScatter(Type *Ty, MaybeAlign Alignment) { return false; }
int getMemcpyCost(const Instruction *I);
int getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, Type *SubTp);

12
llvm/lib/Target/X86/X86TargetTransformInfo.cpp
View File

@ -3296,8 +3296,10 @@ int X86TTIImpl::getGatherScatterOpCost(unsigned Opcode, Type *SrcVTy,
unsigned AddressSpace = PtrTy->getAddressSpace();
bool Scalarize = false;
if ((Opcode == Instruction::Load && !isLegalMaskedGather(SrcVTy)) ||
(Opcode == Instruction::Store && !isLegalMaskedScatter(SrcVTy)))
if ((Opcode == Instruction::Load &&
!isLegalMaskedGather(SrcVTy, MaybeAlign(Alignment))) ||
(Opcode == Instruction::Store &&
!isLegalMaskedScatter(SrcVTy, MaybeAlign(Alignment))))
Scalarize = true;
// Gather / Scatter for vector 2 is not profitable on KNL / SKX
// Vector-4 of gather/scatter instruction does not exist on KNL.
@ -3420,7 +3422,7 @@ bool X86TTIImpl::isLegalMaskedCompressStore(Type *DataTy) {
return isLegalMaskedExpandLoad(DataTy);
}
bool X86TTIImpl::isLegalMaskedGather(Type *DataTy) {
bool X86TTIImpl::isLegalMaskedGather(Type *DataTy, MaybeAlign Alignment) {
// Some CPUs have better gather performance than others.
// TODO: Remove the explicit ST->hasAVX512()?, That would mean we would only
// enable gather with a -march.
@ -3458,11 +3460,11 @@ bool X86TTIImpl::isLegalMaskedGather(Type *DataTy) {
return IntWidth == 32 || IntWidth == 64;
}
bool X86TTIImpl::isLegalMaskedScatter(Type *DataType) {
bool X86TTIImpl::isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) {
// AVX2 doesn't support scatter
if (!ST->hasAVX512())
return false;
return isLegalMaskedGather(DataType);
return isLegalMaskedGather(DataType, Alignment);
}
bool X86TTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) {

4
llvm/lib/Target/X86/X86TargetTransformInfo.h
View File

@ -189,8 +189,8 @@ public:
bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment);
bool isLegalNTLoad(Type *DataType, Align Alignment);
bool isLegalNTStore(Type *DataType, Align Alignment);
bool isLegalMaskedGather(Type *DataType);
bool isLegalMaskedScatter(Type *DataType);
bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment);
bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment);
bool isLegalMaskedExpandLoad(Type *DataType);
bool isLegalMaskedCompressStore(Type *DataType);
bool hasDivRemOp(Type *DataType, bool IsSigned);

19
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -1210,14 +1210,14 @@ public:
/// Returns true if the target machine supports masked scatter operation
/// for the given \p DataType.
bool isLegalMaskedScatter(Type *DataType) {
return TTI.isLegalMaskedScatter(DataType);
bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) {
return TTI.isLegalMaskedScatter(DataType, Alignment);
}
/// Returns true if the target machine supports masked gather operation
/// for the given \p DataType.
bool isLegalMaskedGather(Type *DataType) {
return TTI.isLegalMaskedGather(DataType);
bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) {
return TTI.isLegalMaskedGather(DataType, Alignment);
}
/// Returns true if the target machine can represent \p V as a masked gather
@ -1228,7 +1228,9 @@ public:
if (!LI && !SI)
return false;
auto *Ty = getMemInstValueType(V);
return (LI && isLegalMaskedGather(Ty)) || (SI && isLegalMaskedScatter(Ty));
MaybeAlign Align = getLoadStoreAlignment(V);
return (LI && isLegalMaskedGather(Ty, Align)) ||
(SI && isLegalMaskedScatter(Ty, Align));
}
/// Returns true if \p I is an instruction that will be scalarized with
@ -4579,9 +4581,10 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I, unsigne
return WideningDecision == CM_Scalarize;
}
const MaybeAlign Alignment = getLoadStoreAlignment(I);
return isa<LoadInst>(I) ?
!(isLegalMaskedLoad(Ty, Ptr, Alignment) || isLegalMaskedGather(Ty))
: !(isLegalMaskedStore(Ty, Ptr, Alignment) || isLegalMaskedScatter(Ty));
return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment) ||
isLegalMaskedGather(Ty, Alignment))
: !(isLegalMaskedStore(Ty, Ptr, Alignment) ||
isLegalMaskedScatter(Ty, Alignment));
}
case Instruction::UDiv:
case Instruction::SDiv: