maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[IRBuilder] Add type argument to CreateMaskedLoad/Gather 

Same as other CreateLoad-style APIs, these need an explicit type
argument to support opaque pointers.

Differential Revision: https://reviews.llvm.org/D105395
This commit is contained in:
Nikita Popov 2021-07-03 14:57:41 +02:00
parent 95923c0ba2
commit fabc17192e
10 changed files with 46 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
clang/lib/CodeGen/CGBuiltin.cpp
View File

@ -11746,14 +11746,14 @@ static Value *EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
static Value *EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
Align Alignment) {
// Cast the pointer to right type.
Value *Ptr = CGF.Builder.CreateBitCast(Ops[0],
llvm::PointerType::getUnqual(Ops[1]->getType()));
llvm::Type *Ty = Ops[1]->getType();
Value *Ptr =
CGF.Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
Value *MaskVec = getMaskVecValue(
CGF, Ops[2],
cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements());
CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements());
return CGF.Builder.CreateMaskedLoad(Ptr, Alignment, MaskVec, Ops[1]);
return CGF.Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]);
}
static Value *EmitX86ExpandLoad(CodeGenFunction &CGF,

4
llvm/include/llvm/IR/IRBuilder.h
View File

@ -752,7 +752,7 @@ public:
CallInst *CreateInvariantStart(Value *Ptr, ConstantInt *Size = nullptr);
/// Create a call to Masked Load intrinsic
CallInst *CreateMaskedLoad(Value *Ptr, Align Alignment, Value *Mask,
CallInst *CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask,
Value *PassThru = nullptr, const Twine &Name = "");
/// Create a call to Masked Store intrinsic
@ -760,7 +760,7 @@ public:
Value *Mask);
/// Create a call to Masked Gather intrinsic
CallInst *CreateMaskedGather(Value *Ptrs, Align Alignment,
CallInst *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
Value *Mask = nullptr, Value *PassThru = nullptr,
const Twine &Name = "");

5
llvm/lib/IR/AutoUpgrade.cpp
View File

@ -1421,10 +1421,9 @@ static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,
return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
// Convert the mask from an integer type to a vector of i1.
unsigned NumElts =
cast<FixedVectorType>(Passthru->getType())->getNumElements();
unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
Mask = getX86MaskVec(Builder, Mask, NumElts);
return Builder.CreateMaskedLoad(Ptr, Alignment, Mask, Passthru);
return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
}
static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) {

31
llvm/lib/IR/IRBuilder.cpp
View File

@ -493,6 +493,7 @@ Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
}
/// Create a call to a Masked Load intrinsic.
/// \p Ty - vector type to load
/// \p Ptr - base pointer for the load
/// \p Alignment - alignment of the source location
/// \p Mask - vector of booleans which indicates what vector lanes should
@ -500,16 +501,16 @@ Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
/// \p PassThru - pass-through value that is used to fill the masked-off lanes
/// of the result
/// \p Name - name of the result variable
CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, Align Alignment,
CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
Value *Mask, Value *PassThru,
const Twine &Name) {
auto *PtrTy = cast<PointerType>(Ptr->getType());
Type *DataTy = PtrTy->getElementType();
assert(DataTy->isVectorTy() && "Ptr should point to a vector");
assert(Ty->isVectorTy() && "Type should be vector");
assert(PtrTy->isOpaqueOrPointeeTypeMatches(Ty) && "Wrong element type");
assert(Mask && "Mask should not be all-ones (null)");
if (!PassThru)
PassThru = UndefValue::get(DataTy);
Type *OverloadedTypes[] = { DataTy, PtrTy };
PassThru = UndefValue::get(Ty);
Type *OverloadedTypes[] = { Ty, PtrTy };
Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
OverloadedTypes, Name);
@ -546,6 +547,7 @@ CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
}
/// Create a call to a Masked Gather intrinsic.
/// \p Ty - vector type to gather
/// \p Ptrs - vector of pointers for loading
/// \p Align - alignment for one element
/// \p Mask - vector of booleans which indicates what vector lanes should
@ -553,22 +555,27 @@ CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
/// \p PassThru - pass-through value that is used to fill the masked-off lanes
/// of the result
/// \p Name - name of the result variable
CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, Align Alignment,
Value *Mask, Value *PassThru,
CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
Align Alignment, Value *Mask,
Value *PassThru,
const Twine &Name) {
auto *VecTy = cast<VectorType>(Ty);
ElementCount NumElts = VecTy->getElementCount();
auto *PtrsTy = cast<VectorType>(Ptrs->getType());
auto *PtrTy = cast<PointerType>(PtrsTy->getElementType());
ElementCount NumElts = PtrsTy->getElementCount();
auto *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
assert(cast<PointerType>(PtrsTy->getElementType())
->isOpaqueOrPointeeTypeMatches(
cast<VectorType>(Ty)->getElementType()) &&
"Element type mismatch");
assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");
if (!Mask)
Mask = Constant::getAllOnesValue(
VectorType::get(Type::getInt1Ty(Context), NumElts));
if (!PassThru)
PassThru = UndefValue::get(DataTy);
PassThru = UndefValue::get(Ty);
Type *OverloadedTypes[] = {DataTy, PtrsTy};
Type *OverloadedTypes[] = {Ty, PtrsTy};
Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
// We specify only one type when we create this intrinsic. Types of other

2
llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
View File

@ -475,7 +475,7 @@ auto AlignVectors::createAlignedLoad(IRBuilder<> &Builder, Type *ValTy,
return PassThru;
if (Mask == ConstantInt::getTrue(Mask->getType()))
return Builder.CreateAlignedLoad(ValTy, Ptr, Align(Alignment));
return Builder.CreateMaskedLoad(Ptr, Align(Alignment), Mask, PassThru);
return Builder.CreateMaskedLoad(ValTy, Ptr, Align(Alignment), Mask, PassThru);
}
auto AlignVectors::createAlignedStore(IRBuilder<> &Builder, Value *Val,

4
llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
View File

@ -72,8 +72,8 @@ static Instruction *simplifyX86MaskedLoad(IntrinsicInst &II, InstCombiner &IC) {
Value *PtrCast = IC.Builder.CreateBitCast(Ptr, VecPtrTy, "castvec");
// The pass-through vector for an x86 masked load is a zero vector.
CallInst *NewMaskedLoad =
IC.Builder.CreateMaskedLoad(PtrCast, Align(1), BoolMask, ZeroVec);
CallInst *NewMaskedLoad = IC.Builder.CreateMaskedLoad(
II.getType(), PtrCast, Align(1), BoolMask, ZeroVec);
return IC.replaceInstUsesWith(II, NewMaskedLoad);
}

2
llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
View File

@ -3117,7 +3117,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
if (PropagateShadow) {
std::tie(ShadowPtr, OriginPtr) =
getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
setShadow(&I, IRB.CreateMaskedLoad(ShadowPtr, Alignment, Mask,
setShadow(&I, IRB.CreateMaskedLoad(ShadowTy, ShadowPtr, Alignment, Mask,
getShadow(PassThru), "_msmaskedld"));
} else {
setShadow(&I, getCleanShadow(&I));

8
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -2778,7 +2778,7 @@ void InnerLoopVectorizer::vectorizeInterleaveGroup(
: ShuffledMask;
}
NewLoad =
Builder.CreateMaskedLoad(AddrParts[Part], Group->getAlign(),
Builder.CreateMaskedLoad(VecTy, AddrParts[Part], Group->getAlign(),
GroupMask, PoisonVec, "wide.masked.vec");
}
else
@ -2990,15 +2990,15 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(
if (CreateGatherScatter) {
Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
Value *VectorGep = State.get(Addr, Part);
NewLI = Builder.CreateMaskedGather(VectorGep, Alignment, MaskPart,
NewLI = Builder.CreateMaskedGather(DataTy, VectorGep, Alignment, MaskPart,
nullptr, "wide.masked.gather");
addMetadata(NewLI, LI);
} else {
auto *VecPtr = CreateVecPtr(Part, State.get(Addr, VPIteration(0, 0)));
if (isMaskRequired)
NewLI = Builder.CreateMaskedLoad(
VecPtr, Alignment, BlockInMaskParts[Part], PoisonValue::get(DataTy),
"wide.masked.load");
DataTy, VecPtr, Alignment, BlockInMaskParts[Part],
PoisonValue::get(DataTy), "wide.masked.load");
else
NewLI =
Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load");

2
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
View File

@ -5403,7 +5403,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
for (Value *V : E->Scalars)
CommonAlignment =
commonAlignment(CommonAlignment, cast<LoadInst>(V)->getAlign());
NewLI = Builder.CreateMaskedGather(VecPtr, CommonAlignment);
NewLI = Builder.CreateMaskedGather(VecTy, VecPtr, CommonAlignment);
}
Value *V = propagateMetadata(NewLI, E->Scalars);

13
mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
View File

@ -1516,10 +1516,11 @@ def LLVM_MaskedLoadOp : LLVM_Op<"intr.masked.load"> {
let results = (outs LLVM_Type:$res);
let builders = [LLVM_OneResultOpBuilder];
string llvmBuilder = [{
llvm::Type *Ty = $data->getType()->getPointerElementType();
$res = $pass_thru.empty() ? builder.CreateMaskedLoad(
$data, llvm::Align($alignment), $mask) :
Ty, $data, llvm::Align($alignment), $mask) :
builder.CreateMaskedLoad(
$data, llvm::Align($alignment), $mask, $pass_thru[0]);
Ty, $data, llvm::Align($alignment), $mask, $pass_thru[0]);
}];
let assemblyFormat =
"operands attr-dict `:` functional-type(operands, results)";
@ -1545,10 +1546,14 @@ def LLVM_masked_gather : LLVM_Op<"intr.masked.gather"> {
let results = (outs LLVM_Type:$res);
let builders = [LLVM_OneResultOpBuilder];
string llvmBuilder = [{
llvm::VectorType *PtrVecTy = cast<llvm::VectorType>($ptrs->getType());
llvm::Type *Ty = llvm::VectorType::get(
PtrVecTy->getElementType()->getPointerElementType(),
PtrVecTy->getElementCount());
$res = $pass_thru.empty() ? builder.CreateMaskedGather(
$ptrs, llvm::Align($alignment), $mask) :
Ty, $ptrs, llvm::Align($alignment), $mask) :
builder.CreateMaskedGather(
$ptrs, llvm::Align($alignment), $mask, $pass_thru[0]);
Ty, $ptrs, llvm::Align($alignment), $mask, $pass_thru[0]);
}];
let assemblyFormat =
"operands attr-dict `:` functional-type(operands, results)";