diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h index fda663844a67..e5f62fb9625e 100644 --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -430,10 +430,12 @@ public: CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr); /// \brief Create a call to Masked Load intrinsic - CallInst *CreateMaskedLoad(ArrayRef Ops); + CallInst *CreateMaskedLoad(Value *Ptr, unsigned Align, Value *Mask, + Value *PassThru = 0, const Twine &Name = ""); /// \brief Create a call to Masked Store intrinsic - CallInst *CreateMaskedStore(ArrayRef Ops); + CallInst *CreateMaskedStore(Value *Val, Value *Ptr, unsigned Align, + Value *Mask); /// \brief Create an assume intrinsic call that allows the optimizer to /// assume that the provided condition will be true. @@ -465,7 +467,7 @@ private: /// \brief Create a call to a masked intrinsic with given Id. /// Masked intrinsic has only one overloaded type - data type. CallInst *CreateMaskedIntrinsic(unsigned Id, ArrayRef Ops, - Type *DataTy); + Type *DataTy, const Twine &Name = ""); Value *getCastedInt8PtrValue(Value *Ptr); }; diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp index 9782ecc4d14a..ef1f2267682b 100644 --- a/llvm/lib/IR/IRBuilder.cpp +++ b/llvm/lib/IR/IRBuilder.cpp @@ -185,30 +185,49 @@ CallInst *IRBuilderBase::CreateAssumption(Value *Cond) { } /// Create a call to a Masked Load intrinsic. -/// Ops - an array of operands. -CallInst *IRBuilderBase::CreateMaskedLoad(ArrayRef Ops) { - // The only one overloaded type - the type of passthru value in this case - Type *DataTy = Ops[3]->getType(); - return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, DataTy); +/// Ptr - the base pointer for the load +/// Align - alignment of the source location +/// Mask - an vector of booleans which indicates what vector lanes should +/// be accessed in memory +/// PassThru - a pass-through value that is used to fill the masked-off lanes +/// of the result +/// Name - name of the result variable +CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align, + Value *Mask, Value *PassThru, + const Twine &Name) { + assert(Ptr->getType()->isPointerTy() && "Ptr must be of pointer type"); + // DataTy is the overloaded type + Type *DataTy = cast(Ptr->getType())->getElementType(); + assert(DataTy->isVectorTy() && "Ptr should point to a vector"); + if (!PassThru) + PassThru = UndefValue::get(DataTy); + Value *Ops[] = { Ptr, getInt32(Align), Mask, PassThru}; + return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, DataTy, Name); } /// Create a call to a Masked Store intrinsic. -/// Ops - an array of operands. -CallInst *IRBuilderBase::CreateMaskedStore(ArrayRef Ops) { - // DataTy - type of the data to be stored - the only one overloaded type - Type *DataTy = Ops[0]->getType(); - return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, DataTy); +/// Val - the data to be stored, +/// Ptr - the base pointer for the store +/// Align - alignment of the destination location +/// Mask - an vector of booleans which indicates what vector lanes should +/// be accessed in memory +CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr, + unsigned Align, Value *Mask) { + Value *Ops[] = { Val, Ptr, getInt32(Align), Mask }; + // Type of the data to be stored - the only one overloaded type + return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, Val->getType()); } /// Create a call to a Masked intrinsic, with given intrinsic Id, /// an array of operands - Ops, and one overloaded type - DataTy CallInst *IRBuilderBase::CreateMaskedIntrinsic(unsigned Id, ArrayRef Ops, - Type *DataTy) { + Type *DataTy, + const Twine &Name) { Module *M = BB->getParent()->getParent(); Type *OverloadedTypes[] = { DataTy }; Value *TheFn = Intrinsic::getDeclaration(M, (Intrinsic::ID)Id, OverloadedTypes); - return createCallHelper(TheFn, Ops, this); + return createCallHelper(TheFn, Ops, this, Name); } CallInst *IRBuilderBase::CreateGCStatepoint(Value *ActualCallee, diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index d0457287099a..006efba6a06b 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1852,6 +1852,7 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) { Ptr = Builder.CreateExtractElement(PtrVal[0], Zero); } + VectorParts Mask = createBlockInMask(Instr->getParent()); // Handle Stores: if (SI) { assert(!Legal->isUniform(SI->getPointerOperand()) && @@ -1860,7 +1861,7 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) { // We don't want to update the value in the map as it might be used in // another expression. So don't use a reference type for "StoredVal". VectorParts StoredVal = getVectorValue(SI->getValueOperand()); - + for (unsigned Part = 0; Part < UF; ++Part) { // Calculate the pointer for the specific unroll-part. Value *PartPtr = Builder.CreateGEP(Ptr, Builder.getInt32(Part * VF)); @@ -1879,15 +1880,9 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) { DataTy->getPointerTo(AddressSpace)); Instruction *NewSI; - if (Legal->isMaskRequired(SI)) { - VectorParts Cond = createBlockInMask(SI->getParent()); - SmallVector Ops; - Ops.push_back(StoredVal[Part]); - Ops.push_back(VecPtr); - Ops.push_back(Builder.getInt32(Alignment)); - Ops.push_back(Cond[Part]); - NewSI = Builder.CreateMaskedStore(Ops); - } + if (Legal->isMaskRequired(SI)) + NewSI = Builder.CreateMaskedStore(StoredVal[Part], VecPtr, Alignment, + Mask[Part]); else NewSI = Builder.CreateAlignedStore(StoredVal[Part], VecPtr, Alignment); propagateMetadata(NewSI, SI); @@ -1912,18 +1907,12 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) { Instruction* NewLI; Value *VecPtr = Builder.CreateBitCast(PartPtr, DataTy->getPointerTo(AddressSpace)); - if (Legal->isMaskRequired(LI)) { - VectorParts SrcMask = createBlockInMask(LI->getParent()); - SmallVector Ops; - Ops.push_back(VecPtr); - Ops.push_back(Builder.getInt32(Alignment)); - Ops.push_back(SrcMask[Part]); - Ops.push_back(UndefValue::get(DataTy)); - NewLI = Builder.CreateMaskedLoad(Ops); - } - else { + if (Legal->isMaskRequired(LI)) + NewLI = Builder.CreateMaskedLoad(VecPtr, Alignment, Mask[Part], + UndefValue::get(DataTy), + "wide.masked.load"); + else NewLI = Builder.CreateAlignedLoad(VecPtr, Alignment, "wide.load"); - } propagateMetadata(NewLI, LI); Entry[Part] = Reverse ? reverseVector(NewLI) : NewLI; }