Some code improvements in Masked Load/Store.

No functional changes.

llvm-svn: 224986

llvm/include/llvm/IR/IRBuilder.h

@@ -430,10 +430,12 @@ public:
   CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr);
 
   /// \brief Create a call to Masked Load intrinsic
-  CallInst *CreateMaskedLoad(ArrayRef<Value *> Ops);
+  CallInst *CreateMaskedLoad(Value *Ptr, unsigned Align, Value *Mask,
+                             Value *PassThru = 0, const Twine &Name = "");
 
   /// \brief Create a call to Masked Store intrinsic
-  CallInst *CreateMaskedStore(ArrayRef<Value *> Ops);
+  CallInst *CreateMaskedStore(Value *Val, Value *Ptr, unsigned Align,
+                              Value *Mask);
 
   /// \brief Create an assume intrinsic call that allows the optimizer to
   /// assume that the provided condition will be true.
@@ -465,7 +467,7 @@ private:
   /// \brief Create a call to a masked intrinsic with given Id.
   /// Masked intrinsic has only one overloaded type - data type.
   CallInst *CreateMaskedIntrinsic(unsigned Id, ArrayRef<Value *> Ops,
-                                  Type *DataTy);
+                                  Type *DataTy, const Twine &Name = "");
 
   Value *getCastedInt8PtrValue(Value *Ptr);
 };

llvm/lib/IR/IRBuilder.cpp

@@ -185,30 +185,49 @@ CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
 }
 
 /// Create a call to a Masked Load intrinsic.
-/// Ops - an array of operands.
+/// Ptr      - the base pointer for the load
-CallInst *IRBuilderBase::CreateMaskedLoad(ArrayRef<Value *> Ops) {
+/// Align    - alignment of the source location
-  // The only one overloaded type - the type of passthru value in this case
+/// Mask     - an vector of booleans which indicates what vector lanes should
-  Type *DataTy = Ops[3]->getType();
+///            be accessed in memory
-  return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, DataTy);
+/// PassThru - a pass-through value that is used to fill the masked-off lanes
+///            of the result
+/// Name     - name of the result variable
+CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align,
+                                          Value *Mask, Value *PassThru,
+                                          const Twine &Name) {
+  assert(Ptr->getType()->isPointerTy() && "Ptr must be of pointer type");
+  // DataTy is the overloaded type
+  Type *DataTy = cast<PointerType>(Ptr->getType())->getElementType();
+  assert(DataTy->isVectorTy() && "Ptr should point to a vector");
+  if (!PassThru)
+    PassThru = UndefValue::get(DataTy);
+  Value *Ops[] = { Ptr, getInt32(Align), Mask,  PassThru};
+  return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, DataTy, Name);
 }
 
 /// Create a call to a Masked Store intrinsic.
-/// Ops - an array of operands.
+/// Val   - the data to be stored,
-CallInst *IRBuilderBase::CreateMaskedStore(ArrayRef<Value *> Ops) {
+/// Ptr   - the base pointer for the store
-  // DataTy - type of the data to be stored - the only one overloaded type
+/// Align - alignment of the destination location
-  Type *DataTy = Ops[0]->getType();
+/// Mask  - an vector of booleans which indicates what vector lanes should
-  return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, DataTy);
+///         be accessed in memory
+CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
+                                           unsigned Align, Value *Mask) {
+  Value *Ops[] = { Val, Ptr, getInt32(Align), Mask };
+  // Type of the data to be stored - the only one overloaded type
+  return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, Val->getType());
 }
 
 /// Create a call to a Masked intrinsic, with given intrinsic Id,
 /// an array of operands - Ops, and one overloaded type - DataTy
 CallInst *IRBuilderBase::CreateMaskedIntrinsic(unsigned Id,
                                                ArrayRef<Value *> Ops,
-                                               Type *DataTy) {
+                                               Type *DataTy,
+                                               const Twine &Name) {
   Module *M = BB->getParent()->getParent();
   Type *OverloadedTypes[] = { DataTy };
   Value *TheFn = Intrinsic::getDeclaration(M, (Intrinsic::ID)Id, OverloadedTypes);
-  return createCallHelper(TheFn, Ops, this);
+  return createCallHelper(TheFn, Ops, this, Name);
 }
 
 CallInst *IRBuilderBase::CreateGCStatepoint(Value *ActualCallee,

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -1852,6 +1852,7 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) {
     Ptr = Builder.CreateExtractElement(PtrVal[0], Zero);
   }
 
+  VectorParts Mask = createBlockInMask(Instr->getParent());
   // Handle Stores:
   if (SI) {
     assert(!Legal->isUniform(SI->getPointerOperand()) &&
@@ -1860,7 +1861,7 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) {
     // We don't want to update the value in the map as it might be used in
     // another expression. So don't use a reference type for "StoredVal".
     VectorParts StoredVal = getVectorValue(SI->getValueOperand());
-
+    
     for (unsigned Part = 0; Part < UF; ++Part) {
       // Calculate the pointer for the specific unroll-part.
       Value *PartPtr = Builder.CreateGEP(Ptr, Builder.getInt32(Part * VF));
@@ -1879,15 +1880,9 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) {
                                             DataTy->getPointerTo(AddressSpace));
 
       Instruction *NewSI;
-      if (Legal->isMaskRequired(SI)) {
+      if (Legal->isMaskRequired(SI))
-        VectorParts Cond = createBlockInMask(SI->getParent());
+        NewSI = Builder.CreateMaskedStore(StoredVal[Part], VecPtr, Alignment,
-        SmallVector <Value *, 8> Ops;
+                                          Mask[Part]);
-        Ops.push_back(StoredVal[Part]);
-        Ops.push_back(VecPtr);
-        Ops.push_back(Builder.getInt32(Alignment));
-        Ops.push_back(Cond[Part]);
-        NewSI = Builder.CreateMaskedStore(Ops);
-      }
       else 
         NewSI = Builder.CreateAlignedStore(StoredVal[Part], VecPtr, Alignment);
       propagateMetadata(NewSI, SI);
@@ -1912,18 +1907,12 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) {
     Instruction* NewLI;
     Value *VecPtr = Builder.CreateBitCast(PartPtr,
                                           DataTy->getPointerTo(AddressSpace));
-    if (Legal->isMaskRequired(LI)) {
+    if (Legal->isMaskRequired(LI))
-      VectorParts SrcMask = createBlockInMask(LI->getParent());
+      NewLI = Builder.CreateMaskedLoad(VecPtr, Alignment, Mask[Part],
-      SmallVector <Value *, 8> Ops;
+                                       UndefValue::get(DataTy),
-      Ops.push_back(VecPtr);
+                                       "wide.masked.load");
-      Ops.push_back(Builder.getInt32(Alignment));
+    else
-      Ops.push_back(SrcMask[Part]);
-      Ops.push_back(UndefValue::get(DataTy));
-      NewLI = Builder.CreateMaskedLoad(Ops);
-    }
-    else {
       NewLI = Builder.CreateAlignedLoad(VecPtr, Alignment, "wide.load");
-    }
     propagateMetadata(NewLI, LI);
     Entry[Part] = Reverse ? reverseVector(NewLI) :  NewLI;
   }