From ab3e698ee930770c07a5a5c464044f849cc2a1b7 Mon Sep 17 00:00:00 2001 From: Nadav Rotem Date: Wed, 23 Jan 2013 01:35:00 +0000 Subject: [PATCH] Add support for reverse pointer induction variables. These are loops that contain pointers that count backwards. For example, this is the hot loop in BZIP: do { m = *--p; *p = ( ... ); } while (--n); llvm-svn: 173219 --- .../Transforms/Vectorize/LoopVectorize.cpp | 89 +++++++++++++++++-- .../LoopVectorize/bzip_reverse_loops.ll | 71 +++++++++++++++ .../Transforms/LoopVectorize/ptr_loops.ll | 74 +++++++++++++++ .../Transforms/LoopVectorize/struct_access.ll | 50 +++++++++++ 4 files changed, 277 insertions(+), 7 deletions(-) create mode 100644 llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll create mode 100644 llvm/test/Transforms/LoopVectorize/ptr_loops.ll create mode 100644 llvm/test/Transforms/LoopVectorize/struct_access.ll diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 447f24a99bdb..0996b7b2cd33 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -327,7 +327,8 @@ public: IK_NoInduction, ///< Not an induction variable. IK_IntInduction, ///< Integer induction variable. Step = 1. IK_ReverseIntInduction, ///< Reverse int induction variable. Step = -1. - IK_PtrInduction ///< Pointer induction variable. Step = sizeof(elem). + IK_PtrInduction, ///< Pointer induction var. Step = sizeof(elem). + IK_ReversePtrInduction ///< Reverse ptr indvar. Step = - sizeof(elem). }; /// This POD struct holds information about reduction variables. @@ -734,6 +735,9 @@ Value *InnerLoopVectorizer::getConsecutiveVector(Value* Val, unsigned StartIdx, int LoopVectorizationLegality::isConsecutivePtr(Value *Ptr) { assert(Ptr->getType()->isPointerTy() && "Unexpected non ptr"); + // Make sure that the pointer does not point to structs. + if (cast(Ptr->getType())->getElementType()->isAggregateType()) + return 0; // If this value is a pointer induction variable we know it is consecutive. PHINode *Phi = dyn_cast_or_null(Ptr); @@ -741,6 +745,8 @@ int LoopVectorizationLegality::isConsecutivePtr(Value *Ptr) { InductionInfo II = Inductions[Phi]; if (IK_PtrInduction == II.IK) return 1; + else if (IK_ReversePtrInduction == II.IK) + return -1; } GetElementPtrInst *Gep = dyn_cast_or_null(Ptr); @@ -750,6 +756,29 @@ int LoopVectorizationLegality::isConsecutivePtr(Value *Ptr) { unsigned NumOperands = Gep->getNumOperands(); Value *LastIndex = Gep->getOperand(NumOperands - 1); + Value *GpPtr = Gep->getPointerOperand(); + // If this GEP value is a consecutive pointer induction variable and all of + // the indices are constant then we know it is consecutive. We can + Phi = dyn_cast(GpPtr); + if (Phi && Inductions.count(Phi)) { + + // Make sure that the pointer does not point to structs. + PointerType *GepPtrType = cast(GpPtr->getType()); + if (GepPtrType->getElementType()->isAggregateType()) + return 0; + + // Make sure that all of the index operands are loop invariant. + for (unsigned i = 1; i < NumOperands; ++i) + if (!SE->isLoopInvariant(SE->getSCEV(Gep->getOperand(i)), TheLoop)) + return 0; + + InductionInfo II = Inductions[Phi]; + if (IK_PtrInduction == II.IK) + return 1; + else if (IK_ReversePtrInduction == II.IK) + return -1; + } + // Check that all of the gep indices are uniform except for the last. for (unsigned i = 0; i < NumOperands - 1; ++i) if (!SE->isLoopInvariant(SE->getSCEV(Gep->getOperand(i)), TheLoop)) @@ -1148,6 +1177,18 @@ InnerLoopVectorizer::createEmptyLoop(LoopVectorizationLegality *Legal) { LoopBypassBlocks.back()->getTerminator()); break; } + case LoopVectorizationLegality::IK_ReversePtrInduction: { + // The value at the end of the loop for the reverse pointer is calculated + // by creating a GEP with a negative index starting from the start value. + Value *Zero = ConstantInt::get(CountRoundDown->getType(), 0); + Value *NegIdx = BinaryOperator::CreateSub(Zero, CountRoundDown, + "rev.ind.end", + LoopBypassBlocks.back()->getTerminator()); + EndValue = GetElementPtrInst::Create(II.StartValue, NegIdx, + "rev.ptr.ind.end", + LoopBypassBlocks.back()->getTerminator()); + break; + } }// end of case // The new PHI merges the original incoming value, in case of a bypass, @@ -1625,6 +1666,7 @@ InnerLoopVectorizer::vectorizeBlockInLoop(LoopVectorizationLegality *Legal, } case LoopVectorizationLegality::IK_ReverseIntInduction: case LoopVectorizationLegality::IK_PtrInduction: + case LoopVectorizationLegality::IK_ReversePtrInduction: // Handle reverse integer and pointer inductions. Value *StartIdx = 0; // If we have a single integer induction variable then use it. @@ -1660,15 +1702,23 @@ InnerLoopVectorizer::vectorizeBlockInLoop(LoopVectorizationLegality *Legal, // Handle the pointer induction variable case. assert(P->getType()->isPointerTy() && "Unexpected type."); + // Is this a reverse induction ptr or a consecutive induction ptr. + bool Reverse = (LoopVectorizationLegality::IK_ReversePtrInduction == + II.IK); + // This is the vector of results. Notice that we don't generate // vector geps because scalar geps result in better code. for (unsigned part = 0; part < UF; ++part) { Value *VecVal = UndefValue::get(VectorType::get(P->getType(), VF)); for (unsigned int i = 0; i < VF; ++i) { - Constant *Idx = ConstantInt::get(Induction->getType(), - i + part * VF); - Value *GlobalIdx = Builder.CreateAdd(NormalizedIdx, Idx, - "gep.idx"); + int EltIndex = (i + part * VF) * (Reverse ? -1 : 1); + Constant *Idx = ConstantInt::get(Induction->getType(), EltIndex); + Value *GlobalIdx; + if (!Reverse) + GlobalIdx = Builder.CreateAdd(NormalizedIdx, Idx, "gep.idx"); + else + GlobalIdx = Builder.CreateSub(Idx, NormalizedIdx, "gep.ridx"); + Value *SclrGep = Builder.CreateGEP(II.StartValue, GlobalIdx, "next.gep"); VecVal = Builder.CreateInsertElement(VecVal, SclrGep, @@ -1786,7 +1836,19 @@ InnerLoopVectorizer::vectorizeBlockInLoop(LoopVectorizationLegality *Legal, // Handle consecutive stores. GetElementPtrInst *Gep = dyn_cast(Ptr); - if (Gep) { + if (Gep && Legal->isInductionVariable(Gep->getPointerOperand())) { + Value *PtrOperand = Gep->getPointerOperand(); + Value *FirstBasePtr = getVectorValue(PtrOperand)[0]; + FirstBasePtr = Builder.CreateExtractElement(FirstBasePtr, Zero); + + // Create the new GEP with the new induction variable. + GetElementPtrInst *Gep2 = cast(Gep->clone()); + Gep2->setOperand(0, FirstBasePtr); + Ptr = Builder.Insert(Gep2); + } else if (Gep) { + assert(SE->isLoopInvariant(SE->getSCEV(Gep->getPointerOperand()), + OrigLoop) && "Base ptr must be invariant"); + // The last index does not have to be the induction. It can be // consecutive and be a function of the index. For example A[I+1]; unsigned NumOperands = Gep->getNumOperands(); @@ -1844,7 +1906,18 @@ InnerLoopVectorizer::vectorizeBlockInLoop(LoopVectorizationLegality *Legal, } GetElementPtrInst *Gep = dyn_cast(Ptr); - if (Gep) { + if (Gep && Legal->isInductionVariable(Gep->getPointerOperand())) { + Value *PtrOperand = Gep->getPointerOperand(); + Value *FirstBasePtr = getVectorValue(PtrOperand)[0]; + FirstBasePtr = Builder.CreateExtractElement(FirstBasePtr, Zero); + // Create the new GEP with the new induction variable. + GetElementPtrInst *Gep2 = cast(Gep->clone()); + Gep2->setOperand(0, FirstBasePtr); + Ptr = Builder.Insert(Gep2); + } else if (Gep) { + assert(SE->isLoopInvariant(SE->getSCEV(Gep->getPointerOperand()), + OrigLoop) && "Base ptr must be invariant"); + // The last index does not have to be the induction. It can be // consecutive and be a function of the index. For example A[I+1]; unsigned NumOperands = Gep->getNumOperands(); @@ -2589,6 +2662,8 @@ LoopVectorizationLegality::isInductionVariable(PHINode *Phi) { uint64_t Size = DL->getTypeAllocSize(PhiTy->getPointerElementType()); if (C->getValue()->equalsInt(Size)) return IK_PtrInduction; + else if (C->getValue()->equalsInt(0 - Size)) + return IK_ReversePtrInduction; return IK_NoInduction; } diff --git a/llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll b/llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll new file mode 100644 index 000000000000..431e422c2fbe --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll @@ -0,0 +1,71 @@ +; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -dce -instcombine -S -enable-if-conversion | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.8.0" + +;CHECK: fc +;CHECK: load <4 x i16> +;CHECK-NEXT: shufflevector <4 x i16> +;CHECK: select <4 x i1> +;CHECK: store <4 x i16> +;CHECK: ret +define void @fc(i16* nocapture %p, i32 %n, i32 %size) nounwind uwtable ssp { +entry: + br label %do.body + +do.body: ; preds = %cond.end, %entry + %n.addr.0 = phi i32 [ %n, %entry ], [ %dec, %cond.end ] + %p.addr.0 = phi i16* [ %p, %entry ], [ %incdec.ptr, %cond.end ] + %incdec.ptr = getelementptr inbounds i16* %p.addr.0, i64 -1 + %0 = load i16* %incdec.ptr, align 2, !tbaa !0 + %conv = zext i16 %0 to i32 + %cmp = icmp ult i32 %conv, %size + br i1 %cmp, label %cond.end, label %cond.true + +cond.true: ; preds = %do.body + %sub = sub i32 %conv, %size + %phitmp = trunc i32 %sub to i16 + br label %cond.end + +cond.end: ; preds = %do.body, %cond.true + %cond = phi i16 [ %phitmp, %cond.true ], [ 0, %do.body ] + store i16 %cond, i16* %incdec.ptr, align 2, !tbaa !0 + %dec = add i32 %n.addr.0, -1 + %tobool = icmp eq i32 %dec, 0 + br i1 %tobool, label %do.end, label %do.body + +do.end: ; preds = %cond.end + ret void +} + +;CHECK: example1 +;CHECK: load <4 x i32> +;CHECK-NEXT: shufflevector <4 x i32> +;CHECK: select <4 x i1> +;CHECK: store <4 x i32> +;CHECK: ret +define void @example1(i32* nocapture %a, i32 %n, i32 %wsize) nounwind uwtable ssp { +entry: + br label %do.body + +do.body: ; preds = %do.body, %entry + %n.addr.0 = phi i32 [ %n, %entry ], [ %dec, %do.body ] + %p.0 = phi i32* [ %a, %entry ], [ %incdec.ptr, %do.body ] + %incdec.ptr = getelementptr inbounds i32* %p.0, i64 -1 + %0 = load i32* %incdec.ptr, align 4, !tbaa !3 + %cmp = icmp slt i32 %0, %wsize + %sub = sub nsw i32 %0, %wsize + %cond = select i1 %cmp, i32 0, i32 %sub + store i32 %cond, i32* %incdec.ptr, align 4, !tbaa !3 + %dec = add nsw i32 %n.addr.0, -1 + %tobool = icmp eq i32 %dec, 0 + br i1 %tobool, label %do.end, label %do.body + +do.end: ; preds = %do.body + ret void +} + +!0 = metadata !{metadata !"short", metadata !1} +!1 = metadata !{metadata !"omnipotent char", metadata !2} +!2 = metadata !{metadata !"Simple C/C++ TBAA"} +!3 = metadata !{metadata !"int", metadata !1} diff --git a/llvm/test/Transforms/LoopVectorize/ptr_loops.ll b/llvm/test/Transforms/LoopVectorize/ptr_loops.ll new file mode 100644 index 000000000000..25599f8f4c3c --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/ptr_loops.ll @@ -0,0 +1,74 @@ +; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -dce -instcombine -S -enable-if-conversion | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.8.0" + +@A = global [36 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35], align 16 +@B = global [36 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35], align 16 + +;CHECK:_Z5test1v +;CHECK: load <4 x i32> +;CHECK: shufflevector <4 x i32> +;CHECK: store <4 x i32> +;CHECK: ret +define i32 @_Z5test1v() nounwind uwtable ssp { + br label %1 + +;