maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Add support for reverse pointer induction variables. These are loops that contain pointers that count backwards. 

For example, this is the hot loop in BZIP:

  do {
    m = *--p;
    *p = ( ... );
  } while (--n);

llvm-svn: 173219
This commit is contained in:
Nadav Rotem 2013-01-23 01:35:00 +00:00
parent 7f63983da6
commit ab3e698ee9
4 changed files with 277 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -327,7 +327,8 @@ public:
IK_NoInduction, ///< Not an induction variable.
IK_IntInduction, ///< Integer induction variable. Step = 1.
IK_ReverseIntInduction, ///< Reverse int induction variable. Step = -1.
IK_PtrInduction ///< Pointer induction variable. Step = sizeof(elem).
IK_PtrInduction, ///< Pointer induction var. Step = sizeof(elem).
IK_ReversePtrInduction ///< Reverse ptr indvar. Step = - sizeof(elem).
};
/// This POD struct holds information about reduction variables.
@ -734,6 +735,9 @@ Value *InnerLoopVectorizer::getConsecutiveVector(Value* Val, unsigned StartIdx,
int LoopVectorizationLegality::isConsecutivePtr(Value *Ptr) {
assert(Ptr->getType()->isPointerTy() && "Unexpected non ptr");
// Make sure that the pointer does not point to structs.
if (cast<PointerType>(Ptr->getType())->getElementType()->isAggregateType())
return 0;
// If this value is a pointer induction variable we know it is consecutive.
PHINode *Phi = dyn_cast_or_null<PHINode>(Ptr);
@ -741,6 +745,8 @@ int LoopVectorizationLegality::isConsecutivePtr(Value *Ptr) {
InductionInfo II = Inductions[Phi];
if (IK_PtrInduction == II.IK)
return 1;
else if (IK_ReversePtrInduction == II.IK)
return -1;
}
GetElementPtrInst *Gep = dyn_cast_or_null<GetElementPtrInst>(Ptr);
@ -750,6 +756,29 @@ int LoopVectorizationLegality::isConsecutivePtr(Value *Ptr) {
unsigned NumOperands = Gep->getNumOperands();
Value *LastIndex = Gep->getOperand(NumOperands - 1);
Value *GpPtr = Gep->getPointerOperand();
// If this GEP value is a consecutive pointer induction variable and all of
// the indices are constant then we know it is consecutive. We can
Phi = dyn_cast<PHINode>(GpPtr);
if (Phi && Inductions.count(Phi)) {
// Make sure that the pointer does not point to structs.
PointerType *GepPtrType = cast<PointerType>(GpPtr->getType());
if (GepPtrType->getElementType()->isAggregateType())
return 0;
// Make sure that all of the index operands are loop invariant.
for (unsigned i = 1; i < NumOperands; ++i)
if (!SE->isLoopInvariant(SE->getSCEV(Gep->getOperand(i)), TheLoop))
return 0;
InductionInfo II = Inductions[Phi];
if (IK_PtrInduction == II.IK)
return 1;
else if (IK_ReversePtrInduction == II.IK)
return -1;
}
// Check that all of the gep indices are uniform except for the last.
for (unsigned i = 0; i < NumOperands - 1; ++i)
if (!SE->isLoopInvariant(SE->getSCEV(Gep->getOperand(i)), TheLoop))
@ -1148,6 +1177,18 @@ InnerLoopVectorizer::createEmptyLoop(LoopVectorizationLegality *Legal) {
LoopBypassBlocks.back()->getTerminator());
break;
}
case LoopVectorizationLegality::IK_ReversePtrInduction: {
// The value at the end of the loop for the reverse pointer is calculated
// by creating a GEP with a negative index starting from the start value.
Value *Zero = ConstantInt::get(CountRoundDown->getType(), 0);
Value *NegIdx = BinaryOperator::CreateSub(Zero, CountRoundDown,
"rev.ind.end",
LoopBypassBlocks.back()->getTerminator());
EndValue = GetElementPtrInst::Create(II.StartValue, NegIdx,
"rev.ptr.ind.end",
LoopBypassBlocks.back()->getTerminator());
break;
}
}// end of case
// The new PHI merges the original incoming value, in case of a bypass,
@ -1625,6 +1666,7 @@ InnerLoopVectorizer::vectorizeBlockInLoop(LoopVectorizationLegality *Legal,
}
case LoopVectorizationLegality::IK_ReverseIntInduction:
case LoopVectorizationLegality::IK_PtrInduction:
case LoopVectorizationLegality::IK_ReversePtrInduction:
// Handle reverse integer and pointer inductions.
Value *StartIdx = 0;
// If we have a single integer induction variable then use it.
@ -1660,15 +1702,23 @@ InnerLoopVectorizer::vectorizeBlockInLoop(LoopVectorizationLegality *Legal,
// Handle the pointer induction variable case.
assert(P->getType()->isPointerTy() && "Unexpected type.");
// Is this a reverse induction ptr or a consecutive induction ptr.
bool Reverse = (LoopVectorizationLegality::IK_ReversePtrInduction ==
II.IK);
// This is the vector of results. Notice that we don't generate
// vector geps because scalar geps result in better code.
for (unsigned part = 0; part < UF; ++part) {
Value *VecVal = UndefValue::get(VectorType::get(P->getType(), VF));
for (unsigned int i = 0; i < VF; ++i) {
Constant *Idx = ConstantInt::get(Induction->getType(),
i + part * VF);
Value *GlobalIdx = Builder.CreateAdd(NormalizedIdx, Idx,
"gep.idx");
int EltIndex = (i + part * VF) * (Reverse ? -1 : 1);
Constant *Idx = ConstantInt::get(Induction->getType(), EltIndex);
Value *GlobalIdx;
if (!Reverse)
GlobalIdx = Builder.CreateAdd(NormalizedIdx, Idx, "gep.idx");
else
GlobalIdx = Builder.CreateSub(Idx, NormalizedIdx, "gep.ridx");
Value *SclrGep = Builder.CreateGEP(II.StartValue, GlobalIdx,
"next.gep");
VecVal = Builder.CreateInsertElement(VecVal, SclrGep,
@ -1786,7 +1836,19 @@ InnerLoopVectorizer::vectorizeBlockInLoop(LoopVectorizationLegality *Legal,
// Handle consecutive stores.
GetElementPtrInst *Gep = dyn_cast<GetElementPtrInst>(Ptr);
if (Gep) {
if (Gep && Legal->isInductionVariable(Gep->getPointerOperand())) {
Value *PtrOperand = Gep->getPointerOperand();
Value *FirstBasePtr = getVectorValue(PtrOperand)[0];
FirstBasePtr = Builder.CreateExtractElement(FirstBasePtr, Zero);
// Create the new GEP with the new induction variable.
GetElementPtrInst *Gep2 = cast<GetElementPtrInst>(Gep->clone());
Gep2->setOperand(0, FirstBasePtr);
Ptr = Builder.Insert(Gep2);
} else if (Gep) {
assert(SE->isLoopInvariant(SE->getSCEV(Gep->getPointerOperand()),
OrigLoop) && "Base ptr must be invariant");
// The last index does not have to be the induction. It can be
// consecutive and be a function of the index. For example A[I+1];
unsigned NumOperands = Gep->getNumOperands();
@ -1844,7 +1906,18 @@ InnerLoopVectorizer::vectorizeBlockInLoop(LoopVectorizationLegality *Legal,
}
GetElementPtrInst *Gep = dyn_cast<GetElementPtrInst>(Ptr);
if (Gep) {
if (Gep && Legal->isInductionVariable(Gep->getPointerOperand())) {
Value *PtrOperand = Gep->getPointerOperand();
Value *FirstBasePtr = getVectorValue(PtrOperand)[0];
FirstBasePtr = Builder.CreateExtractElement(FirstBasePtr, Zero);
// Create the new GEP with the new induction variable.
GetElementPtrInst *Gep2 = cast<GetElementPtrInst>(Gep->clone());
Gep2->setOperand(0, FirstBasePtr);
Ptr = Builder.Insert(Gep2);
} else if (Gep) {
assert(SE->isLoopInvariant(SE->getSCEV(Gep->getPointerOperand()),
OrigLoop) && "Base ptr must be invariant");
// The last index does not have to be the induction. It can be
// consecutive and be a function of the index. For example A[I+1];
unsigned NumOperands = Gep->getNumOperands();
@ -2589,6 +2662,8 @@ LoopVectorizationLegality::isInductionVariable(PHINode *Phi) {
uint64_t Size = DL->getTypeAllocSize(PhiTy->getPointerElementType());
if (C->getValue()->equalsInt(Size))
return IK_PtrInduction;
else if (C->getValue()->equalsInt(0 - Size))
return IK_ReversePtrInduction;
return IK_NoInduction;
}

71
llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll Normal file
View File

@ -0,0 +1,71 @@
; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -dce -instcombine -S -enable-if-conversion | FileCheck %s
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.8.0"
;CHECK: fc
;CHECK: load <4 x i16>
;CHECK-NEXT: shufflevector <4 x i16>
;CHECK: select <4 x i1>
;CHECK: store <4 x i16>
;CHECK: ret
define void @fc(i16* nocapture %p, i32 %n, i32 %size) nounwind uwtable ssp {
entry:
br label %do.body
do.body: ; preds = %cond.end, %entry
%n.addr.0 = phi i32 [ %n, %entry ], [ %dec, %cond.end ]
%p.addr.0 = phi i16* [ %p, %entry ], [ %incdec.ptr, %cond.end ]
%incdec.ptr = getelementptr inbounds i16* %p.addr.0, i64 -1
%0 = load i16* %incdec.ptr, align 2, !tbaa !0
%conv = zext i16 %0 to i32
%cmp = icmp ult i32 %conv, %size
br i1 %cmp, label %cond.end, label %cond.true
cond.true: ; preds = %do.body
%sub = sub i32 %conv, %size
%phitmp = trunc i32 %sub to i16
br label %cond.end
cond.end: ; preds = %do.body, %cond.true
%cond = phi i16 [ %phitmp, %cond.true ], [ 0, %do.body ]
store i16 %cond, i16* %incdec.ptr, align 2, !tbaa !0
%dec = add i32 %n.addr.0, -1
%tobool = icmp eq i32 %dec, 0
br i1 %tobool, label %do.end, label %do.body
do.end: ; preds = %cond.end
ret void
}
;CHECK: example1
;CHECK: load <4 x i32>
;CHECK-NEXT: shufflevector <4 x i32>
;CHECK: select <4 x i1>
;CHECK: store <4 x i32>
;CHECK: ret
define void @example1(i32* nocapture %a, i32 %n, i32 %wsize) nounwind uwtable ssp {
entry:
br label %do.body
do.body: ; preds = %do.body, %entry
%n.addr.0 = phi i32 [ %n, %entry ], [ %dec, %do.body ]
%p.0 = phi i32* [ %a, %entry ], [ %incdec.ptr, %do.body ]
%incdec.ptr = getelementptr inbounds i32* %p.0, i64 -1
%0 = load i32* %incdec.ptr, align 4, !tbaa !3
%cmp = icmp slt i32 %0, %wsize
%sub = sub nsw i32 %0, %wsize
%cond = select i1 %cmp, i32 0, i32 %sub
store i32 %cond, i32* %incdec.ptr, align 4, !tbaa !3
%dec = add nsw i32 %n.addr.0, -1
%tobool = icmp eq i32 %dec, 0
br i1 %tobool, label %do.end, label %do.body
do.end: ; preds = %do.body
ret void
}
!0 = metadata !{metadata !"short", metadata !1}
!1 = metadata !{metadata !"omnipotent char", metadata !2}
!2 = metadata !{metadata !"Simple C/C++ TBAA"}
!3 = metadata !{metadata !"int", metadata !1}

74
llvm/test/Transforms/LoopVectorize/ptr_loops.ll Normal file
View File

@ -0,0 +1,74 @@
; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -dce -instcombine -S -enable-if-conversion | FileCheck %s
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.8.0"
@A = global [36 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35], align 16
@B = global [36 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35], align 16
;CHECK:_Z5test1v
;CHECK: load <4 x i32>
;CHECK: shufflevector <4 x i32>
;CHECK: store <4 x i32>
;CHECK: ret
define i32 @_Z5test1v() nounwind uwtable ssp {
br label %1
; <label>:1 ; preds = %0, %1
%p.02 = phi i32* [ getelementptr inbounds ([36 x i32]* @A, i64 0, i64 18), %0 ], [ %4, %1 ]
%b.01 = phi i32* [ getelementptr inbounds ([36 x i32]* @B, i64 0, i64 0), %0 ], [ %5, %1 ]
%2 = load i32* %b.01, align 4
%3 = shl nsw i32 %2, 1
store i32 %3, i32* %p.02, align 4
%4 = getelementptr inbounds i32* %p.02, i64 -1
%5 = getelementptr inbounds i32* %b.01, i64 1
%6 = icmp eq i32* %4, getelementptr ([36 x i32]* @A, i64 128102389400760775, i64 3)
br i1 %6, label %7, label %1
; <label>:7 ; preds = %1
ret i32 0
}
;CHECK:_Z5test2v
;CHECK: load <4 x i32>
;CHECK: shufflevector <4 x i32>
;CHECK: store <4 x i32>
;CHECK: ret
define i32 @_Z5test2v() nounwind uwtable ssp {
br label %1
; <label>:1 ; preds = %0, %1
%p.02 = phi i32* [ getelementptr inbounds ([36 x i32]* @A, i64 0, i64 25), %0 ], [ %3, %1 ]
%b.01 = phi i32* [ getelementptr inbounds ([36 x i32]* @B, i64 0, i64 2), %0 ], [ %4, %1 ]
%2 = load i32* %b.01, align 4
store i32 %2, i32* %p.02, align 4
%3 = getelementptr inbounds i32* %p.02, i64 -1
%4 = getelementptr inbounds i32* %b.01, i64 1
%5 = icmp eq i32* %4, getelementptr inbounds ([36 x i32]* @A, i64 0, i64 18)
br i1 %5, label %6, label %1
; <label>:6 ; preds = %1
ret i32 0
}
;CHECK:_Z5test3v
;CHECK: load <4 x i32>
;CHECK: shufflevector <4 x i32>
;CHECK: store <4 x i32>
;CHECK: ret
define i32 @_Z5test3v() nounwind uwtable ssp {
br label %1
; <label>:1 ; preds = %0, %1
%p.02 = phi i32* [ getelementptr inbounds ([36 x i32]* @A, i64 0, i64 29), %0 ], [ %3, %1 ]
%b.01 = phi i32* [ getelementptr inbounds ([36 x i32]* @B, i64 0, i64 5), %0 ], [ %4, %1 ]
%2 = load i32* %b.01, align 4
store i32 %2, i32* %p.02, align 4
%3 = getelementptr inbounds i32* %p.02, i64 -1
%4 = getelementptr inbounds i32* %b.01, i64 1
%5 = icmp eq i32* %3, getelementptr ([36 x i32]* @A, i64 128102389400760775, i64 3)
br i1 %5, label %6, label %1
; <label>:6 ; preds = %1
ret i32 0
}

50
llvm/test/Transforms/LoopVectorize/struct_access.ll Normal file
View File

@ -0,0 +1,50 @@
; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -instcombine -S | FileCheck %s
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.9.0"
%struct.coordinate = type { i32, i32 }
; Make sure that we don't generate a wide load when accessing the struct.
; struct coordinate {
; int x;
; int y;
; };
;
;
; int foo(struct coordinate *A, int n) {
;
; int sum = 0;
; for (int i = 0; i < n; ++i)
; sum += A[i].x;
;
; return sum;
; }
;CHECK: @foo
;CHECK-NOT: load <4 x i32>
;CHECK: ret
define i32 @foo(%struct.coordinate* nocapture %A, i32 %n) nounwind uwtable readonly ssp {
entry:
%cmp4 = icmp sgt i32 %n, 0
br i1 %cmp4, label %for.body, label %for.end
for.body: ; preds = %entry, %for.body
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
%sum.05 = phi i32 [ %add, %for.body ], [ 0, %entry ]
%x = getelementptr inbounds %struct.coordinate* %A, i64 %indvars.iv, i32 0
%0 = load i32* %x, align 4, !tbaa !0
%add = add nsw i32 %0, %sum.05
%indvars.iv.next = add i64 %indvars.iv, 1
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
%exitcond = icmp eq i32 %lftr.wideiv, %n
br i1 %exitcond, label %for.end, label %for.body
for.end: ; preds = %for.body, %entry
%sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
ret i32 %sum.0.lcssa
}
!0 = metadata !{metadata !"int", metadata !1}
!1 = metadata !{metadata !"omnipotent char", metadata !2}
!2 = metadata !{metadata !"Simple C/C++ TBAA"}