From a0c6ae02a58842e1dcb998e619d0284d62bf8f1d Mon Sep 17 00:00:00 2001 From: Michael Kuperstein Date: Mon, 6 Jun 2016 23:38:33 +0000 Subject: [PATCH] [InstCombine] scalarizePHI should not assume the code it sees has been CSE'd scalarizePHI only looked for phis that have exactly two uses - the "latch" use, and an extract. Unfortunately, we can not assume all equivalent extracts are CSE'd, since InstCombine itself may create an extract which is a duplicate of an existing one. This extends it to handle several distinct extracts from the same index. This should fix at least some of the performance regressions from PR27988. Differential Revision: http://reviews.llvm.org/D20983 llvm-svn: 271961 --- .../InstCombine/InstCombineVectorOps.cpp | 38 +++++++---- .../Transforms/InstCombine/vec_phi_extract.ll | 65 +++++++++++++++++-- .../LoopVectorize/PowerPC/vsx-tsvc-s173.ll | 2 +- 3 files changed, 87 insertions(+), 18 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index d92806f08de1..a76138756148 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -62,21 +62,31 @@ static bool cheapToScalarize(Value *V, bool isConstant) { return false; } -// If we have a PHI node with a vector type that has only 2 uses: feed +// If we have a PHI node with a vector type that is only used to feed // itself and be an operand of extractelement at a constant location, // try to replace the PHI of the vector type with a PHI of a scalar type. Instruction *InstCombiner::scalarizePHI(ExtractElementInst &EI, PHINode *PN) { - // Verify that the PHI node has exactly 2 uses. Otherwise return NULL. - if (!PN->hasNUses(2)) - return nullptr; + SmallVector Extracts; + // The users we want the PHI to have are: + // 1) The EI ExtractElement (we already know this) + // 2) Possibly more ExtractElements with the same index. + // 3) Another operand, which will feed back into the PHI. + Instruction *PHIUser = nullptr; + for (auto U : PN->users()) { + if (ExtractElementInst *EU = dyn_cast(U)) { + if (EI.getIndexOperand() == EU->getIndexOperand()) + Extracts.push_back(EU); + else + return nullptr; + } else if (!PHIUser) { + PHIUser = cast(U); + } else { + return nullptr; + } + } - // If so, it's known at this point that one operand is PHI and the other is - // an extractelement node. Find the PHI user that is not the extractelement - // node. - auto iu = PN->user_begin(); - Instruction *PHIUser = dyn_cast(*iu); - if (PHIUser == cast(&EI)) - PHIUser = cast(*(++iu)); + if (!PHIUser) + return nullptr; // Verify that this PHI user has one use, which is the PHI itself, // and that it is a binary operation which is cheap to scalarize. @@ -126,7 +136,11 @@ Instruction *InstCombiner::scalarizePHI(ExtractElementInst &EI, PHINode *PN) { scalarPHI->addIncoming(newEI, inBB); } } - return replaceInstUsesWith(EI, scalarPHI); + + for (auto E : Extracts) + replaceInstUsesWith(*E, scalarPHI); + + return &EI; } Instruction *InstCombiner::visitExtractElementInst(ExtractElementInst &EI) { diff --git a/llvm/test/Transforms/InstCombine/vec_phi_extract.ll b/llvm/test/Transforms/InstCombine/vec_phi_extract.ll index 1079dc9e7198..9beb98cb1af8 100644 --- a/llvm/test/Transforms/InstCombine/vec_phi_extract.ll +++ b/llvm/test/Transforms/InstCombine/vec_phi_extract.ll @@ -1,8 +1,9 @@ ; RUN: opt < %s -instcombine -S | FileCheck %s define void @f(i64 %val, i32 %limit, i32 *%ptr) { -;CHECK: %0 = trunc i64 -;CHECK: %1 = phi i32 +; CHECK-LABEL: @f +; CHECK: %0 = trunc i64 %val to i32 +; CHECK: %1 = phi i32 [ %0, %entry ], [ {{.*}}, %loop ] entry: %tempvector = insertelement <16 x i64> undef, i64 %val, i32 0 %vector = shufflevector <16 x i64> %tempvector, <16 x i64> undef, <16 x i32> zeroinitializer @@ -25,18 +26,72 @@ ret: ret void } +define void @copy(i64 %val, i32 %limit, i32 *%ptr) { +; CHECK-LABEL: @copy +; CHECK: %0 = trunc i64 %val to i32 +; CHECK: %1 = phi i32 [ %0, %entry ], [ {{.*}}, %loop ] +entry: + %tempvector = insertelement <16 x i64> undef, i64 %val, i32 0 + %vector = shufflevector <16 x i64> %tempvector, <16 x i64> undef, <16 x i32> zeroinitializer + %0 = add <16 x i64> %vector, + %1 = trunc <16 x i64> %0 to <16 x i32> + br label %loop + +loop: + %2 = phi <16 x i32> [ %1, %entry ], [ %inc, %loop ] + %elt = extractelement <16 x i32> %2, i32 0 + %eltcopy = extractelement <16 x i32> %2, i32 0 + %end = icmp ult i32 %elt, %limit + %3 = add i32 10, %eltcopy + %4 = sext i32 %elt to i64 + %5 = getelementptr i32, i32* %ptr, i64 %4 + store i32 %3, i32* %5 + %inc = add <16 x i32> %2, + br i1 %end, label %loop, label %ret + +ret: + ret void +} + +define void @nocopy(i64 %val, i32 %limit, i32 *%ptr) { +; CHECK-LABEL: @nocopy +; CHECK-NOT: phi i32 +; CHECK: phi <16 x i32> [ %1, %entry ], [ %inc, %loop ] +entry: + %tempvector = insertelement <16 x i64> undef, i64 %val, i32 0 + %vector = shufflevector <16 x i64> %tempvector, <16 x i64> undef, <16 x i32> zeroinitializer + %0 = add <16 x i64> %vector, + %1 = trunc <16 x i64> %0 to <16 x i32> + br label %loop + +loop: + %2 = phi <16 x i32> [ %1, %entry ], [ %inc, %loop ] + %elt = extractelement <16 x i32> %2, i32 0 + %eltcopy = extractelement <16 x i32> %2, i32 1 + %end = icmp ult i32 %elt, %limit + %3 = add i32 10, %eltcopy + %4 = sext i32 %elt to i64 + %5 = getelementptr i32, i32* %ptr, i64 %4 + store i32 %3, i32* %5 + %inc = add <16 x i32> %2, + br i1 %end, label %loop, label %ret + +ret: + ret void +} + define i1 @g(<3 x i32> %input_2) { -; CHECK: extractelement +; CHECK-LABEL: @g +; CHECK: extractelement <3 x i32> %input_2, i32 0 entry: br label %for.cond for.cond: -; CHECK: phi i32 %input_2.addr.0 = phi <3 x i32> [ %input_2, %entry ], [ %div45, %for.body ] %input_1.addr.1 = phi <3 x i32> [ undef, %entry ], [ %dec43, %for.body ] br i1 undef, label %for.end, label %for.body -; CHECK: extractelement +; CHECK: extractelement <3 x i32> %{{.*}}, i32 0 for.body: %dec43 = add <3 x i32> %input_1.addr.1, %sub44 = sub <3 x i32> , %dec43 diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/vsx-tsvc-s173.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/vsx-tsvc-s173.ll index 12a16018f589..fed186b9b675 100644 --- a/llvm/test/Transforms/LoopVectorize/PowerPC/vsx-tsvc-s173.ll +++ b/llvm/test/Transforms/LoopVectorize/PowerPC/vsx-tsvc-s173.ll @@ -43,7 +43,7 @@ for.end12: ; preds = %for.end, %entry ; CHECK-LABEL: @s173 ; CHECK: load <4 x float>, <4 x float>* -; CHECK: add nsw i64 %.lhs, 16000 +; CHECK: add nsw i64 %1, 16000 ; CHECK: ret i32 0 }