maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Revert "[VPlan] Add VPReductionPHIRecipe (NFC)." and follow-ups 

This reverts commit 3fed6d443f,
bbcbf21ae6 and
6c3451cd76.

The changes causing build failures with certain configurations, e.g.
https://lab.llvm.org/buildbot/#/builders/67/builds/3365/steps/6/logs/stdio

    lib/libLLVMVectorize.a(LoopVectorize.cpp.o): In function `llvm::VPRecipeBuilder::tryToCreateWidenRecipe(llvm::Instruction*, llvm::ArrayRef<llvm::VPValue*>, llvm::VFRange&, std::unique_ptr<llvm::VPlan, std::default_delete<llvm::VPlan> >&) [clone .localalias.8]':
    LoopVectorize.cpp:(.text._ZN4llvm15VPRecipeBuilder22tryToCreateWidenRecipeEPNS_11InstructionENS_8ArrayRefIPNS_7VPValueEEERNS_7VFRangeERSt10unique_ptrINS_5VPlanESt14default_deleteISA_EE+0x63b): undefined reference to `vtable for llvm::VPReductionPHIRecipe'
    collect2: error: ld returned 1 exit status
This commit is contained in:
Florian Hahn 2021-07-06 12:06:54 +01:00
parent 3fed6d443f
commit 706bbfb35b
No known key found for this signature in database
GPG Key ID: 61D7554B5CECDC0D
7 changed files with 127 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -503,11 +503,11 @@ public:
unsigned UF, ElementCount VF, bool IsPtrLoopInvariant,
SmallBitVector &IsIndexLoopInvariant, VPTransformState &State);
/// Vectorize a single first-order recurrence or pointer induction PHINode in
/// a block. This method handles the induction variable canonicalization. It
/// supports both VF = 1 for unrolled loops and arbitrary length vectors.
void widenPHIInstruction(Instruction *PN, VPWidenPHIRecipe *PhiR,
VPTransformState &State);
/// Vectorize a single PHINode in a block. This method handles the induction
/// variable canonicalization. It supports both VF = 1 for unrolled loops and
/// arbitrary length vectors.
void widenPHIInstruction(Instruction *PN, RecurrenceDescriptor *RdxDesc,
VPWidenPHIRecipe *PhiR, VPTransformState &State);
/// A helper function to scalarize a single Instruction in the innermost loop.
/// Generates a sequence of scalar instances for each lane between \p MinLane
@ -596,7 +596,7 @@ protected:
/// Fix a reduction cross-iteration phi. This is the second phase of
/// vectorizing this phi node.
void fixReduction(VPReductionPHIRecipe *Phi, VPTransformState &State);
void fixReduction(VPWidenPHIRecipe *Phi, VPTransformState &State);
/// Clear NSW/NUW flags from reduction instructions if necessary.
void clearReductionWrapFlags(const RecurrenceDescriptor &RdxDesc,
@ -4135,8 +4135,8 @@ void InnerLoopVectorizer::fixCrossIterationPHIs(VPTransformState &State) {
if (!PhiR)
continue;
auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(PhiR)) {
fixReduction(ReductionPhi, State);
if (PhiR->getRecurrenceDescriptor()) {
fixReduction(PhiR, State);
} else if (Legal->isFirstOrderRecurrence(OrigPhi))
fixFirstOrderRecurrence(PhiR, State);
}
@ -4320,18 +4320,19 @@ void InnerLoopVectorizer::fixFirstOrderRecurrence(VPWidenPHIRecipe *PhiR,
LCSSAPhi.addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock);
}
void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
void InnerLoopVectorizer::fixReduction(VPWidenPHIRecipe *PhiR,
VPTransformState &State) {
PHINode *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
// Get it's reduction variable descriptor.
assert(Legal->isReductionVariable(OrigPhi) &&
"Unable to find the reduction variable");
const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
const RecurrenceDescriptor &RdxDesc = *PhiR->getRecurrenceDescriptor();
RecurKind RK = RdxDesc.getRecurrenceKind();
TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
setDebugLocFromInst(ReductionStartValue);
bool IsInLoopReductionPhi = Cost->isInLoopReduction(OrigPhi);
VPValue *LoopExitInstDef = State.Plan->getVPValue(LoopExitInst);
// This is the vector-clone of the value that leaves the loop.
@ -4346,11 +4347,14 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
// any loop invariant values.
BasicBlock *VectorLoopLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();
unsigned LastPartForNewPhi = PhiR->isOrdered() ? 1 : UF;
for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
bool IsOrdered = IsInLoopReductionPhi && Cost->useOrderedReductions(RdxDesc);
for (unsigned Part = 0; Part < UF; ++Part) {
if (IsOrdered && Part > 0)
break;
Value *VecRdxPhi = State.get(PhiR->getVPSingleValue(), Part);
Value *Val = State.get(PhiR->getBackedgeValue(), Part);
if (PhiR->isOrdered())
if (IsOrdered)
Val = State.get(PhiR->getBackedgeValue(), UF - 1);
cast<PHINode>(VecRdxPhi)->addIncoming(Val, VectorLoopLatch);
@ -4369,7 +4373,7 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
// a Select choosing between the vectorized LoopExitInst and vectorized Phi,
// instead of the former. For an inloop reduction the reduction will already
// be predicated, and does not need to be handled here.
if (Cost->foldTailByMasking() && !PhiR->isInLoop()) {
if (Cost->foldTailByMasking() && !IsInLoopReductionPhi) {
for (unsigned Part = 0; Part < UF; ++Part) {
Value *VecLoopExitInst = State.get(LoopExitInstDef, Part);
Value *Sel = nullptr;
@ -4404,7 +4408,7 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
// then extend the loop exit value to enable InstCombine to evaluate the
// entire expression in the smaller type.
if (VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
assert(!IsInLoopReductionPhi && "Unexpected truncated inloop reduction!");
Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);
Builder.SetInsertPoint(
LI->getLoopFor(LoopVectorBody)->getLoopLatch()->getTerminator());
@ -4442,7 +4446,7 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
// terminate on this line. This is the easiest way to ensure we don't
// accidentally cause an extra step back into the loop while debugging.
setDebugLocFromInst(LoopMiddleBlock->getTerminator());
if (PhiR->isOrdered())
if (IsOrdered)
ReducedPartRdx = State.get(LoopExitInstDef, UF - 1);
else {
// Floating-point operations should have some FMF to enable the reduction.
@ -4461,7 +4465,7 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
// Create the reduction after the loop. Note that inloop reductions create the
// target reduction in the loop using a Reduction recipe.
if (VF.isVector() && !PhiR->isInLoop()) {
if (VF.isVector() && !IsInLoopReductionPhi) {
ReducedPartRdx =
createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx);
// If the reduction can be performed in a smaller type, we need to extend
@ -4725,6 +4729,7 @@ void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPValue *VPDef,
}
void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
RecurrenceDescriptor *RdxDesc,
VPWidenPHIRecipe *PhiR,
VPTransformState &State) {
PHINode *P = cast<PHINode>(PN);
@ -4750,21 +4755,68 @@ void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
// Phi nodes have cycles, so we need to vectorize them in two stages. This is
// stage #1: We create a new vector PHI node with no incoming edges. We'll use
// this value when we vectorize all of the instructions that use the PHI.
if (Legal->isFirstOrderRecurrence(P)) {
Type *VecTy = State.VF.isScalar()
? PN->getType()
: VectorType::get(PN->getType(), State.VF);
if (RdxDesc || Legal->isFirstOrderRecurrence(P)) {
bool ScalarPHI =
(State.VF.isScalar()) || Cost->isInLoopReduction(cast<PHINode>(PN));
Type *VecTy =
ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
for (unsigned Part = 0; Part < State.UF; ++Part) {
bool IsOrdered = Cost->isInLoopReduction(cast<PHINode>(PN)) &&
Cost->useOrderedReductions(*RdxDesc);
unsigned LastPartForNewPhi = IsOrdered ? 1 : State.UF;
for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
Value *EntryPart = PHINode::Create(
VecTy, 2, "vec.phi", &*LoopVectorBody->getFirstInsertionPt());
State.set(PhiR, EntryPart, Part);
}
if (Legal->isFirstOrderRecurrence(P))
return;
VPValue *StartVPV = PhiR->getStartValue();
Value *StartV = StartVPV->getLiveInIRValue();
Value *Iden = nullptr;
assert(Legal->isReductionVariable(P) && StartV &&
"RdxDesc should only be set for reduction variables; in that case "
"a StartV is also required");
RecurKind RK = RdxDesc->getRecurrenceKind();
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
// MinMax reduction have the start value as their identify.
if (ScalarPHI) {
Iden = StartV;
} else {
IRBuilderBase::InsertPointGuard IPBuilder(Builder);
Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
StartV = Iden =
Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
}
} else {
Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity(
RK, VecTy->getScalarType(), RdxDesc->getFastMathFlags());
Iden = IdenC;
if (!ScalarPHI) {
Iden = ConstantVector::getSplat(State.VF, IdenC);
IRBuilderBase::InsertPointGuard IPBuilder(Builder);
Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
Constant *Zero = Builder.getInt32(0);
StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
}
}
for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
Value *EntryPart = State.get(PhiR, Part);
// Make sure to add the reduction start value only to the
// first unroll part.
Value *StartVal = (Part == 0) ? StartV : Iden;
cast<PHINode>(EntryPart)->addIncoming(StartVal, LoopVectorPreHeader);
}
return;
}
assert(!Legal->isReductionVariable(P) &&
"reductions should be handled elsewhere");
"reductions should be handled above");
setDebugLocFromInst(P);
@ -8926,9 +8978,7 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi];
assert(RdxDesc.getRecurrenceStartValue() ==
Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
PhiRecipe = new VPReductionPHIRecipe(Phi, RdxDesc, *StartV,
CM.isInLoopReduction(Phi),
CM.useOrderedReductions(RdxDesc));
PhiRecipe = new VPWidenPHIRecipe(Phi, RdxDesc, *StartV);
} else {
PhiRecipe = new VPWidenPHIRecipe(Phi, *StartV);
}
@ -9443,8 +9493,8 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
}
void VPWidenPHIRecipe::execute(VPTransformState &State) {
State.ILV->widenPHIInstruction(cast<PHINode>(getUnderlyingValue()), this,
State);
State.ILV->widenPHIInstruction(cast<PHINode>(getUnderlyingValue()), RdxDesc,
this, State);
}
void VPBlendRecipe::execute(VPTransformState &State) {

69
llvm/lib/Transforms/Vectorize/VPlan.cpp
View File

@ -763,7 +763,6 @@ void VPlan::execute(VPTransformState *State) {
State->VPValue2Value[Entry.second] = Entry.first;
BasicBlock *VectorPreHeaderBB = State->CFG.PrevBB;
State->CFG.VectorPreHeader = VectorPreHeaderBB;
BasicBlock *VectorHeaderBB = VectorPreHeaderBB->getSingleSuccessor();
assert(VectorHeaderBB && "Loop preheader does not have a single successor.");
@ -1115,74 +1114,6 @@ void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent,
printOperands(O, SlotTracker);
}
void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "WIDEN-REDUCTION-PHI ";
printAsOperand(O, SlotTracker);
O << " = phi ";
printOperands(O, SlotTracker);
}
void VPReductionPHIRecipe::execute(VPTransformState &State) {
PHINode *PN = cast<PHINode>(getUnderlyingValue());
auto &Builder = State.Builder;
// In order to support recurrences we need to be able to vectorize Phi nodes.
// Phi nodes have cycles, so we need to vectorize them in two stages. This is
// stage #1: We create a new vector PHI node with no incoming edges. We'll use
// this value when we vectorize all of the instructions that use the PHI.
bool ScalarPHI = State.VF.isScalar() || IsInLoop;
Type *VecTy =
ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
BasicBlock *HeaderBB = State.CFG.PrevBB;
assert(State.LI->getLoopFor(HeaderBB)->getHeader() == HeaderBB &&
"recipe must be in the vector loop header");
unsigned LastPartForNewPhi = isOrdered() ? 1 : State.UF;
for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
Value *EntryPart =
PHINode::Create(VecTy, 2, "vec.phi", &*HeaderBB->getFirstInsertionPt());
State.set(this, EntryPart, Part);
}
VPValue *StartVPV = getStartValue();
Value *StartV = StartVPV->getLiveInIRValue();
Value *Iden = nullptr;
RecurKind RK = RdxDesc.getRecurrenceKind();
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
// MinMax reduction have the start value as their identify.
if (ScalarPHI) {
Iden = StartV;
} else {
IRBuilderBase::InsertPointGuard IPBuilder(Builder);
Builder.SetInsertPoint(State.CFG.VectorPreHeader->getTerminator());
StartV = Iden =
Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
}
} else {
Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity(
RK, VecTy->getScalarType(), RdxDesc.getFastMathFlags());
Iden = IdenC;
if (!ScalarPHI) {
Iden = ConstantVector::getSplat(State.VF, IdenC);
IRBuilderBase::InsertPointGuard IPBuilder(Builder);
Builder.SetInsertPoint(State.CFG.VectorPreHeader->getTerminator());
Constant *Zero = Builder.getInt32(0);
StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
}
}
for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
Value *EntryPart = State.get(this, Part);
// Make sure to add the reduction start value only to the
// first unroll part.
Value *StartVal = (Part == 0) ? StartV : Iden;
cast<PHINode>(EntryPart)->addIncoming(StartVal, State.CFG.VectorPreHeader);
}
}
void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "BLEND ";

118
llvm/lib/Transforms/Vectorize/VPlan.h
View File

@ -306,13 +306,6 @@ struct VPTransformState {
/// BasicBlock, used for placing the newly created BasicBlocks.
BasicBlock *LastBB = nullptr;
/// The IR BasicBlock that is the preheader of the vector loop in the output
/// IR.
/// FIXME: The vector preheader should also be modeled in VPlan, so any code
/// that needs to be added to the preheader gets directly generated by
/// VPlan. There should be no need to manage a pointer to the IR BasicBlock.
BasicBlock *VectorPreHeader = nullptr;
/// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case
/// of replication, maps the BasicBlock of the last replica created.
SmallDenseMap<VPBasicBlock *, BasicBlock *> VPBB2IRBB;
@ -737,7 +730,8 @@ public:
/// Returns true for PHI-like recipes.
bool isPhi() const {
return getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC;
return getVPDefID() == VPWidenIntOrFpInductionSC || getVPDefID() == VPWidenPHISC ||
getVPDefID() == VPPredInstPHISC || getVPDefID() == VPWidenCanonicalIVSC;
}
/// Returns true if the recipe may read from memory.
@ -1050,48 +1044,54 @@ public:
}
};
/// A recipe for handling first order recurrences and pointer inductions. For
/// first-order recurrences, the start value is the first operand of the recipe
/// and the incoming value from the backedge is the second operand. It also
/// serves as base class for VPReductionPHIRecipe. In the VPlan native path, all
/// incoming VPValues & VPBasicBlock pairs are managed in the recipe directly.
/// A recipe for handling all phi nodes except for integer and FP inductions.
/// For reduction PHIs, RdxDesc must point to the corresponding recurrence
/// descriptor. For reductions and first-order recurrences, the start value is
/// the first operand of the recipe and the incoming value from the backedge is
/// the second operand. In the VPlan native path, all incoming VPValues &
/// VPBasicBlock pairs are managed in the recipe directly.
class VPWidenPHIRecipe : public VPRecipeBase, public VPValue {
/// Descriptor for a reduction PHI.
RecurrenceDescriptor *RdxDesc = nullptr;
/// List of incoming blocks. Only used in the VPlan native path.
SmallVector<VPBasicBlock *, 2> IncomingBlocks;
protected:
VPWidenPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi)
: VPRecipeBase(VPDefID, {}), VPValue(VPVID, Phi, this) {}
public:
/// Create a VPWidenPHIRecipe for \p Phi
VPWidenPHIRecipe(PHINode *Phi)
: VPWidenPHIRecipe(VPVWidenPHISC, VPWidenPHISC, Phi) {}
/// Create a new VPWidenPHIRecipe for the reduction \p Phi described by \p
/// RdxDesc.
VPWidenPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc, VPValue &Start)
: VPWidenPHIRecipe(Phi) {
this->RdxDesc = &RdxDesc;
addOperand(&Start);
}
/// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start.
VPWidenPHIRecipe(PHINode *Phi, VPValue &Start) : VPWidenPHIRecipe(Phi) {
addOperand(&Start);
}
virtual ~VPWidenPHIRecipe() override = default;
/// Create a VPWidenPHIRecipe for \p Phi
VPWidenPHIRecipe(PHINode *Phi)
: VPRecipeBase(VPWidenPHISC, {}),
VPValue(VPValue::VPVWidenPHISC, Phi, this) {}
~VPWidenPHIRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *B) {
return B->getVPDefID() == VPRecipeBase::VPWidenPHISC ||
B->getVPDefID() == VPRecipeBase::VPReductionPHISC;
static inline bool classof(const VPDef *D) {
return D->getVPDefID() == VPRecipeBase::VPWidenPHISC;
}
static inline bool classof(const VPValue *V) {
return V->getVPValueID() == VPValue::VPVWidenPHISC ||
V->getVPValueID() == VPValue::VPVReductionPHISC;
return V->getVPValueID() == VPValue::VPVWidenPHISC;
}
/// Generate the phi/select nodes.
virtual void execute(VPTransformState &State) override;
void execute(VPTransformState &State) override;
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
virtual void print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const override;
void print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const override;
#endif
/// Returns the start value of the phi, if it is a reduction or first-order
@ -1117,62 +1117,8 @@ public:
/// Returns the \p I th incoming VPBasicBlock.
VPBasicBlock *getIncomingBlock(unsigned I) { return IncomingBlocks[I]; }
};
/// A recipe for handling reduction phis. The start value is the first operand
/// of the recipe and the incoming value from the backedge is the second
/// operand.
class VPReductionPHIRecipe : public VPWidenPHIRecipe {
/// Descriptor for the reduction.
RecurrenceDescriptor &RdxDesc;
/// The phi is part of an in-loop reduction.
bool IsInLoop;
/// The phi is part of an ordered reduction. Requires IsInLoop to be true.
bool IsOrdered;
public:
/// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p
/// RdxDesc.
VPReductionPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc,
VPValue &Start, bool IsInLoop = false,
bool IsOrdered = false)
: VPWidenPHIRecipe(VPVReductionPHISC, VPReductionPHISC, Phi),
RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) {
assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
addOperand(&Start);
}
~VPReductionPHIRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *R) {
return R->getVPDefID() == VPRecipeBase::VPReductionPHISC;
}
static inline bool classof(const VPValue *V) {
return V->getVPValueID() == VPValue::VPVReductionPHISC;
}
static inline bool classof(const VPWidenPHIRecipe *R) {
return R->getVPDefID() == VPRecipeBase::VPReductionPHISC;
}
/// Generate the phi/select nodes.
void execute(VPTransformState &State) override;
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
void print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const override;
#endif
RecurrenceDescriptor &getRecurrenceDescriptor() { return RdxDesc; }
/// Returns true, if the phi is part of an ordered reduction.
bool isOrdered() const { return IsOrdered; }
/// Returns true, if the phi is part of an in-loop reduction.
bool isInLoop() const { return IsInLoop; }
RecurrenceDescriptor *getRecurrenceDescriptor() { return RdxDesc; }
};
/// A recipe for vectorizing a phi-node as a sequence of mask-based select
@ -1311,6 +1257,10 @@ public:
return V->getVPValueID() == VPValue::VPVReductionSC;
}
static inline bool classof(const VPDef *D) {
return D->getVPDefID() == VPRecipeBase::VPReductionSC;
}
/// Generate the reduction in the loop
void execute(VPTransformState &State) override;

3
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
View File

@ -234,7 +234,8 @@ bool VPlanTransforms::mergeReplicateRegions(VPlan &Plan) {
auto *UI = dyn_cast<VPRecipeBase>(U);
if (!UI)
continue;
if (isa<VPWidenPHIRecipe>(UI) && !isa<VPReductionPHIRecipe>(UI))
auto *PhiR = dyn_cast<VPWidenPHIRecipe>(UI);
if (PhiR && !PhiR->getRecurrenceDescriptor())
return true;
}
return false;

31
llvm/lib/Transforms/Vectorize/VPlanValue.h
View File

@ -90,22 +90,18 @@ public:
/// type identification.
enum {
VPValueSC,
VPVBlendSC,
VPVInstructionSC,
VPVMemoryInstructionSC,
VPVPredInstPHI,
VPVReductionSC,
VPVReplicateSC,
VPVWidenSC,
VPVWidenCallSC,
VPVWidenGEPSC,
VPVWidenSelectSC,
// Phi-like VPValues. Need to be kept together.
VPVBlendSC,
VPVWidenIntOrFpIndcutionSC,
VPVWidenPHISC,
VPVWidenCanonicalIVSC,
VPVWidenIntOrFpInductionSC,
VPVPredInstPHI,
VPVReductionPHISC,
VPVWidenSelectSC,
};
VPValue(Value *UV = nullptr, VPDef *Def = nullptr)
@ -318,26 +314,21 @@ public:
/// SubclassID field of the VPRecipeBase objects. They are used for concrete
/// type identification.
using VPRecipeTy = enum {
VPBlendSC,
VPBranchOnMaskSC,
VPInstructionSC,
VPInterleaveSC,
VPPredInstPHISC,
VPReductionSC,
VPReplicateSC,
VPWidenCallSC,
VPWidenGEPSC,
VPWidenMemoryInstructionSC,
VPWidenSC,
VPWidenSelectSC,
// Phi-like recipes. Need to be kept together.
VPBlendSC,
VPWidenPHISC,
VPWidenCanonicalIVSC,
VPWidenGEPSC,
VPWidenIntOrFpInductionSC,
VPPredInstPHISC,
VPReductionPHISC,
VPFirstPHISC = VPBlendSC,
VPLastPHISC = VPReductionPHISC,
VPWidenMemoryInstructionSC,
VPWidenPHISC,
VPWidenSC,
VPWidenSelectSC
};
VPDef(const unsigned char SC) : SubclassID(SC) {}

2
llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
View File

@ -143,7 +143,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, i32* %ptr) optsize
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next>
; CHECK-NEXT: WIDEN-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next>
; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%0>
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:

2
llvm/test/Transforms/LoopVectorize/vplan-printing.ll
View File

@ -79,7 +79,7 @@ define float @print_reduction(i64 %n, float* noalias %y) {
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
; CHECK-NEXT: WIDEN-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>
; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx>
; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + reduce.fadd (ir<%lv>)