maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[VPlan] Make VPRecipeBase inherit from VPDef. 

This patch makes VPRecipeBase a direct subclass of VPDef, moving the
SubclassID to VPDef.

Reviewed By: gilr

Differential Revision: https://reviews.llvm.org/D90564
This commit is contained in:
Florian Hahn 2020-12-21 11:12:40 +00:00
parent 6f45049fb6
commit f250892373
No known key found for this signature in database
GPG Key ID: 61D7554B5CECDC0D
4 changed files with 169 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
llvm/lib/Transforms/Vectorize/VPlan.cpp
View File

@ -120,50 +120,18 @@ VPUser *VPRecipeBase::toVPUser() {
} }
VPValue *VPRecipeBase::toVPValue() { VPValue *VPRecipeBase::toVPValue() {
if (getNumDefinedValues() == 1)
return getVPValue();
if (auto *V = dyn_cast<VPInstruction>(this)) if (auto *V = dyn_cast<VPInstruction>(this))
return V; return V;
if (auto *V = dyn_cast<VPReductionRecipe>(this))
return V;
if (auto *V = dyn_cast<VPWidenMemoryInstructionRecipe>(this)) {
if (!V->isStore())
return V->getVPValue();
else
return nullptr;
}
if (auto *V = dyn_cast<VPWidenCallRecipe>(this))
return V;
if (auto *V = dyn_cast<VPWidenSelectRecipe>(this))
return V;
if (auto *V = dyn_cast<VPWidenGEPRecipe>(this))
return V;
if (auto *V = dyn_cast<VPWidenRecipe>(this))
return V;
if (auto *V = dyn_cast<VPReplicateRecipe>(this))
return V;
return nullptr; return nullptr;
} }
const VPValue *VPRecipeBase::toVPValue() const { const VPValue *VPRecipeBase::toVPValue() const {
if (getNumDefinedValues() == 1)
return getVPValue();
if (auto *V = dyn_cast<VPInstruction>(this)) if (auto *V = dyn_cast<VPInstruction>(this))
return V; return V;
if (auto *V = dyn_cast<VPReductionRecipe>(this))
return V;
if (auto *V = dyn_cast<VPWidenMemoryInstructionRecipe>(this)) {
if (!V->isStore())
return V->getVPValue();
else
return nullptr;
}
if (auto *V = dyn_cast<VPWidenCallRecipe>(this))
return V;
if (auto *V = dyn_cast<VPWidenSelectRecipe>(this))
return V;
if (auto *V = dyn_cast<VPWidenGEPRecipe>(this))
return V;
if (auto *V = dyn_cast<VPWidenRecipe>(this))
return V;
if (auto *V = dyn_cast<VPReplicateRecipe>(this))
return V;
return nullptr; return nullptr;
} }

197
llvm/lib/Transforms/Vectorize/VPlan.h
View File

@ -620,47 +620,23 @@ public:
}; };
/// VPRecipeBase is a base class modeling a sequence of one or more output IR /// VPRecipeBase is a base class modeling a sequence of one or more output IR
/// instructions. /// instructions. VPRecipeBase owns the the VPValues it defines through VPDef
class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock> { /// and is responsible for deleting its defined values. Single-value
/// VPRecipeBases that also inherit from VPValue must make sure to inherit from
/// VPRecipeBase before VPValue.
class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
public VPDef {
friend VPBasicBlock; friend VPBasicBlock;
friend class VPBlockUtils; friend class VPBlockUtils;
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
/// Each VPRecipe belongs to a single VPBasicBlock. /// Each VPRecipe belongs to a single VPBasicBlock.
VPBasicBlock *Parent = nullptr; VPBasicBlock *Parent = nullptr;
public: public:
/// An enumeration for keeping track of the concrete subclass of VPRecipeBase VPRecipeBase(const unsigned char SC) : VPDef(SC) {}
/// that is actually instantiated. Values of this enumeration are kept in the
/// SubclassID field of the VPRecipeBase objects. They are used for concrete
/// type identification.
using VPRecipeTy = enum {
VPBlendSC,
VPBranchOnMaskSC,
VPInstructionSC,
VPInterleaveSC,
VPPredInstPHISC,
VPReductionSC,
VPReplicateSC,
VPWidenCallSC,
VPWidenCanonicalIVSC,
VPWidenGEPSC,
VPWidenIntOrFpInductionSC,
VPWidenMemoryInstructionSC,
VPWidenPHISC,
VPWidenSC,
VPWidenSelectSC
};
VPRecipeBase(const unsigned char SC) : SubclassID(SC) {}
virtual ~VPRecipeBase() = default; virtual ~VPRecipeBase() = default;
/// \return an ID for the concrete type of this object.
/// This is used to implement the classof checks. This should not be used
/// for any other purpose, as the values may change as LLVM evolves.
unsigned getVPRecipeID() const { return SubclassID; }
/// \return the VPBasicBlock which this VPRecipe belongs to. /// \return the VPBasicBlock which this VPRecipe belongs to.
VPBasicBlock *getParent() { return Parent; } VPBasicBlock *getParent() { return Parent; }
const VPBasicBlock *getParent() const { return Parent; } const VPBasicBlock *getParent() const { return Parent; }
@ -718,27 +694,33 @@ public:
return cast_or_null<Instruction>(VPV->getUnderlyingValue()); return cast_or_null<Instruction>(VPV->getUnderlyingValue());
return nullptr; return nullptr;
} }
/// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPDef *D) {
// All VPDefs are also VPRecipeBases.
return true;
}
}; };
inline bool VPUser::classof(const VPRecipeBase *Recipe) { inline bool VPUser::classof(const VPDef *Def) {
return Recipe->getVPRecipeID() == VPRecipeBase::VPInstructionSC || return Def->getVPDefID() == VPRecipeBase::VPInstructionSC ||
Recipe->getVPRecipeID() == VPRecipeBase::VPWidenSC || Def->getVPDefID() == VPRecipeBase::VPWidenSC ||
Recipe->getVPRecipeID() == VPRecipeBase::VPWidenCallSC || Def->getVPDefID() == VPRecipeBase::VPWidenCallSC ||
Recipe->getVPRecipeID() == VPRecipeBase::VPWidenSelectSC || Def->getVPDefID() == VPRecipeBase::VPWidenSelectSC ||
Recipe->getVPRecipeID() == VPRecipeBase::VPWidenGEPSC || Def->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
Recipe->getVPRecipeID() == VPRecipeBase::VPBlendSC || Def->getVPDefID() == VPRecipeBase::VPBlendSC ||
Recipe->getVPRecipeID() == VPRecipeBase::VPInterleaveSC || Def->getVPDefID() == VPRecipeBase::VPInterleaveSC ||
Recipe->getVPRecipeID() == VPRecipeBase::VPReplicateSC || Def->getVPDefID() == VPRecipeBase::VPReplicateSC ||
Recipe->getVPRecipeID() == VPRecipeBase::VPReductionSC || Def->getVPDefID() == VPRecipeBase::VPReductionSC ||
Recipe->getVPRecipeID() == VPRecipeBase::VPBranchOnMaskSC || Def->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC ||
Recipe->getVPRecipeID() == VPRecipeBase::VPWidenMemoryInstructionSC; Def->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC;
} }
/// This is a concrete Recipe that models a single VPlan-level instruction. /// This is a concrete Recipe that models a single VPlan-level instruction.
/// While as any Recipe it may generate a sequence of IR instructions when /// While as any Recipe it may generate a sequence of IR instructions when
/// executed, these instructions would always form a single-def expression as /// executed, these instructions would always form a single-def expression as
/// the VPInstruction is also a single def-use vertex. /// the VPInstruction is also a single def-use vertex.
class VPInstruction : public VPUser, public VPValue, public VPRecipeBase { class VPInstruction : public VPValue, public VPUser, public VPRecipeBase {
friend class VPlanSlp; friend class VPlanSlp;
public: public:
@ -764,9 +746,16 @@ protected:
public: public:
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands) VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands)
: VPUser(Operands), VPValue(VPValue::VPVInstructionSC), : VPValue(VPValue::VPVInstructionSC), VPUser(Operands),
VPRecipeBase(VPRecipeBase::VPInstructionSC), Opcode(Opcode) {} VPRecipeBase(VPRecipeBase::VPInstructionSC), Opcode(Opcode) {}
VPInstruction(unsigned Opcode, ArrayRef<VPInstruction *> Operands)
: VPValue(VPValue::VPVInstructionSC), VPUser({}),
VPRecipeBase(VPRecipeBase::VPInstructionSC), Opcode(Opcode) {
for (auto *I : Operands)
addOperand(I->getVPValue());
}
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands) VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands)
: VPInstruction(Opcode, ArrayRef<VPValue *>(Operands)) {} : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands)) {}
@ -781,8 +770,8 @@ public:
} }
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *R) { static inline bool classof(const VPDef *R) {
return R->getVPRecipeID() == VPRecipeBase::VPInstructionSC; return R->getVPDefID() == VPRecipeBase::VPInstructionSC;
} }
unsigned getOpcode() const { return Opcode; } unsigned getOpcode() const { return Opcode; }
@ -836,14 +825,14 @@ class VPWidenRecipe : public VPRecipeBase, public VPValue, public VPUser {
public: public:
template <typename IterT> template <typename IterT>
VPWidenRecipe(Instruction &I, iterator_range<IterT> Operands) VPWidenRecipe(Instruction &I, iterator_range<IterT> Operands)
: VPRecipeBase(VPRecipeBase::VPWidenSC), VPValue(VPValue::VPVWidenSC, &I), : VPRecipeBase(VPRecipeBase::VPWidenSC),
VPUser(Operands) {} VPValue(VPValue::VPVWidenSC, &I, this), VPUser(Operands) {}
~VPWidenRecipe() override = default; ~VPWidenRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *V) { static inline bool classof(const VPDef *D) {
return V->getVPRecipeID() == VPRecipeBase::VPWidenSC; return D->getVPDefID() == VPRecipeBase::VPWidenSC;
} }
static inline bool classof(const VPValue *V) { static inline bool classof(const VPValue *V) {
return V->getVPValueID() == VPValue::VPVWidenSC; return V->getVPValueID() == VPValue::VPVWidenSC;
@ -858,10 +847,7 @@ public:
}; };
/// A recipe for widening Call instructions. /// A recipe for widening Call instructions.
class VPWidenCallRecipe : public VPRecipeBase, class VPWidenCallRecipe : public VPRecipeBase, public VPUser, public VPValue {
public VPDef,
public VPUser,
public VPValue {
public: public:
template <typename IterT> template <typename IterT>
@ -872,8 +858,8 @@ public:
~VPWidenCallRecipe() override = default; ~VPWidenCallRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *V) { static inline bool classof(const VPDef *D) {
return V->getVPRecipeID() == VPRecipeBase::VPWidenCallSC; return D->getVPDefID() == VPRecipeBase::VPWidenCallSC;
} }
/// Produce a widened version of the call instruction. /// Produce a widened version of the call instruction.
@ -885,10 +871,7 @@ public:
}; };
/// A recipe for widening select instructions. /// A recipe for widening select instructions.
class VPWidenSelectRecipe : public VPRecipeBase, class VPWidenSelectRecipe : public VPRecipeBase, public VPUser, public VPValue {
public VPDef,
public VPUser,
public VPValue {
/// Is the condition of the select loop invariant? /// Is the condition of the select loop invariant?
bool InvariantCond; bool InvariantCond;
@ -904,8 +887,8 @@ public:
~VPWidenSelectRecipe() override = default; ~VPWidenSelectRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *V) { static inline bool classof(const VPDef *D) {
return V->getVPRecipeID() == VPRecipeBase::VPWidenSelectSC; return D->getVPDefID() == VPRecipeBase::VPWidenSelectSC;
} }
/// Produce a widened version of the select instruction. /// Produce a widened version of the select instruction.
@ -918,7 +901,6 @@ public:
/// A recipe for handling GEP instructions. /// A recipe for handling GEP instructions.
class VPWidenGEPRecipe : public VPRecipeBase, class VPWidenGEPRecipe : public VPRecipeBase,
public VPDef,
public VPUser, public VPUser,
public VPValue { public VPValue {
bool IsPtrLoopInvariant; bool IsPtrLoopInvariant;
@ -945,8 +927,8 @@ public:
~VPWidenGEPRecipe() override = default; ~VPWidenGEPRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *V) { static inline bool classof(const VPDef *D) {
return V->getVPRecipeID() == VPRecipeBase::VPWidenGEPSC; return D->getVPDefID() == VPRecipeBase::VPWidenGEPSC;
} }
/// Generate the gep nodes. /// Generate the gep nodes.
@ -965,12 +947,17 @@ class VPWidenIntOrFpInductionRecipe : public VPRecipeBase {
public: public:
VPWidenIntOrFpInductionRecipe(PHINode *IV, TruncInst *Trunc = nullptr) VPWidenIntOrFpInductionRecipe(PHINode *IV, TruncInst *Trunc = nullptr)
: VPRecipeBase(VPWidenIntOrFpInductionSC), IV(IV), Trunc(Trunc) {} : VPRecipeBase(VPWidenIntOrFpInductionSC), IV(IV), Trunc(Trunc) {
if (Trunc)
new VPValue(Trunc, this);
else
new VPValue(IV, this);
}
~VPWidenIntOrFpInductionRecipe() override = default; ~VPWidenIntOrFpInductionRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *V) { static inline bool classof(const VPDef *D) {
return V->getVPRecipeID() == VPRecipeBase::VPWidenIntOrFpInductionSC; return D->getVPDefID() == VPRecipeBase::VPWidenIntOrFpInductionSC;
} }
/// Generate the vectorized and scalarized versions of the phi node as /// Generate the vectorized and scalarized versions of the phi node as
@ -987,12 +974,14 @@ class VPWidenPHIRecipe : public VPRecipeBase {
PHINode *Phi; PHINode *Phi;
public: public:
VPWidenPHIRecipe(PHINode *Phi) : VPRecipeBase(VPWidenPHISC), Phi(Phi) {} VPWidenPHIRecipe(PHINode *Phi) : VPRecipeBase(VPWidenPHISC), Phi(Phi) {
new VPValue(Phi, this);
}
~VPWidenPHIRecipe() override = default; ~VPWidenPHIRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *V) { static inline bool classof(const VPDef *D) {
return V->getVPRecipeID() == VPRecipeBase::VPWidenPHISC; return D->getVPDefID() == VPRecipeBase::VPWidenPHISC;
} }
/// Generate the phi/select nodes. /// Generate the phi/select nodes.
@ -1014,6 +1003,7 @@ public:
/// might be incoming with a full mask for which there is no VPValue. /// might be incoming with a full mask for which there is no VPValue.
VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands) VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands)
: VPRecipeBase(VPBlendSC), VPUser(Operands), Phi(Phi) { : VPRecipeBase(VPBlendSC), VPUser(Operands), Phi(Phi) {
new VPValue(Phi, this);
assert(Operands.size() > 0 && assert(Operands.size() > 0 &&
((Operands.size() == 1) || (Operands.size() % 2 == 0)) && ((Operands.size() == 1) || (Operands.size() % 2 == 0)) &&
"Expected either a single incoming value or a positive even number " "Expected either a single incoming value or a positive even number "
@ -1021,8 +1011,8 @@ public:
} }
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *V) { static inline bool classof(const VPDef *D) {
return V->getVPRecipeID() == VPRecipeBase::VPBlendSC; return D->getVPDefID() == VPRecipeBase::VPBlendSC;
} }
/// Return the number of incoming values, taking into account that a single /// Return the number of incoming values, taking into account that a single
@ -1047,7 +1037,7 @@ public:
/// or stores into one wide load/store and shuffles. The first operand of a /// or stores into one wide load/store and shuffles. The first operand of a
/// VPInterleave recipe is the address, followed by the stored values, followed /// VPInterleave recipe is the address, followed by the stored values, followed
/// by an optional mask. /// by an optional mask.
class VPInterleaveRecipe : public VPRecipeBase, public VPDef, public VPUser { class VPInterleaveRecipe : public VPRecipeBase, public VPUser {
const InterleaveGroup<Instruction> *IG; const InterleaveGroup<Instruction> *IG;
bool HasMask = false; bool HasMask = false;
@ -1073,8 +1063,8 @@ public:
~VPInterleaveRecipe() override = default; ~VPInterleaveRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *V) { static inline bool classof(const VPDef *D) {
return V->getVPRecipeID() == VPRecipeBase::VPInterleaveSC; return D->getVPDefID() == VPRecipeBase::VPInterleaveSC;
} }
/// Return the address accessed by this recipe. /// Return the address accessed by this recipe.
@ -1111,7 +1101,7 @@ public:
/// A recipe to represent inloop reduction operations, performing a reduction on /// A recipe to represent inloop reduction operations, performing a reduction on
/// a vector operand into a scalar value, and adding the result to a chain. /// a vector operand into a scalar value, and adding the result to a chain.
/// The Operands are {ChainOp, VecOp, [Condition]}. /// The Operands are {ChainOp, VecOp, [Condition]}.
class VPReductionRecipe : public VPRecipeBase, public VPValue, public VPUser { class VPReductionRecipe : public VPRecipeBase, public VPUser, public VPValue {
/// The recurrence decriptor for the reduction in question. /// The recurrence decriptor for the reduction in question.
RecurrenceDescriptor *RdxDesc; RecurrenceDescriptor *RdxDesc;
/// Fast math flags to use for the resulting reduction operation. /// Fast math flags to use for the resulting reduction operation.
@ -1123,9 +1113,9 @@ public:
VPReductionRecipe(RecurrenceDescriptor *R, Instruction *I, VPValue *ChainOp, VPReductionRecipe(RecurrenceDescriptor *R, Instruction *I, VPValue *ChainOp,
VPValue *VecOp, VPValue *CondOp, bool NoNaN, VPValue *VecOp, VPValue *CondOp, bool NoNaN,
const TargetTransformInfo *TTI) const TargetTransformInfo *TTI)
: VPRecipeBase(VPRecipeBase::VPReductionSC), : VPRecipeBase(VPRecipeBase::VPReductionSC), VPUser({ChainOp, VecOp}),
VPValue(VPValue::VPVReductionSC, I), VPUser({ChainOp, VecOp}), VPValue(VPValue::VPVReductionSC, I, this), RdxDesc(R), NoNaN(NoNaN),
RdxDesc(R), NoNaN(NoNaN), TTI(TTI) { TTI(TTI) {
if (CondOp) if (CondOp)
addOperand(CondOp); addOperand(CondOp);
} }
@ -1136,8 +1126,9 @@ public:
static inline bool classof(const VPValue *V) { static inline bool classof(const VPValue *V) {
return V->getVPValueID() == VPValue::VPVReductionSC; return V->getVPValueID() == VPValue::VPVReductionSC;
} }
static inline bool classof(const VPRecipeBase *V) {
return V->getVPRecipeID() == VPRecipeBase::VPReductionSC; static inline bool classof(const VPDef *D) {
return D->getVPDefID() == VPRecipeBase::VPReductionSC;
} }
/// Generate the reduction in the loop /// Generate the reduction in the loop
@ -1176,7 +1167,7 @@ public:
VPReplicateRecipe(Instruction *I, iterator_range<IterT> Operands, VPReplicateRecipe(Instruction *I, iterator_range<IterT> Operands,
bool IsUniform, bool IsPredicated = false) bool IsUniform, bool IsPredicated = false)
: VPRecipeBase(VPReplicateSC), VPUser(Operands), : VPRecipeBase(VPReplicateSC), VPUser(Operands),
VPValue(VPVReplicateSC, I), IsUniform(IsUniform), VPValue(VPVReplicateSC, I, this), IsUniform(IsUniform),
IsPredicated(IsPredicated) { IsPredicated(IsPredicated) {
// Retain the previous behavior of predicateInstructions(), where an // Retain the previous behavior of predicateInstructions(), where an
// insert-element of a predicated instruction got hoisted into the // insert-element of a predicated instruction got hoisted into the
@ -1189,8 +1180,8 @@ public:
~VPReplicateRecipe() override = default; ~VPReplicateRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *V) { static inline bool classof(const VPDef *D) {
return V->getVPRecipeID() == VPRecipeBase::VPReplicateSC; return D->getVPDefID() == VPRecipeBase::VPReplicateSC;
} }
static inline bool classof(const VPValue *V) { static inline bool classof(const VPValue *V) {
@ -1220,8 +1211,8 @@ public:
} }
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *V) { static inline bool classof(const VPDef *D) {
return V->getVPRecipeID() == VPRecipeBase::VPBranchOnMaskSC; return D->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC;
} }
/// Generate the extraction of the appropriate bit from the block mask and the /// Generate the extraction of the appropriate bit from the block mask and the
@ -1259,12 +1250,14 @@ public:
/// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi /// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
/// nodes after merging back from a Branch-on-Mask. /// nodes after merging back from a Branch-on-Mask.
VPPredInstPHIRecipe(VPValue *PredV) VPPredInstPHIRecipe(VPValue *PredV)
: VPRecipeBase(VPPredInstPHISC), VPUser(PredV) {} : VPRecipeBase(VPPredInstPHISC), VPUser(PredV) {
new VPValue(VPValue::VPValueSC, PredV->getUnderlyingValue(), this);
}
~VPPredInstPHIRecipe() override = default; ~VPPredInstPHIRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *V) { static inline bool classof(const VPDef *D) {
return V->getVPRecipeID() == VPRecipeBase::VPPredInstPHISC; return D->getVPDefID() == VPRecipeBase::VPPredInstPHISC;
} }
/// Generates phi nodes for live-outs as needed to retain SSA form. /// Generates phi nodes for live-outs as needed to retain SSA form.
@ -1282,7 +1275,6 @@ public:
/// TODO: We currently execute only per-part unless a specific instance is /// TODO: We currently execute only per-part unless a specific instance is
/// provided. /// provided.
class VPWidenMemoryInstructionRecipe : public VPRecipeBase, class VPWidenMemoryInstructionRecipe : public VPRecipeBase,
public VPDef,
public VPUser { public VPUser {
Instruction &Ingredient; Instruction &Ingredient;
@ -1312,8 +1304,8 @@ public:
} }
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *V) { static inline bool classof(const VPDef *D) {
return V->getVPRecipeID() == VPRecipeBase::VPWidenMemoryInstructionSC; return D->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC;
} }
/// Return the address accessed by this recipe. /// Return the address accessed by this recipe.
@ -1347,21 +1339,16 @@ public:
/// A Recipe for widening the canonical induction variable of the vector loop. /// A Recipe for widening the canonical induction variable of the vector loop.
class VPWidenCanonicalIVRecipe : public VPRecipeBase { class VPWidenCanonicalIVRecipe : public VPRecipeBase {
/// A VPValue representing the canonical vector IV.
VPValue Val;
public: public:
VPWidenCanonicalIVRecipe() : VPRecipeBase(VPWidenCanonicalIVSC) {} VPWidenCanonicalIVRecipe() : VPRecipeBase(VPWidenCanonicalIVSC) {
new VPValue(nullptr, this);
}
~VPWidenCanonicalIVRecipe() override = default; ~VPWidenCanonicalIVRecipe() override = default;
/// Return the VPValue representing the canonical vector induction variable of
/// the vector loop.
const VPValue *getVPValue() const { return &Val; }
VPValue *getVPValue() { return &Val; }
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *V) { static inline bool classof(const VPDef *D) {
return V->getVPRecipeID() == VPRecipeBase::VPWidenCanonicalIVSC; return D->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
} }
/// Generate a canonical vector induction variable of the vector loop, with /// Generate a canonical vector induction variable of the vector loop, with

35
llvm/lib/Transforms/Vectorize/VPlanValue.h
View File

@ -45,6 +45,7 @@ class VPWidenMemoryInstructionRecipe;
class VPValue { class VPValue {
friend class VPBuilder; friend class VPBuilder;
friend class VPDef; friend class VPDef;
friend class VPInstruction;
friend struct VPlanTransforms; friend struct VPlanTransforms;
friend class VPBasicBlock; friend class VPBasicBlock;
friend class VPInterleavedAccessInfo; friend class VPInterleavedAccessInfo;
@ -236,7 +237,7 @@ public:
} }
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPRecipeBase *Recipe); static inline bool classof(const VPDef *Recipe);
}; };
/// This class augments a recipe with a set of VPValues defined by the recipe. /// This class augments a recipe with a set of VPValues defined by the recipe.
@ -247,6 +248,9 @@ public:
class VPDef { class VPDef {
friend class VPValue; friend class VPValue;
/// Subclass identifier (for isa/dyn_cast).
const unsigned char SubclassID;
/// The VPValues defined by this VPDef. /// The VPValues defined by this VPDef.
TinyPtrVector<VPValue *> DefinedValues; TinyPtrVector<VPValue *> DefinedValues;
@ -269,6 +273,30 @@ class VPDef {
} }
public: public:
/// An enumeration for keeping track of the concrete subclass of VPRecipeBase
/// that is actually instantiated. Values of this enumeration are kept in the
/// SubclassID field of the VPRecipeBase objects. They are used for concrete
/// type identification.
using VPRecipeTy = enum {
VPBlendSC,
VPBranchOnMaskSC,
VPInstructionSC,
VPInterleaveSC,
VPPredInstPHISC,
VPReductionSC,
VPReplicateSC,
VPWidenCallSC,
VPWidenCanonicalIVSC,
VPWidenGEPSC,
VPWidenIntOrFpInductionSC,
VPWidenMemoryInstructionSC,
VPWidenPHISC,
VPWidenSC,
VPWidenSelectSC
};
VPDef(const unsigned char SC) : SubclassID(SC) {}
virtual ~VPDef() { virtual ~VPDef() {
for (VPValue *D : make_early_inc_range(DefinedValues)) { for (VPValue *D : make_early_inc_range(DefinedValues)) {
assert(D->Def == this && assert(D->Def == this &&
@ -295,6 +323,11 @@ public:
/// Returns the number of values defined by the VPDef. /// Returns the number of values defined by the VPDef.
unsigned getNumDefinedValues() const { return DefinedValues.size(); } unsigned getNumDefinedValues() const { return DefinedValues.size(); }
/// \return an ID for the concrete type of this object.
/// This is used to implement the classof checks. This should not be used
/// for any other purpose, as the values may change as LLVM evolves.
unsigned getVPDefID() const { return SubclassID; }
}; };
class VPlan; class VPlan;

53
llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
View File

@ -365,7 +365,7 @@ TEST(VPRecipeTest, CastVPInstructionToVPUser) {
EXPECT_TRUE(isa<VPUser>(&Recipe)); EXPECT_TRUE(isa<VPUser>(&Recipe));
VPRecipeBase *BaseR = &Recipe; VPRecipeBase *BaseR = &Recipe;
EXPECT_TRUE(isa<VPUser>(BaseR)); EXPECT_TRUE(isa<VPUser>(BaseR));
EXPECT_EQ(&Recipe, BaseR->toVPUser()); EXPECT_EQ(&Recipe, BaseR);
} }
TEST(VPRecipeTest, CastVPWidenRecipeToVPUser) { TEST(VPRecipeTest, CastVPWidenRecipeToVPUser) {
@ -383,11 +383,11 @@ TEST(VPRecipeTest, CastVPWidenRecipeToVPUser) {
EXPECT_TRUE(isa<VPUser>(&WidenR)); EXPECT_TRUE(isa<VPUser>(&WidenR));
VPRecipeBase *WidenRBase = &WidenR; VPRecipeBase *WidenRBase = &WidenR;
EXPECT_TRUE(isa<VPUser>(WidenRBase)); EXPECT_TRUE(isa<VPUser>(WidenRBase));
EXPECT_EQ(&WidenR, WidenRBase->toVPUser()); EXPECT_EQ(&WidenR, WidenRBase);
delete AI; delete AI;
} }
TEST(VPRecipeTest, CastVPWidenCallRecipeToVPUser) { TEST(VPRecipeTest, CastVPWidenCallRecipeToVPUserAndVPDef) {
LLVMContext C; LLVMContext C;
IntegerType *Int32 = IntegerType::get(C, 32); IntegerType *Int32 = IntegerType::get(C, 32);
@ -402,11 +402,16 @@ TEST(VPRecipeTest, CastVPWidenCallRecipeToVPUser) {
EXPECT_TRUE(isa<VPUser>(&Recipe)); EXPECT_TRUE(isa<VPUser>(&Recipe));
VPRecipeBase *BaseR = &Recipe; VPRecipeBase *BaseR = &Recipe;
EXPECT_TRUE(isa<VPUser>(BaseR)); EXPECT_TRUE(isa<VPUser>(BaseR));
EXPECT_EQ(&Recipe, BaseR->toVPUser()); EXPECT_EQ(&Recipe, BaseR);
VPValue *VPV = &Recipe;
EXPECT_TRUE(isa<VPRecipeBase>(VPV->getDef()));
EXPECT_EQ(&Recipe, dyn_cast<VPRecipeBase>(VPV->getDef()));
delete Call; delete Call;
} }
TEST(VPRecipeTest, CastVPWidenSelectRecipeToVPUser) { TEST(VPRecipeTest, CastVPWidenSelectRecipeToVPUserAndVPDef) {
LLVMContext C; LLVMContext C;
IntegerType *Int1 = IntegerType::get(C, 1); IntegerType *Int1 = IntegerType::get(C, 1);
@ -425,11 +430,16 @@ TEST(VPRecipeTest, CastVPWidenSelectRecipeToVPUser) {
EXPECT_TRUE(isa<VPUser>(&WidenSelectR)); EXPECT_TRUE(isa<VPUser>(&WidenSelectR));
VPRecipeBase *BaseR = &WidenSelectR; VPRecipeBase *BaseR = &WidenSelectR;
EXPECT_TRUE(isa<VPUser>(BaseR)); EXPECT_TRUE(isa<VPUser>(BaseR));
EXPECT_EQ(&WidenSelectR, BaseR->toVPUser()); EXPECT_EQ(&WidenSelectR, BaseR);
VPValue *VPV = &WidenSelectR;
EXPECT_TRUE(isa<VPRecipeBase>(VPV->getDef()));
EXPECT_EQ(&WidenSelectR, dyn_cast<VPRecipeBase>(VPV->getDef()));
delete SelectI; delete SelectI;
} }
TEST(VPRecipeTest, CastVPWidenGEPRecipeToVPUser) { TEST(VPRecipeTest, CastVPWidenGEPRecipeToVPUserAndVPDef) {
LLVMContext C; LLVMContext C;
IntegerType *Int32 = IntegerType::get(C, 32); IntegerType *Int32 = IntegerType::get(C, 32);
@ -445,7 +455,12 @@ TEST(VPRecipeTest, CastVPWidenGEPRecipeToVPUser) {
EXPECT_TRUE(isa<VPUser>(&Recipe)); EXPECT_TRUE(isa<VPUser>(&Recipe));
VPRecipeBase *BaseR = &Recipe; VPRecipeBase *BaseR = &Recipe;
EXPECT_TRUE(isa<VPUser>(BaseR)); EXPECT_TRUE(isa<VPUser>(BaseR));
EXPECT_EQ(&Recipe, BaseR->toVPUser()); EXPECT_EQ(&Recipe, BaseR);
VPValue *VPV = &Recipe;
EXPECT_TRUE(isa<VPRecipeBase>(VPV->getDef()));
EXPECT_EQ(&Recipe, dyn_cast<VPRecipeBase>(VPV->getDef()));
delete GEP; delete GEP;
} }
@ -476,7 +491,7 @@ TEST(VPRecipeTest, CastVPInterleaveRecipeToVPUser) {
EXPECT_TRUE(isa<VPUser>(&Recipe)); EXPECT_TRUE(isa<VPUser>(&Recipe));
VPRecipeBase *BaseR = &Recipe; VPRecipeBase *BaseR = &Recipe;
EXPECT_TRUE(isa<VPUser>(BaseR)); EXPECT_TRUE(isa<VPUser>(BaseR));
EXPECT_EQ(&Recipe, BaseR->toVPUser()); EXPECT_EQ(&Recipe, BaseR);
} }
TEST(VPRecipeTest, CastVPReplicateRecipeToVPUser) { TEST(VPRecipeTest, CastVPReplicateRecipeToVPUser) {
@ -503,10 +518,10 @@ TEST(VPRecipeTest, CastVPBranchOnMaskRecipeToVPUser) {
EXPECT_TRUE(isa<VPUser>(&Recipe)); EXPECT_TRUE(isa<VPUser>(&Recipe));
VPRecipeBase *BaseR = &Recipe; VPRecipeBase *BaseR = &Recipe;
EXPECT_TRUE(isa<VPUser>(BaseR)); EXPECT_TRUE(isa<VPUser>(BaseR));
EXPECT_EQ(&Recipe, BaseR->toVPUser()); EXPECT_EQ(&Recipe, BaseR);
} }
TEST(VPRecipeTest, CastVPWidenMemoryInstructionRecipeToVPUser) { TEST(VPRecipeTest, CastVPWidenMemoryInstructionRecipeToVPUserAndVPDef) {
LLVMContext C; LLVMContext C;
IntegerType *Int32 = IntegerType::get(C, 32); IntegerType *Int32 = IntegerType::get(C, 32);
@ -519,7 +534,12 @@ TEST(VPRecipeTest, CastVPWidenMemoryInstructionRecipeToVPUser) {
EXPECT_TRUE(isa<VPUser>(&Recipe)); EXPECT_TRUE(isa<VPUser>(&Recipe));
VPRecipeBase *BaseR = &Recipe; VPRecipeBase *BaseR = &Recipe;
EXPECT_TRUE(isa<VPUser>(BaseR)); EXPECT_TRUE(isa<VPUser>(BaseR));
EXPECT_EQ(&Recipe, BaseR->toVPUser()); EXPECT_EQ(&Recipe, BaseR);
VPValue *VPV = Recipe.getVPValue();
EXPECT_TRUE(isa<VPRecipeBase>(VPV->getDef()));
EXPECT_EQ(&Recipe, dyn_cast<VPRecipeBase>(VPV->getDef()));
delete Load; delete Load;
} }
@ -536,11 +556,16 @@ TEST(VPRecipeTest, CastVPReductionRecipeToVPUser) {
EXPECT_TRUE(isa<VPUser>(BaseR)); EXPECT_TRUE(isa<VPUser>(BaseR));
} }
struct VPDoubleValueDef : public VPUser, public VPDef { struct VPDoubleValueDef : public VPRecipeBase, public VPUser {
VPDoubleValueDef(ArrayRef<VPValue *> Operands) : VPUser(Operands), VPDef() { VPDoubleValueDef(ArrayRef<VPValue *> Operands)
: VPRecipeBase(99), VPUser(Operands) {
new VPValue(nullptr, this); new VPValue(nullptr, this);
new VPValue(nullptr, this); new VPValue(nullptr, this);
} }
void execute(struct VPTransformState &State) override{};
void print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const override {}
}; };
TEST(VPDoubleValueDefTest, traverseUseLists) { TEST(VPDoubleValueDefTest, traverseUseLists) {