From cc1a6361d34e270b407f91a6e2e76c7fb324ee2d Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 18 May 2021 08:47:17 +0100 Subject: [PATCH] [VPlan] Add VPUserID to distinguish between recipes and others. This allows cast/dyn_cast'ing from VPUser to recipes. This is needed because there are VPUsers that are not recipes. Reviewed By: gilr, a.elovikov Differential Revision: https://reviews.llvm.org/D100257 --- llvm/lib/Transforms/Vectorize/VPlan.cpp | 42 +++---------------- llvm/lib/Transforms/Vectorize/VPlan.h | 48 +++++++++++++++++++--- llvm/lib/Transforms/Vectorize/VPlanValue.h | 42 +++++++++++++------ 3 files changed, 78 insertions(+), 54 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index f4eacc9492cc..9ee2744df0c9 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -188,56 +188,26 @@ VPBlockBase *VPBlockBase::getEnclosingBlockWithPredecessors() { return Parent->getEnclosingBlockWithPredecessors(); } -static VPValue *getSingleOperandOrNull(VPUser &U) { - if (U.getNumOperands() == 1) - return U.getOperand(0); - - return nullptr; -} - -static const VPValue *getSingleOperandOrNull(const VPUser &U) { - if (U.getNumOperands() == 1) - return U.getOperand(0); - - return nullptr; -} - -static void resetSingleOpUser(VPUser &U, VPValue *NewVal) { - assert(U.getNumOperands() <= 1 && "Didn't expect more than one operand!"); - if (!NewVal) { - if (U.getNumOperands() == 1) - U.removeLastOperand(); - return; - } - - if (U.getNumOperands() == 1) - U.setOperand(0, NewVal); - else - U.addOperand(NewVal); -} - VPValue *VPBlockBase::getCondBit() { - return getSingleOperandOrNull(CondBitUser); + return CondBitUser.getSingleOperandOrNull(); } const VPValue *VPBlockBase::getCondBit() const { - return getSingleOperandOrNull(CondBitUser); + return CondBitUser.getSingleOperandOrNull(); } -void VPBlockBase::setCondBit(VPValue *CV) { - resetSingleOpUser(CondBitUser, CV); -} +void VPBlockBase::setCondBit(VPValue *CV) { CondBitUser.resetSingleOpUser(CV); } VPValue *VPBlockBase::getPredicate() { - return getSingleOperandOrNull(PredicateUser); + return PredicateUser.getSingleOperandOrNull(); } const VPValue *VPBlockBase::getPredicate() const { - return getSingleOperandOrNull(PredicateUser); + return PredicateUser.getSingleOperandOrNull(); } void VPBlockBase::setPredicate(VPValue *CV) { - resetSingleOpUser(PredicateUser, CV); + PredicateUser.resetSingleOpUser(CV); } void VPBlockBase::deleteCFG(VPBlockBase *Entry) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index cf59a34235ab..bfb8adb7ae6f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -341,6 +341,41 @@ struct VPTransformState { VPlan *Plan; }; +/// VPUsers instance used by VPBlockBase to manage CondBit and the block +/// predicate. Currently VPBlockUsers are used in VPBlockBase for historical +/// reasons, but in the future the only VPUsers should either be recipes or +/// live-outs.VPBlockBase uses. +struct VPBlockUser : public VPUser { + VPBlockUser() : VPUser({}, VPUserID::Block) {} + + VPValue *getSingleOperandOrNull() { + if (getNumOperands() == 1) + return getOperand(0); + + return nullptr; + } + const VPValue *getSingleOperandOrNull() const { + if (getNumOperands() == 1) + return getOperand(0); + + return nullptr; + } + + void resetSingleOpUser(VPValue *NewVal) { + assert(getNumOperands() <= 1 && "Didn't expect more than one operand!"); + if (!NewVal) { + if (getNumOperands() == 1) + removeLastOperand(); + return; + } + + if (getNumOperands() == 1) + setOperand(0, NewVal); + else + addOperand(NewVal); + } +}; + /// VPBlockBase is the building block of the Hierarchical Control-Flow Graph. /// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock. class VPBlockBase { @@ -364,12 +399,12 @@ class VPBlockBase { /// Successor selector managed by a VPUser. For blocks with zero or one /// successors, there is no operand. Otherwise there is exactly one operand /// which is the branch condition. - VPUser CondBitUser; + VPBlockUser CondBitUser; /// If the block is predicated, its predicate is stored as an operand of this /// VPUser to maintain the def-use relations. Otherwise there is no operand /// here. - VPUser PredicateUser; + VPBlockUser PredicateUser; /// VPlan containing the block. Can only be set on the entry block of the /// plan. @@ -621,17 +656,16 @@ class VPRecipeBase : public ilist_node_with_parent, friend VPBasicBlock; friend class VPBlockUtils; - /// Each VPRecipe belongs to a single VPBasicBlock. VPBasicBlock *Parent = nullptr; public: VPRecipeBase(const unsigned char SC, ArrayRef Operands) - : VPDef(SC), VPUser(Operands) {} + : VPDef(SC), VPUser(Operands, VPUser::VPUserID::Recipe) {} template VPRecipeBase(const unsigned char SC, iterator_range Operands) - : VPDef(SC), VPUser(Operands) {} + : VPDef(SC), VPUser(Operands, VPUser::VPUserID::Recipe) {} virtual ~VPRecipeBase() = default; /// \return the VPBasicBlock which this VPRecipe belongs to. @@ -683,6 +717,10 @@ public: return true; } + static inline bool classof(const VPUser *U) { + return U->getVPUserID() == VPUser::VPUserID::Recipe; + } + /// Returns true if the recipe may have side-effects. bool mayHaveSideEffects() const; diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index 6c41af603ccd..7b19e2f8a249 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -191,28 +191,42 @@ raw_ostream &operator<<(raw_ostream &OS, const VPValue &V); /// This class augments VPValue with operands which provide the inverse def-use /// edges from VPValue's users to their defs. class VPUser { +public: + /// Subclass identifier (for isa/dyn_cast). + enum class VPUserID { + Recipe, + // TODO: Currently VPUsers are used in VPBlockBase, but in the future the + // only VPUsers should either be recipes or live-outs. + Block + }; + +private: SmallVector Operands; + VPUserID ID; + protected: #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// Print the operands to \p O. void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const; #endif + VPUser(ArrayRef Operands, VPUserID ID) : ID(ID) { + for (VPValue *Operand : Operands) + addOperand(Operand); + } + + VPUser(std::initializer_list Operands, VPUserID ID) + : VPUser(ArrayRef(Operands), ID) {} + + template + VPUser(iterator_range Operands, VPUserID ID) : ID(ID) { + for (VPValue *Operand : Operands) + addOperand(Operand); + } + public: - VPUser() {} - VPUser(ArrayRef Operands) { - for (VPValue *Operand : Operands) - addOperand(Operand); - } - - VPUser(std::initializer_list Operands) - : VPUser(ArrayRef(Operands)) {} - template VPUser(iterator_range Operands) { - for (VPValue *Operand : Operands) - addOperand(Operand); - } - + VPUser() = delete; VPUser(const VPUser &) = delete; VPUser &operator=(const VPUser &) = delete; virtual ~VPUser() { @@ -220,6 +234,8 @@ public: Op->removeUser(*this); } + VPUserID getVPUserID() const { return ID; } + void addOperand(VPValue *Operand) { Operands.push_back(Operand); Operand->addUser(*this);