maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[VPlan] Keep start value in VPWidenIntOrFpInductionRecipe (NFC). 

This patch updates VPWidenIntOrFpInductionRecipe to hold the start value
for the induction variable. This makes the start value explicit and
allows for adjusting the start value for a VPlan.

The flexibility will be used in further patches.

Reviewed By: Ayal

Differential Revision: https://reviews.llvm.org/D92129
This commit is contained in:
Florian Hahn 2021-01-06 11:30:23 +00:00
parent aa280c99f7
commit 816dba48af
No known key found for this signature in database
GPG Key ID: 61D7554B5CECDC0D
4 changed files with 43 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -526,7 +526,8 @@ public:
/// Widen an integer or floating-point induction variable \p IV. If \p Trunc
/// is provided, the integer induction variable will first be truncated to
/// the corresponding type.
void widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc = nullptr);
void widenIntOrFpInduction(PHINode *IV, Value *Start,
TruncInst *Trunc = nullptr);
/// getOrCreateVectorValue and getOrCreateScalarValue coordinate to generate a
/// vector or scalar value on-demand if one is not yet available. When
@ -666,7 +667,8 @@ protected:
/// truncate instruction, instead of widening the original IV, we widen a
/// version of the IV truncated to \p EntryVal's type.
void createVectorIntOrFpInductionPHI(const InductionDescriptor &II,
Value *Step, Instruction *EntryVal);
Value *Step, Value *Start,
Instruction *EntryVal);
/// Returns true if an instruction \p I should be scalarized instead of
/// vectorized for the chosen vectorization factor.
@ -2005,10 +2007,10 @@ Value *InnerLoopVectorizer::getBroadcastInstrs(Value *V) {
}
void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
const InductionDescriptor &II, Value *Step, Instruction *EntryVal) {
const InductionDescriptor &II, Value *Step, Value *Start,
Instruction *EntryVal) {
assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
"Expected either an induction phi-node or a truncate of it!");
Value *Start = II.getStartValue();
// Construct the initial value of the vector IV in the vector loop preheader
auto CurrIP = Builder.saveIP();
@ -2126,7 +2128,8 @@ void InnerLoopVectorizer::recordVectorLoopValueForInductionCast(
VectorLoopValueMap.setVectorValue(CastInst, Part, VectorLoopVal);
}
void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc) {
void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start,
TruncInst *Trunc) {
assert((IV->getType()->isIntegerTy() || IV != OldInduction) &&
"Primary induction variable must have an integer type");
@ -2208,7 +2211,7 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc) {
// least one user in the loop that is not widened.
auto NeedsScalarIV = needsScalarInduction(EntryVal);
if (!NeedsScalarIV) {
createVectorIntOrFpInductionPHI(ID, Step, EntryVal);
createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal);
return;
}
@ -2216,7 +2219,7 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc) {
// create the phi node, we will splat the scalar induction variable in each
// loop iteration.
if (!shouldScalarizeInstruction(EntryVal)) {
createVectorIntOrFpInductionPHI(ID, Step, EntryVal);
createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal);
Value *ScalarIV = CreateScalarIV(Step);
// Create scalar steps that can be used by instructions we will later
// scalarize. Note that the addition of the scalar steps will not increase
@ -8061,20 +8064,22 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(Instruction *I, VFRange &Range,
}
VPWidenIntOrFpInductionRecipe *
VPRecipeBuilder::tryToOptimizeInductionPHI(PHINode *Phi) const {
VPRecipeBuilder::tryToOptimizeInductionPHI(PHINode *Phi, VPlan &Plan) const {
// Check if this is an integer or fp induction. If so, build the recipe that
// produces its scalar and vector values.
InductionDescriptor II = Legal->getInductionVars().lookup(Phi);
if (II.getKind() == InductionDescriptor::IK_IntInduction ||
II.getKind() == InductionDescriptor::IK_FpInduction)
return new VPWidenIntOrFpInductionRecipe(Phi);
II.getKind() == InductionDescriptor::IK_FpInduction) {
VPValue *Start = Plan.getOrAddVPValue(II.getStartValue());
return new VPWidenIntOrFpInductionRecipe(Phi, Start);
}
return nullptr;
}
VPWidenIntOrFpInductionRecipe *
VPRecipeBuilder::tryToOptimizeInductionTruncate(TruncInst *I,
VFRange &Range) const {
VPRecipeBuilder::tryToOptimizeInductionTruncate(TruncInst *I, VFRange &Range,
VPlan &Plan) const {
// Optimize the special case where the source is a constant integer
// induction variable. Notice that we can only optimize the 'trunc' case
// because (a) FP conversions lose precision, (b) sext/zext may wrap, and
@ -8090,9 +8095,14 @@ VPRecipeBuilder::tryToOptimizeInductionTruncate(TruncInst *I,
};
if (LoopVectorizationPlanner::getDecisionAndClampRange(
isOptimizableIVTruncate(I), Range))
isOptimizableIVTruncate(I), Range)) {
InductionDescriptor II =
Legal->getInductionVars().lookup(cast<PHINode>(I->getOperand(0)));
VPValue *Start = Plan.getOrAddVPValue(II.getStartValue());
return new VPWidenIntOrFpInductionRecipe(cast<PHINode>(I->getOperand(0)),
I);
Start, I);
}
return nullptr;
}
@ -8304,13 +8314,13 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
if (auto Phi = dyn_cast<PHINode>(Instr)) {
if (Phi->getParent() != OrigLoop->getHeader())
return tryToBlend(Phi, Plan);
if ((Recipe = tryToOptimizeInductionPHI(Phi)))
if ((Recipe = tryToOptimizeInductionPHI(Phi, *Plan)))
return Recipe;
return new VPWidenPHIRecipe(Phi);
}
if (isa<TruncInst>(Instr) &&
(Recipe = tryToOptimizeInductionTruncate(cast<TruncInst>(Instr), Range)))
if (isa<TruncInst>(Instr) && (Recipe = tryToOptimizeInductionTruncate(
cast<TruncInst>(Instr), Range, *Plan)))
return Recipe;
if (!shouldWiden(Instr, Range))
@ -8705,7 +8715,8 @@ void VPWidenGEPRecipe::execute(VPTransformState &State) {
void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
assert(!State.Instance && "Int or FP induction being replicated.");
State.ILV->widenIntOrFpInduction(IV, Trunc);
State.ILV->widenIntOrFpInduction(IV, getStartValue()->getLiveInIRValue(),
Trunc);
}
void VPWidenPHIRecipe::execute(VPTransformState &State) {

6
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
View File

@ -66,12 +66,14 @@ class VPRecipeBuilder {
/// Check if an induction recipe should be constructed for \I. If so build and
/// return it. If not, return null.
VPWidenIntOrFpInductionRecipe *tryToOptimizeInductionPHI(PHINode *Phi) const;
VPWidenIntOrFpInductionRecipe *tryToOptimizeInductionPHI(PHINode *Phi,
VPlan &Plan) const;
/// Optimize the special case where the operand of \p I is a constant integer
/// induction variable.
VPWidenIntOrFpInductionRecipe *
tryToOptimizeInductionTruncate(TruncInst *I, VFRange &Range) const;
tryToOptimizeInductionTruncate(TruncInst *I, VFRange &Range,
VPlan &Plan) const;
/// Handle non-loop phi nodes. Currently all such phi nodes are turned into
/// a sequence of select instructions as the vectorizer currently performs

11
llvm/lib/Transforms/Vectorize/VPlan.h
View File

@ -932,13 +932,15 @@ public:
/// A recipe for handling phi nodes of integer and floating-point inductions,
/// producing their vector and scalar values.
class VPWidenIntOrFpInductionRecipe : public VPRecipeBase {
class VPWidenIntOrFpInductionRecipe : public VPRecipeBase, public VPUser {
PHINode *IV;
TruncInst *Trunc;
public:
VPWidenIntOrFpInductionRecipe(PHINode *IV, TruncInst *Trunc = nullptr)
: VPRecipeBase(VPWidenIntOrFpInductionSC), IV(IV), Trunc(Trunc) {
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start,
TruncInst *Trunc = nullptr)
: VPRecipeBase(VPWidenIntOrFpInductionSC), VPUser({Start}), IV(IV),
Trunc(Trunc) {
if (Trunc)
new VPValue(Trunc, this);
else
@ -958,6 +960,9 @@ public:
/// Print the recipe.
void print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const override;
/// Returns the start value of the induction.
VPValue *getStartValue() { return getOperand(0); }
};
/// A recipe for handling all phi nodes except for integer and FP inductions.

3
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
View File

@ -68,7 +68,8 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
InductionDescriptor II = Inductions.lookup(Phi);
if (II.getKind() == InductionDescriptor::IK_IntInduction ||
II.getKind() == InductionDescriptor::IK_FpInduction) {
NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi);
VPValue *Start = Plan->getOrAddVPValue(II.getStartValue());
NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start);
} else
NewRecipe = new VPWidenPHIRecipe(Phi);
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {