maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[LegalizeDAG] Share Vector/Scalar CTTZ Expansion 

As suggested on D53258, this patch demonstrates sharing common CTTZ expansion code between VectorLegalizer and SelectionDAGLegalize by putting it in TargetLowering.

I intend to move CTLZ and (scalar) CTPOP over as well and then update D53258 accordingly.

Differential Revision: https://reviews.llvm.org/D53474

llvm-svn: 345039
This commit is contained in:
Simon Pilgrim 2018-10-23 15:37:19 +00:00
parent b555b76ed3
commit b975ff4700
4 changed files with 71 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
llvm/include/llvm/CodeGen/TargetLowering.h
View File

@ -3647,6 +3647,13 @@ public:
/// Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs.
SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const;
/// Expand CTTZ/CTTZ_ZERO_UNDEF nodes. Expands vector/scalar CTTZ nodes,
/// vector nodes can only succeed if all operations are legal/custom.
/// \param N Node to expand
/// \param Result output after conversion
/// \returns True, if the expansion was successful, false otherwise
bool expandCTTZ(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
/// Turn load of vector type into a load of the individual elements.
/// \param LD load to expand
/// \returns MERGE_VALUEs of the scalar loads with their chains.

36
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
View File

@ -2789,35 +2789,6 @@ SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
Op = DAG.getNOT(dl, Op, VT);
return DAG.getNode(ISD::CTPOP, dl, VT, Op);
}
case ISD::CTTZ_ZERO_UNDEF:
// This trivially expands to CTTZ.
return DAG.getNode(ISD::CTTZ, dl, VT, Op);
case ISD::CTTZ: {
if (TLI.isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
EVT SetCCVT = getSetCCResultType(VT);
SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
SDValue Zero = DAG.getConstant(0, dl, VT);
SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
DAG.getConstant(Len, dl, VT), CTTZ);
}
// for now, we use: { return popcount(~x & (x - 1)); }
// unless the target has ctlz but not ctpop, in which case we use:
// { return 32 - nlz(~x & (x-1)); }
// Ref: "Hacker's Delight" by Henry Warren
SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
DAG.getNOT(dl, Op, VT),
DAG.getNode(ISD::SUB, dl, VT, Op,
DAG.getConstant(1, dl, VT)));
// If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
if (!TLI.isOperationLegal(ISD::CTPOP, VT) &&
TLI.isOperationLegal(ISD::CTLZ, VT))
return DAG.getNode(ISD::SUB, dl, VT,
DAG.getConstant(Len, dl, VT),
DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
}
}
}
@ -2831,11 +2802,14 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
case ISD::CTPOP:
case ISD::CTLZ:
case ISD::CTLZ_ZERO_UNDEF:
case ISD::CTTZ:
case ISD::CTTZ_ZERO_UNDEF:
Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
Results.push_back(Tmp1);
break;
case ISD::CTTZ:
case ISD::CTTZ_ZERO_UNDEF:
if (TLI.expandCTTZ(Node, Tmp1, DAG))
Results.push_back(Tmp1);
break;
case ISD::BITREVERSE:
Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
break;

20
llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
View File

@ -1105,23 +1105,11 @@ SDValue VectorLegalizer::ExpandCTLZ(SDValue Op) {
SDValue VectorLegalizer::ExpandCTTZ(SDValue Op) {
EVT VT = Op.getValueType();
unsigned NumBitsPerElt = VT.getScalarSizeInBits();
// If the non-ZERO_UNDEF version is supported we can use that instead.
if (TLI.isOperationLegalOrCustom(ISD::CTTZ, VT)) {
SDLoc DL(Op);
return DAG.getNode(ISD::CTTZ, DL, VT, Op.getOperand(0));
}
// If we have the appropriate vector bit operations, it is better to use them
// than unrolling and expanding each component.
if (isPowerOf2_32(NumBitsPerElt) &&
(TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) ||
TLI.isOperationLegalOrCustom(ISD::CTLZ, VT)) &&
TLI.isOperationLegalOrCustom(ISD::SUB, VT) &&
TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
TLI.isOperationLegalOrCustomOrPromote(ISD::XOR, VT))
return Op;
// Attempt to expand using TargetLowering.
SDValue Result;
if (TLI.expandCTTZ(Op.getNode(), Result, DAG))
return Result;
// Otherwise go ahead and unroll.
return DAG.UnrollVectorOp(Op.getNode());

55
llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
View File

@ -4142,6 +4142,61 @@ SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
return SDValue();
}
bool TargetLowering::expandCTTZ(SDNode *Node, SDValue &Result,
SelectionDAG &DAG) const {
SDLoc dl(Node);
EVT VT = Node->getValueType(0);
SDValue Op = Node->getOperand(0);
unsigned NumBitsPerElt = VT.getScalarSizeInBits();
// If the non-ZERO_UNDEF version is supported we can use that instead.
if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF &&
isOperationLegalOrCustom(ISD::CTTZ, VT)) {
Result = DAG.getNode(ISD::CTTZ, dl, VT, Op);
return true;
}
// If the ZERO_UNDEF version is supported use that and handle the zero case.
if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
EVT SetCCVT =
getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
SDValue Zero = DAG.getConstant(0, dl, VT);
SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ);
return true;
}
// Only expand vector types if we have the appropriate vector bit operations.
if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
(!isOperationLegalOrCustom(ISD::CTPOP, VT) &&
!isOperationLegalOrCustom(ISD::CTLZ, VT)) ||
!isOperationLegalOrCustom(ISD::SUB, VT) ||
!isOperationLegalOrCustomOrPromote(ISD::AND, VT) ||
!isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
return false;
// for now, we use: { return popcount(~x & (x - 1)); }
// unless the target has ctlz but not ctpop, in which case we use:
// { return 32 - nlz(~x & (x-1)); }
// Ref: "Hacker's Delight" by Henry Warren
SDValue Tmp = DAG.getNode(
ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT),
DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT)));
// If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) {
Result =
DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT),
DAG.getNode(ISD::CTLZ, dl, VT, Tmp));
return true;
}
Result = DAG.getNode(ISD::CTPOP, dl, VT, Tmp);
return true;
}
SDValue TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,
SelectionDAG &DAG) const {
SDLoc SL(LD);