maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[X86] LowerFunnelShift - enable v8i16 lowering

This commit is contained in:
Simon Pilgrim 2022-01-29 16:20:36 +00:00
parent 6777289dd9
commit ee9eeed773
3 changed files with 127 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
llvm/lib/Target/X86/X86ISelLowering.cpp
View File

@ -1095,13 +1095,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
if (VT == MVT::v2i64) continue; if (VT == MVT::v2i64) continue;
setOperationAction(ISD::ROTL, VT, Custom); setOperationAction(ISD::ROTL, VT, Custom);
setOperationAction(ISD::ROTR, VT, Custom); setOperationAction(ISD::ROTR, VT, Custom);
setOperationAction(ISD::FSHL, VT, Custom);
setOperationAction(ISD::FSHR, VT, Custom);
} }
setOperationAction(ISD::FSHL, MVT::v16i8, Custom);
setOperationAction(ISD::FSHR, MVT::v16i8, Custom);
setOperationAction(ISD::FSHL, MVT::v4i32, Custom);
setOperationAction(ISD::FSHR, MVT::v4i32, Custom);
setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal); setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal);
setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal); setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal);
setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal); setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal);
@ -29846,7 +29843,8 @@ static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
{Op0, Op1, Amt}, DAG, Subtarget); {Op0, Op1, Amt}, DAG, Subtarget);
} }
assert((VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8 || assert((VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8 ||
VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) && VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v8i32 ||
VT == MVT::v16i32) &&
"Unexpected funnel shift type!"); "Unexpected funnel shift type!");
// fshl(x,y,z) -> unpack(y,x) << (z & (bw-1))) >> bw. // fshl(x,y,z) -> unpack(y,x) << (z & (bw-1))) >> bw.
@ -29858,6 +29856,10 @@ static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
bool IsCst = ISD::isBuildVectorOfConstantSDNodes(AmtMod.getNode()); bool IsCst = ISD::isBuildVectorOfConstantSDNodes(AmtMod.getNode());
// Constant vXi16 funnel shifts can be efficiently handled by default.
if (IsCst && EltSizeInBits == 16)
return SDValue();
unsigned ShiftOpc = IsFSHR ? ISD::SRL : ISD::SHL; unsigned ShiftOpc = IsFSHR ? ISD::SRL : ISD::SHL;
unsigned NumElts = VT.getVectorNumElements(); unsigned NumElts = VT.getVectorNumElements();
MVT ExtSVT = MVT::getIntegerVT(2 * EltSizeInBits); MVT ExtSVT = MVT::getIntegerVT(2 * EltSizeInBits);
@ -29877,6 +29879,10 @@ static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
// Attempt to fold scalar shift as unpack(y,x) << zext(splat(z)) // Attempt to fold scalar shift as unpack(y,x) << zext(splat(z))
if (supportedVectorShiftWithBaseAmnt(ExtVT, Subtarget, ShiftOpc)) { if (supportedVectorShiftWithBaseAmnt(ExtVT, Subtarget, ShiftOpc)) {
if (SDValue ScalarAmt = DAG.getSplatValue(AmtMod)) { if (SDValue ScalarAmt = DAG.getSplatValue(AmtMod)) {
// Uniform vXi16 funnel shifts can be efficiently handled by default.
if (EltSizeInBits == 16)
return SDValue();
SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0)); SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0));
SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0)); SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0));
ScalarAmt = DAG.getZExtOrTrunc(ScalarAmt, DL, MVT::i32); ScalarAmt = DAG.getZExtOrTrunc(ScalarAmt, DL, MVT::i32);
@ -29915,7 +29921,7 @@ static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
} }
// Attempt to fold per-element (ExtVT) shift as unpack(y,x) << zext(z) // Attempt to fold per-element (ExtVT) shift as unpack(y,x) << zext(z)
if (((IsCst || !Subtarget.hasAVX512()) && !IsFSHR && EltSizeInBits == 8) || if (((IsCst || !Subtarget.hasAVX512()) && !IsFSHR && EltSizeInBits <= 16) ||
supportedVectorVarShift(ExtVT, Subtarget, ShiftOpc)) { supportedVectorVarShift(ExtVT, Subtarget, ShiftOpc)) {
SDValue Z = DAG.getConstant(0, DL, VT); SDValue Z = DAG.getConstant(0, DL, VT);
SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0)); SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0));

285
llvm/test/CodeGen/X86/vector-fshl-128.ll
View File

@ -424,39 +424,8 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) nounwind { define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) nounwind {
; SSE2-LABEL: var_funnnel_v8i16: ; SSE2-LABEL: var_funnnel_v8i16:
; SSE2: # %bb.0: ; SSE2: # %bb.0:
; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 ; SSE2-NEXT: movdqa %xmm1, %xmm3
; SSE2-NEXT: pxor %xmm2, %xmm4 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
; SSE2-NEXT: psllw $12, %xmm4
; SSE2-NEXT: movdqa %xmm4, %xmm3
; SSE2-NEXT: psraw $15, %xmm3
; SSE2-NEXT: movdqa %xmm1, %xmm5
; SSE2-NEXT: psrlw $1, %xmm5
; SSE2-NEXT: psrlw $9, %xmm1
; SSE2-NEXT: pand %xmm3, %xmm1
; SSE2-NEXT: pandn %xmm5, %xmm3
; SSE2-NEXT: por %xmm1, %xmm3
; SSE2-NEXT: paddw %xmm4, %xmm4
; SSE2-NEXT: movdqa %xmm4, %xmm1
; SSE2-NEXT: psraw $15, %xmm1
; SSE2-NEXT: movdqa %xmm1, %xmm5
; SSE2-NEXT: pandn %xmm3, %xmm5
; SSE2-NEXT: psrlw $4, %xmm3
; SSE2-NEXT: pand %xmm1, %xmm3
; SSE2-NEXT: por %xmm5, %xmm3
; SSE2-NEXT: paddw %xmm4, %xmm4
; SSE2-NEXT: movdqa %xmm4, %xmm1
; SSE2-NEXT: psraw $15, %xmm1
; SSE2-NEXT: movdqa %xmm1, %xmm5
; SSE2-NEXT: pandn %xmm3, %xmm5
; SSE2-NEXT: psrlw $2, %xmm3
; SSE2-NEXT: pand %xmm1, %xmm3
; SSE2-NEXT: por %xmm5, %xmm3
; SSE2-NEXT: paddw %xmm4, %xmm4
; SSE2-NEXT: psraw $15, %xmm4
; SSE2-NEXT: movdqa %xmm4, %xmm1
; SSE2-NEXT: pandn %xmm3, %xmm1
; SSE2-NEXT: psrlw $1, %xmm3
; SSE2-NEXT: pand %xmm4, %xmm3
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 ; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE2-NEXT: movdqa %xmm2, %xmm4 ; SSE2-NEXT: movdqa %xmm2, %xmm4
; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7] ; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
@ -464,132 +433,98 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216] ; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
; SSE2-NEXT: paddd %xmm5, %xmm4 ; SSE2-NEXT: paddd %xmm5, %xmm4
; SSE2-NEXT: cvttps2dq %xmm4, %xmm4 ; SSE2-NEXT: cvttps2dq %xmm4, %xmm4
; SSE2-NEXT: pslld $16, %xmm4 ; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]
; SSE2-NEXT: psrad $16, %xmm4 ; SSE2-NEXT: pmuludq %xmm4, %xmm3
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
; SSE2-NEXT: pmuludq %xmm6, %xmm4
; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
; SSE2-NEXT: psrad $16, %xmm3
; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3] ; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
; SSE2-NEXT: pslld $23, %xmm2 ; SSE2-NEXT: pslld $23, %xmm2
; SSE2-NEXT: paddd %xmm5, %xmm2 ; SSE2-NEXT: paddd %xmm5, %xmm2
; SSE2-NEXT: cvttps2dq %xmm2, %xmm2 ; SSE2-NEXT: cvttps2dq %xmm2, %xmm2
; SSE2-NEXT: pslld $16, %xmm2 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
; SSE2-NEXT: psrad $16, %xmm2 ; SSE2-NEXT: pmuludq %xmm2, %xmm1
; SSE2-NEXT: packssdw %xmm4, %xmm2 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
; SSE2-NEXT: pmullw %xmm2, %xmm0 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
; SSE2-NEXT: por %xmm1, %xmm0 ; SSE2-NEXT: pmuludq %xmm4, %xmm1
; SSE2-NEXT: por %xmm3, %xmm0 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; SSE2-NEXT: psrad $16, %xmm0
; SSE2-NEXT: packssdw %xmm3, %xmm0
; SSE2-NEXT: retq ; SSE2-NEXT: retq
; ;
; SSE41-LABEL: var_funnnel_v8i16: ; SSE41-LABEL: var_funnnel_v8i16:
; SSE41: # %bb.0: ; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm0, %xmm3 ; SSE41-NEXT: movdqa %xmm1, %xmm3
; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15] ; SSE41-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE41-NEXT: pandn %xmm5, %xmm0 ; SSE41-NEXT: pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; SSE41-NEXT: movdqa %xmm0, %xmm4
; SSE41-NEXT: psllw $12, %xmm4
; SSE41-NEXT: psllw $4, %xmm0
; SSE41-NEXT: por %xmm4, %xmm0
; SSE41-NEXT: movdqa %xmm0, %xmm4
; SSE41-NEXT: paddw %xmm0, %xmm4
; SSE41-NEXT: movdqa %xmm1, %xmm6
; SSE41-NEXT: psrlw $9, %xmm6
; SSE41-NEXT: psrlw $1, %xmm1
; SSE41-NEXT: pblendvb %xmm0, %xmm6, %xmm1
; SSE41-NEXT: movdqa %xmm1, %xmm6
; SSE41-NEXT: psrlw $4, %xmm6
; SSE41-NEXT: movdqa %xmm4, %xmm0
; SSE41-NEXT: pblendvb %xmm0, %xmm6, %xmm1
; SSE41-NEXT: movdqa %xmm1, %xmm6
; SSE41-NEXT: psrlw $2, %xmm6
; SSE41-NEXT: paddw %xmm4, %xmm4
; SSE41-NEXT: movdqa %xmm4, %xmm0
; SSE41-NEXT: pblendvb %xmm0, %xmm6, %xmm1
; SSE41-NEXT: movdqa %xmm1, %xmm6
; SSE41-NEXT: psrlw $1, %xmm6
; SSE41-NEXT: paddw %xmm4, %xmm4
; SSE41-NEXT: movdqa %xmm4, %xmm0
; SSE41-NEXT: pblendvb %xmm0, %xmm6, %xmm1
; SSE41-NEXT: pand %xmm5, %xmm2
; SSE41-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; SSE41-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; SSE41-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
; SSE41-NEXT: pslld $23, %xmm2 ; SSE41-NEXT: pslld $23, %xmm2
; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] ; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
; SSE41-NEXT: paddd %xmm4, %xmm2 ; SSE41-NEXT: paddd %xmm5, %xmm2
; SSE41-NEXT: cvttps2dq %xmm2, %xmm2 ; SSE41-NEXT: cvttps2dq %xmm2, %xmm2
; SSE41-NEXT: pslld $23, %xmm0 ; SSE41-NEXT: pmulld %xmm3, %xmm2
; SSE41-NEXT: paddd %xmm4, %xmm0 ; SSE41-NEXT: psrld $16, %xmm2
; SSE41-NEXT: cvttps2dq %xmm0, %xmm0 ; SSE41-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; SSE41-NEXT: pslld $23, %xmm4
; SSE41-NEXT: paddd %xmm5, %xmm4
; SSE41-NEXT: cvttps2dq %xmm4, %xmm0
; SSE41-NEXT: pmulld %xmm1, %xmm0
; SSE41-NEXT: psrld $16, %xmm0
; SSE41-NEXT: packusdw %xmm2, %xmm0 ; SSE41-NEXT: packusdw %xmm2, %xmm0
; SSE41-NEXT: pmullw %xmm3, %xmm0
; SSE41-NEXT: por %xmm1, %xmm0
; SSE41-NEXT: retq ; SSE41-NEXT: retq
; ;
; AVX1-LABEL: var_funnnel_v8i16: ; AVX1-LABEL: var_funnnel_v8i16:
; AVX1: # %bb.0: ; AVX1: # %bb.0:
; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15] ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX1-NEXT: vpandn %xmm3, %xmm2, %xmm4 ; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
; AVX1-NEXT: vpsllw $12, %xmm4, %xmm5 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4,4,5,5,6,6,7,7]
; AVX1-NEXT: vpsllw $4, %xmm4, %xmm4 ; AVX1-NEXT: vpslld $23, %xmm4, %xmm4
; AVX1-NEXT: vpor %xmm5, %xmm4, %xmm4 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
; AVX1-NEXT: vpaddw %xmm4, %xmm4, %xmm5 ; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4
; AVX1-NEXT: vpsrlw $9, %xmm1, %xmm6 ; AVX1-NEXT: vcvttps2dq %xmm4, %xmm4
; AVX1-NEXT: vpsrlw $1, %xmm1, %xmm1 ; AVX1-NEXT: vpmulld %xmm4, %xmm3, %xmm3
; AVX1-NEXT: vpblendvb %xmm4, %xmm6, %xmm1, %xmm1 ; AVX1-NEXT: vpsrld $16, %xmm3, %xmm3
; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm4 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX1-NEXT: vpblendvb %xmm5, %xmm4, %xmm1, %xmm1 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; AVX1-NEXT: vpsrlw $2, %xmm1, %xmm4 ; AVX1-NEXT: vpslld $23, %xmm1, %xmm1
; AVX1-NEXT: vpaddw %xmm5, %xmm5, %xmm5 ; AVX1-NEXT: vpaddd %xmm5, %xmm1, %xmm1
; AVX1-NEXT: vpblendvb %xmm5, %xmm4, %xmm1, %xmm1 ; AVX1-NEXT: vcvttps2dq %xmm1, %xmm1
; AVX1-NEXT: vpsrlw $1, %xmm1, %xmm4 ; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpaddw %xmm5, %xmm5, %xmm5 ; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
; AVX1-NEXT: vpblendvb %xmm5, %xmm4, %xmm1, %xmm1 ; AVX1-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]
; AVX1-NEXT: vpslld $23, %xmm3, %xmm3
; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
; AVX1-NEXT: vcvttps2dq %xmm3, %xmm3
; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; AVX1-NEXT: vpslld $23, %xmm2, %xmm2
; AVX1-NEXT: vpaddd %xmm4, %xmm2, %xmm2
; AVX1-NEXT: vcvttps2dq %xmm2, %xmm2
; AVX1-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX1-NEXT: vpmullw %xmm2, %xmm0, %xmm0
; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0
; AVX1-NEXT: retq ; AVX1-NEXT: retq
; ;
; AVX2-LABEL: var_funnnel_v8i16: ; AVX2-LABEL: var_funnnel_v8i16:
; AVX2: # %bb.0: ; AVX2: # %bb.0:
; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
; AVX2-NEXT: vpandn %xmm3, %xmm2, %xmm4
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
; AVX2-NEXT: vpsrlw $1, %xmm1, %xmm1
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX2-NEXT: vpsrlvd %ymm4, %ymm1, %ymm1
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm4
; AVX2-NEXT: vpackusdw %xmm4, %xmm1, %xmm1
; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVX2-NEXT: vpsllvd %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: vpslld $16, %ymm0, %ymm0
; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u] ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7],ymm1[8],ymm0[9],ymm1[10],ymm0[11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] ; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm1
; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX2-NEXT: vpsllvd %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vzeroupper ; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq ; AVX2-NEXT: retq
; ;
; AVX512F-LABEL: var_funnnel_v8i16: ; AVX512F-LABEL: var_funnnel_v8i16:
; AVX512F: # %bb.0: ; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
; AVX512F-NEXT: vpandn %xmm3, %xmm2, %xmm4
; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
; AVX512F-NEXT: vpsrlw $1, %xmm1, %xmm1
; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero ; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX512F-NEXT: vpsrlvd %ymm4, %ymm1, %ymm1
; AVX512F-NEXT: vpand %xmm3, %xmm2, %xmm2
; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero ; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVX512F-NEXT: vpsllvd %ymm2, %ymm0, %ymm0 ; AVX512F-NEXT: vpslld $16, %ymm0, %ymm0
; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX512F-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7],ymm1[8],ymm0[9],ymm1[10],ymm0[11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm1
; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX512F-NEXT: vpsllvd %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vpsrld $16, %ymm0, %ymm0
; AVX512F-NEXT: vpmovdw %zmm0, %ymm0 ; AVX512F-NEXT: vpmovdw %zmm0, %ymm0
; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: vzeroupper
@ -597,17 +532,14 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
; ;
; AVX512VL-LABEL: var_funnnel_v8i16: ; AVX512VL-LABEL: var_funnnel_v8i16:
; AVX512VL: # %bb.0: ; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
; AVX512VL-NEXT: vpandn %xmm3, %xmm2, %xmm4
; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
; AVX512VL-NEXT: vpsrlw $1, %xmm1, %xmm1
; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero ; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX512VL-NEXT: vpsrlvd %ymm4, %ymm1, %ymm1
; AVX512VL-NEXT: vpand %xmm3, %xmm2, %xmm2
; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero ; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVX512VL-NEXT: vpsllvd %ymm2, %ymm0, %ymm0 ; AVX512VL-NEXT: vpslld $16, %ymm0, %ymm0
; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX512VL-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7],ymm1[8],ymm0[9],ymm1[10],ymm0[11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
; AVX512VL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm1
; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX512VL-NEXT: vpsllvd %ymm1, %ymm0, %ymm0
; AVX512VL-NEXT: vpsrld $16, %ymm0, %ymm0
; AVX512VL-NEXT: vpmovdw %ymm0, %xmm0 ; AVX512VL-NEXT: vpmovdw %ymm0, %xmm0
; AVX512VL-NEXT: vzeroupper ; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq ; AVX512VL-NEXT: retq
@ -666,58 +598,37 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
; ;
; X86-SSE2-LABEL: var_funnnel_v8i16: ; X86-SSE2-LABEL: var_funnnel_v8i16:
; X86-SSE2: # %bb.0: ; X86-SSE2: # %bb.0:
; X86-SSE2-NEXT: pcmpeqd %xmm4, %xmm4 ; X86-SSE2-NEXT: movdqa %xmm1, %xmm3
; X86-SSE2-NEXT: pxor %xmm2, %xmm4 ; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
; X86-SSE2-NEXT: psllw $12, %xmm4
; X86-SSE2-NEXT: movdqa %xmm4, %xmm3
; X86-SSE2-NEXT: psraw $15, %xmm3
; X86-SSE2-NEXT: movdqa %xmm1, %xmm5
; X86-SSE2-NEXT: psrlw $1, %xmm5
; X86-SSE2-NEXT: psrlw $9, %xmm1
; X86-SSE2-NEXT: pand %xmm3, %xmm1
; X86-SSE2-NEXT: pandn %xmm5, %xmm3
; X86-SSE2-NEXT: por %xmm1, %xmm3
; X86-SSE2-NEXT: paddw %xmm4, %xmm4
; X86-SSE2-NEXT: movdqa %xmm4, %xmm1
; X86-SSE2-NEXT: psraw $15, %xmm1
; X86-SSE2-NEXT: movdqa %xmm1, %xmm5
; X86-SSE2-NEXT: pandn %xmm3, %xmm5
; X86-SSE2-NEXT: psrlw $4, %xmm3
; X86-SSE2-NEXT: pand %xmm1, %xmm3
; X86-SSE2-NEXT: por %xmm5, %xmm3
; X86-SSE2-NEXT: paddw %xmm4, %xmm4
; X86-SSE2-NEXT: movdqa %xmm4, %xmm1
; X86-SSE2-NEXT: psraw $15, %xmm1
; X86-SSE2-NEXT: movdqa %xmm1, %xmm5
; X86-SSE2-NEXT: pandn %xmm3, %xmm5
; X86-SSE2-NEXT: psrlw $2, %xmm3
; X86-SSE2-NEXT: pand %xmm1, %xmm3
; X86-SSE2-NEXT: por %xmm5, %xmm3
; X86-SSE2-NEXT: paddw %xmm4, %xmm4
; X86-SSE2-NEXT: psraw $15, %xmm4
; X86-SSE2-NEXT: movdqa %xmm4, %xmm1
; X86-SSE2-NEXT: pandn %xmm3, %xmm1
; X86-SSE2-NEXT: psrlw $1, %xmm3
; X86-SSE2-NEXT: pand %xmm4, %xmm3
; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2 ; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2
; X86-SSE2-NEXT: movdqa %xmm2, %xmm4 ; X86-SSE2-NEXT: movdqa %xmm2, %xmm5
; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7] ; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
; X86-SSE2-NEXT: pslld $23, %xmm4 ; X86-SSE2-NEXT: pslld $23, %xmm5
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216] ; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
; X86-SSE2-NEXT: paddd %xmm5, %xmm4 ; X86-SSE2-NEXT: paddd %xmm4, %xmm5
; X86-SSE2-NEXT: cvttps2dq %xmm4, %xmm4 ; X86-SSE2-NEXT: cvttps2dq %xmm5, %xmm5
; X86-SSE2-NEXT: pslld $16, %xmm4 ; X86-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]
; X86-SSE2-NEXT: psrad $16, %xmm4 ; X86-SSE2-NEXT: pmuludq %xmm5, %xmm3
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
; X86-SSE2-NEXT: pmuludq %xmm6, %xmm5
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
; X86-SSE2-NEXT: psrad $16, %xmm3
; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3] ; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
; X86-SSE2-NEXT: pslld $23, %xmm2 ; X86-SSE2-NEXT: pslld $23, %xmm2
; X86-SSE2-NEXT: paddd %xmm5, %xmm2 ; X86-SSE2-NEXT: paddd %xmm4, %xmm2
; X86-SSE2-NEXT: cvttps2dq %xmm2, %xmm2 ; X86-SSE2-NEXT: cvttps2dq %xmm2, %xmm2
; X86-SSE2-NEXT: pslld $16, %xmm2 ; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
; X86-SSE2-NEXT: psrad $16, %xmm2 ; X86-SSE2-NEXT: pmuludq %xmm2, %xmm1
; X86-SSE2-NEXT: packssdw %xmm4, %xmm2 ; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
; X86-SSE2-NEXT: pmullw %xmm2, %xmm0 ; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
; X86-SSE2-NEXT: por %xmm1, %xmm0 ; X86-SSE2-NEXT: pmuludq %xmm4, %xmm1
; X86-SSE2-NEXT: por %xmm3, %xmm0 ; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; X86-SSE2-NEXT: psrad $16, %xmm0
; X86-SSE2-NEXT: packssdw %xmm3, %xmm0
; X86-SSE2-NEXT: retl ; X86-SSE2-NEXT: retl
%res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
ret <8 x i16> %res ret <8 x i16> %res

45
llvm/test/CodeGen/X86/vector-fshr-128.ll
View File

@ -561,37 +561,28 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
; ;
; AVX2-LABEL: var_funnnel_v8i16: ; AVX2-LABEL: var_funnnel_v8i16:
; AVX2: # %bb.0: ; AVX2: # %bb.0:
; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm4
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX2-NEXT: vpsrlvd %ymm4, %ymm1, %ymm1
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm4
; AVX2-NEXT: vpackusdw %xmm4, %xmm1, %xmm1
; AVX2-NEXT: vpandn %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
; AVX2-NEXT: vpsllw $1, %xmm0, %xmm0
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVX2-NEXT: vpsllvd %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: vpslld $16, %ymm0, %ymm0
; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7],ymm1[8],ymm0[9],ymm1[10],ymm0[11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm1
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX2-NEXT: vpsrlvd %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u] ; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
; AVX2-NEXT: vzeroupper ; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq ; AVX2-NEXT: retq
; ;
; AVX512F-LABEL: var_funnnel_v8i16: ; AVX512F-LABEL: var_funnnel_v8i16:
; AVX512F: # %bb.0: ; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
; AVX512F-NEXT: vpand %xmm3, %xmm2, %xmm4
; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero ; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX512F-NEXT: vpsrlvd %ymm4, %ymm1, %ymm1
; AVX512F-NEXT: vpandn %xmm3, %xmm2, %xmm2
; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
; AVX512F-NEXT: vpsllw $1, %xmm0, %xmm0
; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero ; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVX512F-NEXT: vpsllvd %ymm2, %ymm0, %ymm0 ; AVX512F-NEXT: vpslld $16, %ymm0, %ymm0
; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX512F-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7],ymm1[8],ymm0[9],ymm1[10],ymm0[11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm1
; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX512F-NEXT: vpsrlvd %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vpmovdw %zmm0, %ymm0 ; AVX512F-NEXT: vpmovdw %zmm0, %ymm0
; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: vzeroupper
@ -599,17 +590,13 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
; ;
; AVX512VL-LABEL: var_funnnel_v8i16: ; AVX512VL-LABEL: var_funnnel_v8i16:
; AVX512VL: # %bb.0: ; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
; AVX512VL-NEXT: vpand %xmm3, %xmm2, %xmm4
; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero ; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX512VL-NEXT: vpsrlvd %ymm4, %ymm1, %ymm1
; AVX512VL-NEXT: vpandn %xmm3, %xmm2, %xmm2
; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
; AVX512VL-NEXT: vpsllw $1, %xmm0, %xmm0
; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero ; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVX512VL-NEXT: vpsllvd %ymm2, %ymm0, %ymm0 ; AVX512VL-NEXT: vpslld $16, %ymm0, %ymm0
; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX512VL-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7],ymm1[8],ymm0[9],ymm1[10],ymm0[11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
; AVX512VL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm1
; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX512VL-NEXT: vpsrlvd %ymm1, %ymm0, %ymm0
; AVX512VL-NEXT: vpmovdw %ymm0, %xmm0 ; AVX512VL-NEXT: vpmovdw %ymm0, %xmm0
; AVX512VL-NEXT: vzeroupper ; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq ; AVX512VL-NEXT: retq