forked from OSchip/llvm-project
[x86] Fix a few more tiny patterns with the new vector shuffle lowering
that keep cropping up in the regression test suite. This also addresses one of the issues raised on the mailing list with failing to form 'movsd' in as many cases as we realistically should. There will be corresponding patches forthcoming for v4f32 at least. This was a lot of fuss for a relatively small gain, but all the fuss was on my end trying different ways of holding the pieces of the x86 fragment patterns *just right*. Now that it works, the code is reasonably simple. In the new test cases I'm adding here, v2i64 sticks out as just plain horrible. I've not come up with any great ideas here other than that it would be nice to recognize when we're *going* to take a domain crossing hit and cross earlier to get the decent instructions. At least with AVX it is slightly less silly.... llvm-svn: 218756
This commit is contained in:
Chandler Carruth
parent
048486109b
commit
6c02c031b8
|
|
@ -7738,6 +7738,17 @@ static SDValue lowerVectorShuffleAsZeroOrAnyExtend(
|
|||
return SDValue();
|
||||
}
|
||||
|
||||
/// \brief Try to get a scalar value for a specific element of a vector.
|
||||
///
|
||||
/// Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar.
|
||||
static SDValue getScalarValueForVectorElement(SDValue V, int Idx) {
|
||||
if (V.getOpcode() == ISD::BUILD_VECTOR ||
|
||||
(Idx == 0 && V.getOpcode() == ISD::SCALAR_TO_VECTOR))
|
||||
return V.getOperand(Idx);
|
||||
|
||||
return SDValue();
|
||||
}
|
||||
|
||||
/// \brief Try to lower insertion of a single element into a zero vector.
|
||||
///
|
||||
/// This is a common pattern that we have especially efficient patterns to lower
|
||||
|
|
@ -7773,13 +7784,10 @@ static SDValue lowerVectorShuffleAsElementInsertion(
|
|||
// all the smarts here sunk into that routine. However, the current
|
||||
// lowering of BUILD_VECTOR makes that nearly impossible until the old
|
||||
// vector shuffle lowering is dead.
|
||||
if (!((V2.getOpcode() == ISD::SCALAR_TO_VECTOR &&
|
||||
Mask[V2Index] == (int)Mask.size()) ||
|
||||
V2.getOpcode() == ISD::BUILD_VECTOR))
|
||||
SDValue V2S = getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size());
|
||||
if (!V2S)
|
||||
return SDValue();
|
||||
|
||||
SDValue V2S = V2.getOperand(Mask[V2Index] - Mask.size());
|
||||
|
||||
// First, we need to zext the scalar if it is smaller than an i32.
|
||||
MVT ExtVT = VT;
|
||||
MVT EltVT = VT.getVectorElementType();
|
||||
|
|
@ -7910,6 +7918,16 @@ static SDValue lowerV2F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
|
|||
MVT::v2f64, DL, V1, V2, Mask, Subtarget, DAG))
|
||||
return Insertion;
|
||||
|
||||
// Try to use one of the special instruction patterns to handle two common
|
||||
// blend patterns if a zero-blend above didn't work.
|
||||
if (isShuffleEquivalent(Mask, 0, 3) || isShuffleEquivalent(Mask, 1, 3))
|
||||
if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0]))
|
||||
// We can either use a special instruction to load over the low double or
|
||||
// to move just the low double.
|
||||
return DAG.getNode(ISD::isNON_EXTLoad(V1S.getNode()) ? X86ISD::MOVLPD
|
||||
: X86ISD::MOVSD,
|
||||
DL, MVT::v2f64, V2, V1S);
|
||||
|
||||
if (Subtarget->hasSSE41())
|
||||
if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,
|
||||
Subtarget, DAG))
|
||||
|
|
|
|||
|
|
@ -714,6 +714,196 @@ define <2 x double> @insert_mem_and_zero_v2f64(double* %ptr) {
|
|||
ret <2 x double> %shuffle
|
||||
}
|
||||
|
||||
define <2 x i64> @insert_reg_lo_v2i64(i64 %a, <2 x i64> %b) {
|
||||
; SSE2-LABEL: insert_reg_lo_v2i64:
|
||||
; SSE2: # BB#0:
|
||||
; SSE2-NEXT: movd %rdi, %xmm1
|
||||
; SSE2-NEXT: shufpd {{.*#+}} xmm1 = xmm1[0],xmm0[1]
|
||||
; SSE2-NEXT: movapd %xmm1, %xmm0
|
||||
; SSE2-NEXT: retq
|
||||
;
|
||||
; SSE3-LABEL: insert_reg_lo_v2i64:
|
||||
; SSE3: # BB#0:
|
||||
; SSE3-NEXT: movd %rdi, %xmm1
|
||||
; SSE3-NEXT: shufpd {{.*#+}} xmm1 = xmm1[0],xmm0[1]
|
||||
; SSE3-NEXT: movapd %xmm1, %xmm0
|
||||
; SSE3-NEXT: retq
|
||||
;
|
||||
; SSSE3-LABEL: insert_reg_lo_v2i64:
|
||||
; SSSE3: # BB#0:
|
||||
; SSSE3-NEXT: movd %rdi, %xmm1
|
||||
; SSSE3-NEXT: shufpd {{.*#+}} xmm1 = xmm1[0],xmm0[1]
|
||||
; SSSE3-NEXT: movapd %xmm1, %xmm0
|
||||
; SSSE3-NEXT: retq
|
||||
;
|
||||
; SSE41-LABEL: insert_reg_lo_v2i64:
|
||||
; SSE41: # BB#0:
|
||||
; SSE41-NEXT: movd %rdi, %xmm1
|
||||
; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5,6,7]
|
||||
; SSE41-NEXT: movdqa %xmm1, %xmm0
|
||||
; SSE41-NEXT: retq
|
||||
;
|
||||
; AVX1-LABEL: insert_reg_lo_v2i64:
|
||||
; AVX1: # BB#0:
|
||||
; AVX1-NEXT: vmovq %rdi, %xmm1
|
||||
; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
|
||||
; AVX1-NEXT: retq
|
||||
;
|
||||
; AVX2-LABEL: insert_reg_lo_v2i64:
|
||||
; AVX2: # BB#0:
|
||||
; AVX2-NEXT: vmovq %rdi, %xmm1
|
||||
; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
|
||||
; AVX2-NEXT: retq
|
||||
%v = insertelement <2 x i64> undef, i64 %a, i32 0
|
||||
%shuffle = shufflevector <2 x i64> %v, <2 x i64> %b, <2 x i32> <i32 0, i32 3>
|
||||
ret <2 x i64> %shuffle
|
||||
}
|
||||
|
||||
define <2 x i64> @insert_mem_lo_v2i64(i64* %ptr, <2 x i64> %b) {
|
||||
; SSE2-LABEL: insert_mem_lo_v2i64:
|
||||
; SSE2: # BB#0:
|
||||
; SSE2-NEXT: movq (%rdi), %xmm1
|
||||
; SSE2-NEXT: shufpd {{.*#+}} xmm1 = xmm1[0],xmm0[1]
|
||||
; SSE2-NEXT: movapd %xmm1, %xmm0
|
||||
; SSE2-NEXT: retq
|
||||
;
|
||||
; SSE3-LABEL: insert_mem_lo_v2i64:
|
||||
; SSE3: # BB#0:
|
||||
; SSE3-NEXT: movq (%rdi), %xmm1
|
||||
; SSE3-NEXT: shufpd {{.*#+}} xmm1 = xmm1[0],xmm0[1]
|
||||
; SSE3-NEXT: movapd %xmm1, %xmm0
|
||||
; SSE3-NEXT: retq
|
||||
;
|
||||
; SSSE3-LABEL: insert_mem_lo_v2i64:
|
||||
; SSSE3: # BB#0:
|
||||
; SSSE3-NEXT: movq (%rdi), %xmm1
|
||||
; SSSE3-NEXT: shufpd {{.*#+}} xmm1 = xmm1[0],xmm0[1]
|
||||
; SSSE3-NEXT: movapd %xmm1, %xmm0
|
||||
; SSSE3-NEXT: retq
|
||||
;
|
||||
; SSE41-LABEL: insert_mem_lo_v2i64:
|
||||
; SSE41: # BB#0:
|
||||
; SSE41-NEXT: movq (%rdi), %xmm1
|
||||
; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5,6,7]
|
||||
; SSE41-NEXT: movdqa %xmm1, %xmm0
|
||||
; SSE41-NEXT: retq
|
||||
;
|
||||
; AVX1-LABEL: insert_mem_lo_v2i64:
|
||||
; AVX1: # BB#0:
|
||||
; AVX1-NEXT: vmovq (%rdi), %xmm1
|
||||
; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
|
||||
; AVX1-NEXT: retq
|
||||
;
|
||||
; AVX2-LABEL: insert_mem_lo_v2i64:
|
||||
; AVX2: # BB#0:
|
||||
; AVX2-NEXT: vmovq (%rdi), %xmm1
|
||||
; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
|
||||
; AVX2-NEXT: retq
|
||||
%a = load i64* %ptr
|
||||
%v = insertelement <2 x i64> undef, i64 %a, i32 0
|
||||
%shuffle = shufflevector <2 x i64> %v, <2 x i64> %b, <2 x i32> <i32 0, i32 3>
|
||||
ret <2 x i64> %shuffle
|
||||
}
|
||||
|
||||
define <2 x i64> @insert_reg_hi_v2i64(i64 %a, <2 x i64> %b) {
|
||||
; SSE-LABEL: insert_reg_hi_v2i64:
|
||||
; SSE: # BB#0:
|
||||
; SSE-NEXT: movd %rdi, %xmm1
|
||||
; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
|
||||
; SSE-NEXT: retq
|
||||
;
|
||||
; AVX-LABEL: insert_reg_hi_v2i64:
|
||||
; AVX: # BB#0:
|
||||
; AVX-NEXT: vmovq %rdi, %xmm1
|
||||
; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
|
||||
; AVX-NEXT: retq
|
||||
%v = insertelement <2 x i64> undef, i64 %a, i32 0
|
||||
%shuffle = shufflevector <2 x i64> %v, <2 x i64> %b, <2 x i32> <i32 2, i32 0>
|
||||
ret <2 x i64> %shuffle
|
||||
}
|
||||
|
||||
define <2 x i64> @insert_mem_hi_v2i64(i64* %ptr, <2 x i64> %b) {
|
||||
; SSE-LABEL: insert_mem_hi_v2i64:
|
||||
; SSE: # BB#0:
|
||||
; SSE-NEXT: movq (%rdi), %xmm1
|
||||
; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
|
||||
; SSE-NEXT: retq
|
||||
;
|
||||
; AVX-LABEL: insert_mem_hi_v2i64:
|
||||
; AVX: # BB#0:
|
||||
; AVX-NEXT: vmovq (%rdi), %xmm1
|
||||
; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
|
||||
; AVX-NEXT: retq
|
||||
%a = load i64* %ptr
|
||||
%v = insertelement <2 x i64> undef, i64 %a, i32 0
|
||||
%shuffle = shufflevector <2 x i64> %v, <2 x i64> %b, <2 x i32> <i32 2, i32 0>
|
||||
ret <2 x i64> %shuffle
|
||||
}
|
||||
|
||||
define <2 x double> @insert_reg_lo_v2f64(double %a, <2 x double> %b) {
|
||||
; SSE-LABEL: insert_reg_lo_v2f64:
|
||||
; SSE: # BB#0:
|
||||
; SSE-NEXT: movsd %xmm0, %xmm1
|
||||
; SSE-NEXT: movaps %xmm1, %xmm0
|
||||
; SSE-NEXT: retq
|
||||
;
|
||||
; AVX-LABEL: insert_reg_lo_v2f64:
|
||||
; AVX: # BB#0:
|
||||
; AVX-NEXT: vmovsd %xmm0, %xmm1, %xmm0
|
||||
; AVX-NEXT: retq
|
||||
%v = insertelement <2 x double> undef, double %a, i32 0
|
||||
%shuffle = shufflevector <2 x double> %v, <2 x double> %b, <2 x i32> <i32 0, i32 3>
|
||||
ret <2 x double> %shuffle
|
||||
}
|
||||
|
||||
define <2 x double> @insert_mem_lo_v2f64(double* %ptr, <2 x double> %b) {
|
||||
; SSE-LABEL: insert_mem_lo_v2f64:
|
||||
; SSE: # BB#0:
|
||||
; SSE-NEXT: movlpd (%rdi), %xmm0
|
||||
; SSE-NEXT: retq
|
||||
;
|
||||
; AVX-LABEL: insert_mem_lo_v2f64:
|
||||
; AVX: # BB#0:
|
||||
; AVX-NEXT: vmovlpd (%rdi), %xmm0, %xmm0
|
||||
; AVX-NEXT: retq
|
||||
%a = load double* %ptr
|
||||
%v = insertelement <2 x double> undef, double %a, i32 0
|
||||
%shuffle = shufflevector <2 x double> %v, <2 x double> %b, <2 x i32> <i32 0, i32 3>
|
||||
ret <2 x double> %shuffle
|
||||
}
|
||||
|
||||
define <2 x double> @insert_reg_hi_v2f64(double %a, <2 x double> %b) {
|
||||
; SSE-LABEL: insert_reg_hi_v2f64:
|
||||
; SSE: # BB#0:
|
||||
; SSE-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
|
||||
; SSE-NEXT: movapd %xmm1, %xmm0
|
||||
; SSE-NEXT: retq
|
||||
;
|
||||
; AVX-LABEL: insert_reg_hi_v2f64:
|
||||
; AVX: # BB#0:
|
||||
; AVX-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
|
||||
; AVX-NEXT: retq
|
||||
%v = insertelement <2 x double> undef, double %a, i32 0
|
||||
%shuffle = shufflevector <2 x double> %v, <2 x double> %b, <2 x i32> <i32 2, i32 0>
|
||||
ret <2 x double> %shuffle
|
||||
}
|
||||
|
||||
define <2 x double> @insert_mem_hi_v2f64(double* %ptr, <2 x double> %b) {
|
||||
; SSE-LABEL: insert_mem_hi_v2f64:
|
||||
; SSE: # BB#0:
|
||||
; SSE-NEXT: movhpd (%rdi), %xmm0
|
||||
; SSE-NEXT: retq
|
||||
;
|
||||
; AVX-LABEL: insert_mem_hi_v2f64:
|
||||
; AVX: # BB#0:
|
||||
; AVX-NEXT: vmovhpd (%rdi), %xmm0, %xmm0
|
||||
; AVX-NEXT: retq
|
||||
%a = load double* %ptr
|
||||
%v = insertelement <2 x double> undef, double %a, i32 0
|
||||
%shuffle = shufflevector <2 x double> %v, <2 x double> %b, <2 x i32> <i32 2, i32 0>
|
||||
ret <2 x double> %shuffle
|
||||
}
|
||||
|
||||
define <2 x double> @insert_dup_reg_v2f64(double %a) {
|
||||
; FIXME: We should match movddup for SSE3 and higher here.
|
||||
;
|
||||
|
|
|
|||
Loading…
Reference in New Issue