diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 10fdcc3d6ca8..2f8e97e63fd4 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -44457,12 +44457,15 @@ static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC, // MOVMSK(PCMPEQ(X,0)) == -1 -> PTESTZ(X,X). // MOVMSK(PCMPEQ(X,0)) != -1 -> !PTESTZ(X,X). + // MOVMSK(PCMPEQ(X,Y)) == -1 -> PTESTZ(SUB(X,Y),SUB(X,Y)). + // MOVMSK(PCMPEQ(X,Y)) != -1 -> !PTESTZ(SUB(X,Y),SUB(X,Y)). if (IsAllOf && Subtarget.hasSSE41()) { SDValue BC = peekThroughBitcasts(Vec); - if (BC.getOpcode() == X86ISD::PCMPEQ && - ISD::isBuildVectorAllZeros(BC.getOperand(1).getNode())) { + if (BC.getOpcode() == X86ISD::PCMPEQ) { MVT TestVT = VecVT.is128BitVector() ? MVT::v2i64 : MVT::v4i64; - SDValue V = DAG.getBitcast(TestVT, BC.getOperand(0)); + SDValue V = DAG.getNode(ISD::SUB, SDLoc(BC), BC.getValueType(), + BC.getOperand(0), BC.getOperand(1)); + V = DAG.getBitcast(TestVT, V); return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V); } } @@ -44500,7 +44503,14 @@ static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC, VecOp1.getConstantOperandAPInt(1) == 8 && (IsAnyOf || (SignExt0 && SignExt1))) { SDLoc DL(EFLAGS); - SDValue Result = DAG.getBitcast(MVT::v32i8, VecOp0.getOperand(0)); + SDValue Result = peekThroughBitcasts(VecOp0.getOperand(0)); + if (IsAllOf && Result.getOpcode() == X86ISD::PCMPEQ) { + SDValue V = DAG.getNode(ISD::SUB, DL, Result.getValueType(), + Result.getOperand(0), Result.getOperand(1)); + V = DAG.getBitcast(MVT::v4i64, V); + return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V); + } + Result = DAG.getBitcast(MVT::v32i8, Result); Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result); unsigned CmpMask = IsAnyOf ? 0 : 0xFFFFFFFF; if (!SignExt0 || !SignExt1) { diff --git a/llvm/test/CodeGen/X86/movmsk-cmp.ll b/llvm/test/CodeGen/X86/movmsk-cmp.ll index 4488aeb273f7..68adeda025d7 100644 --- a/llvm/test/CodeGen/X86/movmsk-cmp.ll +++ b/llvm/test/CodeGen/X86/movmsk-cmp.ll @@ -4162,17 +4162,15 @@ define i1 @movmsk_or_v2i64(<2 x i64> %x, <2 x i64> %y) { ; ; SSE41-LABEL: movmsk_or_v2i64: ; SSE41: # %bb.0: -; SSE41-NEXT: pcmpeqq %xmm1, %xmm0 -; SSE41-NEXT: movmskpd %xmm0, %eax -; SSE41-NEXT: cmpl $3, %eax +; SSE41-NEXT: psubq %xmm1, %xmm0 +; SSE41-NEXT: ptest %xmm0, %xmm0 ; SSE41-NEXT: setne %al ; SSE41-NEXT: retq ; ; AVX1OR2-LABEL: movmsk_or_v2i64: ; AVX1OR2: # %bb.0: -; AVX1OR2-NEXT: vpcmpeqq %xmm1, %xmm0, %xmm0 -; AVX1OR2-NEXT: vmovmskpd %xmm0, %eax -; AVX1OR2-NEXT: cmpl $3, %eax +; AVX1OR2-NEXT: vpsubq %xmm1, %xmm0, %xmm0 +; AVX1OR2-NEXT: vptest %xmm0, %xmm0 ; AVX1OR2-NEXT: setne %al ; AVX1OR2-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vector-compare-all_of.ll b/llvm/test/CodeGen/X86/vector-compare-all_of.ll index a4a8367f9329..b39971255360 100644 --- a/llvm/test/CodeGen/X86/vector-compare-all_of.ll +++ b/llvm/test/CodeGen/X86/vector-compare-all_of.ll @@ -1249,9 +1249,8 @@ define i1 @bool_reduction_v8i32(<8 x i32> %x, <8 x i32> %y) { ; AVX2-LABEL: bool_reduction_v8i32: ; AVX2: # %bb.0: ; AVX2-NEXT: vpminud %ymm1, %ymm0, %ymm1 -; AVX2-NEXT: vpcmpeqd %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vmovmskps %ymm0, %eax -; AVX2-NEXT: cmpb $-1, %al +; AVX2-NEXT: vpsubd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vptest %ymm0, %ymm0 ; AVX2-NEXT: sete %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq @@ -1301,9 +1300,8 @@ define i1 @bool_reduction_v16i16(<16 x i16> %x, <16 x i16> %y) { ; ; AVX2-LABEL: bool_reduction_v16i16: ; AVX2: # %bb.0: -; AVX2-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vpmovmskb %ymm0, %eax -; AVX2-NEXT: cmpl $-1, %eax +; AVX2-NEXT: vpsubw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vptest %ymm0, %ymm0 ; AVX2-NEXT: sete %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq @@ -1354,9 +1352,8 @@ define i1 @bool_reduction_v32i8(<32 x i8> %x, <32 x i8> %y) { ; ; AVX2-LABEL: bool_reduction_v32i8: ; AVX2: # %bb.0: -; AVX2-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vpmovmskb %ymm0, %eax -; AVX2-NEXT: cmpl $-1, %eax +; AVX2-NEXT: vpsubb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vptest %ymm0, %ymm0 ; AVX2-NEXT: sete %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vector-compare-any_of.ll b/llvm/test/CodeGen/X86/vector-compare-any_of.ll index ee7e5f983c58..9af3d5e4969e 100644 --- a/llvm/test/CodeGen/X86/vector-compare-any_of.ll +++ b/llvm/test/CodeGen/X86/vector-compare-any_of.ll @@ -959,17 +959,15 @@ define i1 @bool_reduction_v2i64(<2 x i64> %x, <2 x i64> %y) { define i1 @bool_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) { ; SSE-LABEL: bool_reduction_v4i32: ; SSE: # %bb.0: -; SSE-NEXT: pcmpeqd %xmm1, %xmm0 -; SSE-NEXT: movmskps %xmm0, %eax -; SSE-NEXT: cmpl $15, %eax +; SSE-NEXT: psubd %xmm1, %xmm0 +; SSE-NEXT: ptest %xmm0, %xmm0 ; SSE-NEXT: setne %al ; SSE-NEXT: retq ; ; AVX-LABEL: bool_reduction_v4i32: ; AVX: # %bb.0: -; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 -; AVX-NEXT: vmovmskps %xmm0, %eax -; AVX-NEXT: cmpl $15, %eax +; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vptest %xmm0, %xmm0 ; AVX-NEXT: setne %al ; AVX-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll index c9aef7b8e740..be9ebf466eb0 100644 --- a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll +++ b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll @@ -1210,10 +1210,7 @@ define i1 @icmp0_v16i16_v16i1(<16 x i16>) { ; ; AVX2-LABEL: icmp0_v16i16_v16i1: ; AVX2: # %bb.0: -; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX2-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vpmovmskb %ymm0, %eax -; AVX2-NEXT: cmpl $-1, %eax +; AVX2-NEXT: vptest %ymm0, %ymm0 ; AVX2-NEXT: sete %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq @@ -1679,17 +1676,15 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) { ; ; SSE41-LABEL: icmp_v2i64_v2i1: ; SSE41: # %bb.0: -; SSE41-NEXT: pcmpeqq %xmm1, %xmm0 -; SSE41-NEXT: movmskpd %xmm0, %eax -; SSE41-NEXT: cmpb $3, %al +; SSE41-NEXT: psubq %xmm1, %xmm0 +; SSE41-NEXT: ptest %xmm0, %xmm0 ; SSE41-NEXT: sete %al ; SSE41-NEXT: retq ; ; AVX-LABEL: icmp_v2i64_v2i1: ; AVX: # %bb.0: -; AVX-NEXT: vpcmpeqq %xmm1, %xmm0, %xmm0 -; AVX-NEXT: vmovmskpd %xmm0, %eax -; AVX-NEXT: cmpb $3, %al +; AVX-NEXT: vpsubq %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vptest %xmm0, %xmm0 ; AVX-NEXT: sete %al ; AVX-NEXT: retq ; @@ -1728,19 +1723,25 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) { } define i1 @icmp_v4i32_v4i1(<4 x i32>, <4 x i32>) { -; SSE-LABEL: icmp_v4i32_v4i1: -; SSE: # %bb.0: -; SSE-NEXT: pcmpeqd %xmm1, %xmm0 -; SSE-NEXT: movmskps %xmm0, %eax -; SSE-NEXT: cmpb $15, %al -; SSE-NEXT: sete %al -; SSE-NEXT: retq +; SSE2-LABEL: icmp_v4i32_v4i1: +; SSE2: # %bb.0: +; SSE2-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE2-NEXT: movmskps %xmm0, %eax +; SSE2-NEXT: cmpb $15, %al +; SSE2-NEXT: sete %al +; SSE2-NEXT: retq +; +; SSE41-LABEL: icmp_v4i32_v4i1: +; SSE41: # %bb.0: +; SSE41-NEXT: psubd %xmm1, %xmm0 +; SSE41-NEXT: ptest %xmm0, %xmm0 +; SSE41-NEXT: sete %al +; SSE41-NEXT: retq ; ; AVX-LABEL: icmp_v4i32_v4i1: ; AVX: # %bb.0: -; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 -; AVX-NEXT: vmovmskps %xmm0, %eax -; AVX-NEXT: cmpb $15, %al +; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vptest %xmm0, %xmm0 ; AVX-NEXT: sete %al ; AVX-NEXT: retq ; @@ -1832,27 +1833,32 @@ define i1 @icmp_v8i16_v8i1(<8 x i16>, <8 x i16>) { } define i1 @icmp_v16i8_v16i1(<16 x i8>, <16 x i8>) { -; SSE-LABEL: icmp_v16i8_v16i1: -; SSE: # %bb.0: -; SSE-NEXT: pcmpeqb %xmm1, %xmm0 -; SSE-NEXT: pmovmskb %xmm0, %eax -; SSE-NEXT: cmpw $-1, %ax -; SSE-NEXT: sete %al -; SSE-NEXT: retq +; SSE2-LABEL: icmp_v16i8_v16i1: +; SSE2: # %bb.0: +; SSE2-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE2-NEXT: pmovmskb %xmm0, %eax +; SSE2-NEXT: cmpw $-1, %ax +; SSE2-NEXT: sete %al +; SSE2-NEXT: retq +; +; SSE41-LABEL: icmp_v16i8_v16i1: +; SSE41: # %bb.0: +; SSE41-NEXT: psubb %xmm1, %xmm0 +; SSE41-NEXT: ptest %xmm0, %xmm0 +; SSE41-NEXT: sete %al +; SSE41-NEXT: retq ; ; AVX-LABEL: icmp_v16i8_v16i1: ; AVX: # %bb.0: -; AVX-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 -; AVX-NEXT: vpmovmskb %xmm0, %eax -; AVX-NEXT: cmpw $-1, %ax +; AVX-NEXT: vpsubb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vptest %xmm0, %xmm0 ; AVX-NEXT: sete %al ; AVX-NEXT: retq ; ; AVX512F-LABEL: icmp_v16i8_v16i1: ; AVX512F: # %bb.0: -; AVX512F-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 -; AVX512F-NEXT: vpmovmskb %xmm0, %eax -; AVX512F-NEXT: cmpw $-1, %ax +; AVX512F-NEXT: vpsubb %xmm1, %xmm0, %xmm0 +; AVX512F-NEXT: vptest %xmm0, %xmm0 ; AVX512F-NEXT: sete %al ; AVX512F-NEXT: retq ; @@ -1917,9 +1923,8 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>, <4 x i64>) { ; ; AVX2-LABEL: icmp_v4i64_v4i1: ; AVX2: # %bb.0: -; AVX2-NEXT: vpcmpeqq %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vmovmskpd %ymm0, %eax -; AVX2-NEXT: cmpb $15, %al +; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vptest %ymm0, %ymm0 ; AVX2-NEXT: sete %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq @@ -1986,9 +1991,8 @@ define i1 @icmp_v8i32_v8i1(<8 x i32>, <8 x i32>) { ; ; AVX2-LABEL: icmp_v8i32_v8i1: ; AVX2: # %bb.0: -; AVX2-NEXT: vpcmpeqd %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vmovmskps %ymm0, %eax -; AVX2-NEXT: cmpb $-1, %al +; AVX2-NEXT: vpsubd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vptest %ymm0, %ymm0 ; AVX2-NEXT: sete %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq @@ -2054,9 +2058,8 @@ define i1 @icmp_v16i16_v16i1(<16 x i16>, <16 x i16>) { ; ; AVX2-LABEL: icmp_v16i16_v16i1: ; AVX2: # %bb.0: -; AVX2-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vpmovmskb %ymm0, %eax -; AVX2-NEXT: cmpl $-1, %eax +; AVX2-NEXT: vpsubw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vptest %ymm0, %ymm0 ; AVX2-NEXT: sete %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq @@ -2119,9 +2122,8 @@ define i1 @icmp_v32i8_v32i1(<32 x i8>, <32 x i8>) { ; ; AVX2-LABEL: icmp_v32i8_v32i1: ; AVX2: # %bb.0: -; AVX2-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vpmovmskb %ymm0, %eax -; AVX2-NEXT: cmpl $-1, %eax +; AVX2-NEXT: vpsubb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vptest %ymm0, %ymm0 ; AVX2-NEXT: sete %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq