From 5910ebe72068718d67aa1a6b556980b51252fbbf Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Mon, 20 Feb 2017 12:16:38 +0000 Subject: [PATCH] [X86][AVX512] Add support for ASHR v2i64/v4i64 support without VLX Use v8i64 ASHR instructions if we don't have VLX. Differential Revision: https://reviews.llvm.org/D28537 llvm-svn: 295656 --- llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +- llvm/lib/Target/X86/X86InstrAVX512.td | 27 +++++++++++++++++++ llvm/test/CodeGen/X86/avx512-ext.ll | 18 +++++-------- .../X86/avx512dq-intrinsics-upgrade.ll | 3 +-- llvm/test/CodeGen/X86/compress_expand.ll | 3 +-- .../test/CodeGen/X86/vector-shift-ashr-128.ll | 9 +++---- .../test/CodeGen/X86/vector-shift-ashr-256.ll | 9 +++---- 7 files changed, 44 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 7d16dd9b623f..161cf17bfd16 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -21387,7 +21387,7 @@ static bool SupportedVectorShiftWithImm(MVT VT, const X86Subtarget &Subtarget, bool LShift = VT.is128BitVector() || (VT.is256BitVector() && Subtarget.hasInt256()); - bool AShift = LShift && (Subtarget.hasVLX() || + bool AShift = LShift && (Subtarget.hasAVX512() || (VT != MVT::v2i64 && VT != MVT::v4i64)); return (Opcode == ISD::SRA) ? AShift : LShift; } diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td index 5fda485dc8e0..20708afe2dc4 100644 --- a/llvm/lib/Target/X86/X86InstrAVX512.td +++ b/llvm/lib/Target/X86/X86InstrAVX512.td @@ -4798,6 +4798,33 @@ defm VPSLL : avx512_shift_types<0xF2, 0xF3, 0xF1, "vpsll", X86vshl>; defm VPSRA : avx512_shift_types<0xE2, 0xE2, 0xE1, "vpsra", X86vsra>; defm VPSRL : avx512_shift_types<0xD2, 0xD3, 0xD1, "vpsrl", X86vsrl>; +// Use 512bit VPSRA/VPSRAI version to implement v2i64/v4i64 in case NoVLX. +let Predicates = [HasAVX512, NoVLX] in { + def : Pat<(v4i64 (X86vsra (v4i64 VR256X:$src1), (v2i64 VR128X:$src2))), + (EXTRACT_SUBREG (v8i64 + (VPSRAQZrr + (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)), + VR128X:$src2)), sub_ymm)>; + + def : Pat<(v2i64 (X86vsra (v2i64 VR128X:$src1), (v2i64 VR128X:$src2))), + (EXTRACT_SUBREG (v8i64 + (VPSRAQZrr + (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)), + VR128X:$src2)), sub_xmm)>; + + def : Pat<(v4i64 (X86vsrai (v4i64 VR256X:$src1), (i8 imm:$src2))), + (EXTRACT_SUBREG (v8i64 + (VPSRAQZri + (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)), + imm:$src2)), sub_ymm)>; + + def : Pat<(v2i64 (X86vsrai (v2i64 VR128X:$src1), (i8 imm:$src2))), + (EXTRACT_SUBREG (v8i64 + (VPSRAQZri + (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)), + imm:$src2)), sub_xmm)>; +} + //===-------------------------------------------------------------------===// // Variable Bit Shifts //===-------------------------------------------------------------------===// diff --git a/llvm/test/CodeGen/X86/avx512-ext.ll b/llvm/test/CodeGen/X86/avx512-ext.ll index f1f984110608..b86b09833dba 100644 --- a/llvm/test/CodeGen/X86/avx512-ext.ll +++ b/llvm/test/CodeGen/X86/avx512-ext.ll @@ -491,8 +491,7 @@ define <2 x i64> @zext_2x8mem_to_2x64(<2 x i8> *%i , <2 x i1> %mask) nounwind re ; KNL-LABEL: zext_2x8mem_to_2x64: ; KNL: ## BB#0: ; KNL-NEXT: vpsllq $63, %xmm0, %xmm0 -; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 -; KNL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; KNL-NEXT: vpsraq $63, %zmm0, %zmm0 ; KNL-NEXT: vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero ; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 ; KNL-NEXT: retq @@ -512,8 +511,7 @@ define <2 x i64> @sext_2x8mem_to_2x64mask(<2 x i8> *%i , <2 x i1> %mask) nounwin ; KNL-LABEL: sext_2x8mem_to_2x64mask: ; KNL: ## BB#0: ; KNL-NEXT: vpsllq $63, %xmm0, %xmm0 -; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 -; KNL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; KNL-NEXT: vpsraq $63, %zmm0, %zmm0 ; KNL-NEXT: vpmovsxbq (%rdi), %xmm1 ; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 ; KNL-NEXT: retq @@ -872,8 +870,7 @@ define <2 x i64> @zext_2x16mem_to_2x64(<2 x i16> *%i , <2 x i1> %mask) nounwind ; KNL-LABEL: zext_2x16mem_to_2x64: ; KNL: ## BB#0: ; KNL-NEXT: vpsllq $63, %xmm0, %xmm0 -; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 -; KNL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; KNL-NEXT: vpsraq $63, %zmm0, %zmm0 ; KNL-NEXT: vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero ; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 ; KNL-NEXT: retq @@ -894,8 +891,7 @@ define <2 x i64> @sext_2x16mem_to_2x64mask(<2 x i16> *%i , <2 x i1> %mask) nounw ; KNL-LABEL: sext_2x16mem_to_2x64mask: ; KNL: ## BB#0: ; KNL-NEXT: vpsllq $63, %xmm0, %xmm0 -; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 -; KNL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; KNL-NEXT: vpsraq $63, %zmm0, %zmm0 ; KNL-NEXT: vpmovsxwq (%rdi), %xmm1 ; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 ; KNL-NEXT: retq @@ -1061,8 +1057,7 @@ define <2 x i64> @zext_2x32mem_to_2x64(<2 x i32> *%i , <2 x i1> %mask) nounwind ; KNL-LABEL: zext_2x32mem_to_2x64: ; KNL: ## BB#0: ; KNL-NEXT: vpsllq $63, %xmm0, %xmm0 -; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 -; KNL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; KNL-NEXT: vpsraq $63, %zmm0, %zmm0 ; KNL-NEXT: vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero ; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 ; KNL-NEXT: retq @@ -1083,8 +1078,7 @@ define <2 x i64> @sext_2x32mem_to_2x64mask(<2 x i32> *%i , <2 x i1> %mask) nounw ; KNL-LABEL: sext_2x32mem_to_2x64mask: ; KNL: ## BB#0: ; KNL-NEXT: vpsllq $63, %xmm0, %xmm0 -; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 -; KNL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; KNL-NEXT: vpsraq $63, %zmm0, %zmm0 ; KNL-NEXT: vpmovsxdq (%rdi), %xmm1 ; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 ; KNL-NEXT: retq diff --git a/llvm/test/CodeGen/X86/avx512dq-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512dq-intrinsics-upgrade.ll index f25acb603753..5cd81fafcf77 100644 --- a/llvm/test/CodeGen/X86/avx512dq-intrinsics-upgrade.ll +++ b/llvm/test/CodeGen/X86/avx512dq-intrinsics-upgrade.ll @@ -18,8 +18,7 @@ define <2 x double>@test_int_x86_avx512_mask_vextractf64x2_512(<8 x double> %x0, ; CHECK-NEXT: vmovq %rax, %xmm3 ; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] ; CHECK-NEXT: vpsllq $63, %xmm2, %xmm2 -; CHECK-NEXT: vpsrad $31, %xmm2, %xmm2 -; CHECK-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; CHECK-NEXT: vpsraq $63, %zmm2, %zmm2 ; CHECK-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm1 ; CHECK-NEXT: vandpd %xmm0, %xmm2, %xmm2 ; CHECK-NEXT: vaddpd %xmm0, %xmm1, %xmm0 diff --git a/llvm/test/CodeGen/X86/compress_expand.ll b/llvm/test/CodeGen/X86/compress_expand.ll index c1a3a1b92bbc..7331e2b51022 100644 --- a/llvm/test/CodeGen/X86/compress_expand.ll +++ b/llvm/test/CodeGen/X86/compress_expand.ll @@ -200,8 +200,7 @@ define void @test11(i64* %base, <2 x i64> %V, <2 x i1> %mask) { ; KNL: # BB#0: ; KNL-NEXT: # kill: %XMM0 %XMM0 %ZMM0 ; KNL-NEXT: vpsllq $63, %xmm1, %xmm1 -; KNL-NEXT: vpsrad $31, %xmm1, %xmm1 -; KNL-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; KNL-NEXT: vpsraq $63, %zmm1, %zmm1 ; KNL-NEXT: vpxord %zmm2, %zmm2, %zmm2 ; KNL-NEXT: vinserti32x4 $0, %xmm1, %zmm2, %zmm1 ; KNL-NEXT: vpsllq $63, %zmm1, %zmm1 diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll index ba520540d80f..a5e2cb66eba8 100644 --- a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll +++ b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll @@ -649,8 +649,7 @@ define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { ; AVX512-LABEL: splatvar_shift_v2i64: ; AVX512: # BB#0: ; AVX512-NEXT: # kill: %XMM0 %XMM0 %ZMM0 -; AVX512-NEXT: vpbroadcastq %xmm1, %xmm1 -; AVX512-NEXT: vpsravq %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: vpsraq %xmm1, %zmm0, %zmm0 ; AVX512-NEXT: # kill: %XMM0 %XMM0 %ZMM0 ; AVX512-NEXT: retq ; @@ -1556,9 +1555,9 @@ define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) nounwind { ; ; AVX512-LABEL: splatconstant_shift_v2i64: ; AVX512: # BB#0: -; AVX512-NEXT: vpsrad $7, %xmm0, %xmm1 -; AVX512-NEXT: vpsrlq $7, %xmm0, %xmm0 -; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] +; AVX512-NEXT: # kill: %XMM0 %XMM0 %ZMM0 +; AVX512-NEXT: vpsraq $7, %zmm0, %zmm0 +; AVX512-NEXT: # kill: %XMM0 %XMM0 %ZMM0 ; AVX512-NEXT: retq ; ; AVX512VL-LABEL: splatconstant_shift_v2i64: diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll index c09e6b2bc8d5..af3ddcf8048e 100644 --- a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll +++ b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll @@ -491,8 +491,7 @@ define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind { ; AVX512-LABEL: splatvar_shift_v4i64: ; AVX512: # BB#0: ; AVX512-NEXT: # kill: %YMM0 %YMM0 %ZMM0 -; AVX512-NEXT: vpbroadcastq %xmm1, %ymm1 -; AVX512-NEXT: vpsravq %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: vpsraq %xmm1, %zmm0, %zmm0 ; AVX512-NEXT: # kill: %YMM0 %YMM0 %ZMM0 ; AVX512-NEXT: retq ; @@ -1198,9 +1197,9 @@ define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) nounwind { ; ; AVX512-LABEL: splatconstant_shift_v4i64: ; AVX512: # BB#0: -; AVX512-NEXT: vpsrad $7, %ymm0, %ymm1 -; AVX512-NEXT: vpsrlq $7, %ymm0, %ymm0 -; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7] +; AVX512-NEXT: # kill: %YMM0 %YMM0 %ZMM0 +; AVX512-NEXT: vpsraq $7, %zmm0, %zmm0 +; AVX512-NEXT: # kill: %YMM0 %YMM0 %ZMM0 ; AVX512-NEXT: retq ; ; AVX512VL-LABEL: splatconstant_shift_v4i64: