forked from OSchip/llvm-project
[AVX-512] Don't use unmasked VMOVDQU8/16 for 8-bit or 16-bit element stores even when BWI instructions are supported. Always use VMOVDQA32/VMOVDQU32.
We were already using the 32 bit element opcode if BWI isn't enabled, but there's no reason to change opcode if we have BWI. We will still use the 8/16 opcodes for masked stores though. This allows us to use the aligned opcode when we can which makes our test output more consistent between different modes. It also reduces the number of isel patterns we need. This is a slight inconsistency with loads which default to 64 bit element opcodes. I'll probably rectify that in a future patch. Differential Revision: https://reviews.llvm.org/D35978 llvm-svn: 309693
This commit is contained in:
Craig Topper
parent
78b305f8d5
commit
2462a713ae
|
|
@ -3348,7 +3348,8 @@ multiclass avx512_load_vl<bits<8> opc, string OpcodeStr,
|
|||
}
|
||||
|
||||
multiclass avx512_store<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
|
||||
PatFrag st_frag, PatFrag mstore, string Name> {
|
||||
PatFrag st_frag, PatFrag mstore, string Name,
|
||||
bit NoMRPattern = 0> {
|
||||
|
||||
let hasSideEffects = 0 in {
|
||||
def rr_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst), (ins _.RC:$src),
|
||||
|
|
@ -3366,9 +3367,12 @@ multiclass avx512_store<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
|
|||
[], _.ExeDomain>, EVEX, EVEX_KZ, FoldGenData<Name#rrkz>;
|
||||
}
|
||||
|
||||
let hasSideEffects = 0, mayStore = 1 in
|
||||
def mr : AVX512PI<opc, MRMDestMem, (outs), (ins _.MemOp:$dst, _.RC:$src),
|
||||
!strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
|
||||
[(st_frag (_.VT _.RC:$src), addr:$dst)], _.ExeDomain>, EVEX;
|
||||
!if(NoMRPattern, [],
|
||||
[(st_frag (_.VT _.RC:$src), addr:$dst)]),
|
||||
_.ExeDomain>, EVEX;
|
||||
def mrk : AVX512PI<opc, MRMDestMem, (outs),
|
||||
(ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src),
|
||||
OpcodeStr # "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}",
|
||||
|
|
@ -3382,16 +3386,18 @@ multiclass avx512_store<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
|
|||
|
||||
multiclass avx512_store_vl< bits<8> opc, string OpcodeStr,
|
||||
AVX512VLVectorVTInfo _, Predicate prd,
|
||||
string Name> {
|
||||
string Name, bit NoMRPattern = 0> {
|
||||
let Predicates = [prd] in
|
||||
defm Z : avx512_store<opc, OpcodeStr, _.info512, store,
|
||||
masked_store_unaligned, Name#Z>, EVEX_V512;
|
||||
masked_store_unaligned, Name#Z, NoMRPattern>, EVEX_V512;
|
||||
|
||||
let Predicates = [prd, HasVLX] in {
|
||||
defm Z256 : avx512_store<opc, OpcodeStr, _.info256, store,
|
||||
masked_store_unaligned, Name#Z256>, EVEX_V256;
|
||||
masked_store_unaligned, Name#Z256,
|
||||
NoMRPattern>, EVEX_V256;
|
||||
defm Z128 : avx512_store<opc, OpcodeStr, _.info128, store,
|
||||
masked_store_unaligned, Name#Z128>, EVEX_V128;
|
||||
masked_store_unaligned, Name#Z128,
|
||||
NoMRPattern>, EVEX_V128;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3448,12 +3454,12 @@ defm VMOVDQA64 : avx512_alignedload_vl<0x6F, "vmovdqa64", avx512vl_i64_info,
|
|||
|
||||
defm VMOVDQU8 : avx512_load_vl<0x6F, "vmovdqu8", avx512vl_i8_info, HasBWI, 1>,
|
||||
avx512_store_vl<0x7F, "vmovdqu8", avx512vl_i8_info,
|
||||
HasBWI, "VMOVDQU8">,
|
||||
HasBWI, "VMOVDQU8", 1>,
|
||||
XD, EVEX_CD8<8, CD8VF>;
|
||||
|
||||
defm VMOVDQU16 : avx512_load_vl<0x6F, "vmovdqu16", avx512vl_i16_info, HasBWI, 1>,
|
||||
avx512_store_vl<0x7F, "vmovdqu16", avx512vl_i16_info,
|
||||
HasBWI, "VMOVDQU16">,
|
||||
HasBWI, "VMOVDQU16", 1>,
|
||||
XD, VEX_W, EVEX_CD8<16, CD8VF>;
|
||||
|
||||
defm VMOVDQU32 : avx512_load_vl<0x6F, "vmovdqu32", avx512vl_i32_info, HasAVX512,
|
||||
|
|
@ -3538,8 +3544,20 @@ def : Pat<(v8i32 (vselect (v8i1 VK8WM:$mask), (v8i32 VR256X:$src1),
|
|||
sub_ymm)>;
|
||||
}
|
||||
|
||||
let Predicates = [HasVLX, NoBWI] in {
|
||||
// 128-bit load/store without BWI.
|
||||
let Predicates = [HasAVX512] in {
|
||||
// 512-bit store.
|
||||
def : Pat<(alignedstore512 (v32i16 VR512:$src), addr:$dst),
|
||||
(VMOVDQA32Zmr addr:$dst, VR512:$src)>;
|
||||
def : Pat<(alignedstore512 (v64i8 VR512:$src), addr:$dst),
|
||||
(VMOVDQA32Zmr addr:$dst, VR512:$src)>;
|
||||
def : Pat<(store (v32i16 VR512:$src), addr:$dst),
|
||||
(VMOVDQU32Zmr addr:$dst, VR512:$src)>;
|
||||
def : Pat<(store (v64i8 VR512:$src), addr:$dst),
|
||||
(VMOVDQU32Zmr addr:$dst, VR512:$src)>;
|
||||
}
|
||||
|
||||
let Predicates = [HasVLX] in {
|
||||
// 128-bit store.
|
||||
def : Pat<(alignedstore (v8i16 VR128X:$src), addr:$dst),
|
||||
(VMOVDQA32Z128mr addr:$dst, VR128X:$src)>;
|
||||
def : Pat<(alignedstore (v16i8 VR128X:$src), addr:$dst),
|
||||
|
|
@ -3549,7 +3567,7 @@ let Predicates = [HasVLX, NoBWI] in {
|
|||
def : Pat<(store (v16i8 VR128X:$src), addr:$dst),
|
||||
(VMOVDQU32Z128mr addr:$dst, VR128X:$src)>;
|
||||
|
||||
// 256-bit load/store without BWI.
|
||||
// 256-bit store.
|
||||
def : Pat<(alignedstore256 (v16i16 VR256X:$src), addr:$dst),
|
||||
(VMOVDQA32Z256mr addr:$dst, VR256X:$src)>;
|
||||
def : Pat<(alignedstore256 (v32i8 VR256X:$src), addr:$dst),
|
||||
|
|
@ -3558,9 +3576,7 @@ let Predicates = [HasVLX, NoBWI] in {
|
|||
(VMOVDQU32Z256mr addr:$dst, VR256X:$src)>;
|
||||
def : Pat<(store (v32i8 VR256X:$src), addr:$dst),
|
||||
(VMOVDQU32Z256mr addr:$dst, VR256X:$src)>;
|
||||
}
|
||||
|
||||
let Predicates = [HasVLX] in {
|
||||
// Special patterns for storing subvector extracts of lower 128-bits of 256.
|
||||
// Its cheaper to just use VMOVAPS/VMOVUPS instead of VEXTRACTF128mr
|
||||
def : Pat<(alignedstore (v2f64 (extract_subvector
|
||||
|
|
|
|||
|
|
@ -712,7 +712,7 @@ define void @avg_v64i8(<64 x i8>* %a, <64 x i8>* %b) {
|
|||
; AVX512BW: # BB#0:
|
||||
; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm0
|
||||
; AVX512BW-NEXT: vpavgb (%rdi), %zmm0, %zmm0
|
||||
; AVX512BW-NEXT: vmovdqu8 %zmm0, (%rax)
|
||||
; AVX512BW-NEXT: vmovdqu32 %zmm0, (%rax)
|
||||
; AVX512BW-NEXT: vzeroupper
|
||||
; AVX512BW-NEXT: retq
|
||||
%1 = load <64 x i8>, <64 x i8>* %a
|
||||
|
|
@ -1101,7 +1101,7 @@ define void @avg_v32i16(<32 x i16>* %a, <32 x i16>* %b) {
|
|||
; AVX512BW: # BB#0:
|
||||
; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm0
|
||||
; AVX512BW-NEXT: vpavgw (%rdi), %zmm0, %zmm0
|
||||
; AVX512BW-NEXT: vmovdqu16 %zmm0, (%rax)
|
||||
; AVX512BW-NEXT: vmovdqu32 %zmm0, (%rax)
|
||||
; AVX512BW-NEXT: vzeroupper
|
||||
; AVX512BW-NEXT: retq
|
||||
%1 = load <32 x i16>, <32 x i16>* %a
|
||||
|
|
@ -1734,7 +1734,7 @@ define void @avg_v64i8_2(<64 x i8>* %a, <64 x i8>* %b) {
|
|||
; AVX512BW: # BB#0:
|
||||
; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm0
|
||||
; AVX512BW-NEXT: vpavgb %zmm0, %zmm0, %zmm0
|
||||
; AVX512BW-NEXT: vmovdqu8 %zmm0, (%rax)
|
||||
; AVX512BW-NEXT: vmovdqu32 %zmm0, (%rax)
|
||||
; AVX512BW-NEXT: vzeroupper
|
||||
; AVX512BW-NEXT: retq
|
||||
%1 = load <64 x i8>, <64 x i8>* %a
|
||||
|
|
@ -2124,7 +2124,7 @@ define void @avg_v32i16_2(<32 x i16>* %a, <32 x i16>* %b) {
|
|||
; AVX512BW: # BB#0:
|
||||
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
|
||||
; AVX512BW-NEXT: vpavgw (%rsi), %zmm0, %zmm0
|
||||
; AVX512BW-NEXT: vmovdqu16 %zmm0, (%rax)
|
||||
; AVX512BW-NEXT: vmovdqu32 %zmm0, (%rax)
|
||||
; AVX512BW-NEXT: vzeroupper
|
||||
; AVX512BW-NEXT: retq
|
||||
%1 = load <32 x i16>, <32 x i16>* %a
|
||||
|
|
@ -2649,7 +2649,7 @@ define void @avg_v64i8_const(<64 x i8>* %a) {
|
|||
; AVX512BW: # BB#0:
|
||||
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
|
||||
; AVX512BW-NEXT: vpavgb {{.*}}(%rip), %zmm0, %zmm0
|
||||
; AVX512BW-NEXT: vmovdqu8 %zmm0, (%rax)
|
||||
; AVX512BW-NEXT: vmovdqu32 %zmm0, (%rax)
|
||||
; AVX512BW-NEXT: vzeroupper
|
||||
; AVX512BW-NEXT: retq
|
||||
%1 = load <64 x i8>, <64 x i8>* %a
|
||||
|
|
@ -2957,7 +2957,7 @@ define void @avg_v32i16_const(<32 x i16>* %a) {
|
|||
; AVX512BW: # BB#0:
|
||||
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
|
||||
; AVX512BW-NEXT: vpavgw {{.*}}(%rip), %zmm0, %zmm0
|
||||
; AVX512BW-NEXT: vmovdqu16 %zmm0, (%rax)
|
||||
; AVX512BW-NEXT: vmovdqu32 %zmm0, (%rax)
|
||||
; AVX512BW-NEXT: vzeroupper
|
||||
; AVX512BW-NEXT: retq
|
||||
%1 = load <32 x i16>, <32 x i16>* %a
|
||||
|
|
|
|||
|
|
@ -1839,21 +1839,13 @@ define float @test_extractelement_variable_v16f32(<16 x float> %t1, i32 %index)
|
|||
}
|
||||
|
||||
define i16 @test_extractelement_variable_v8i16(<8 x i16> %t1, i32 %index) {
|
||||
; KNL-LABEL: test_extractelement_variable_v8i16:
|
||||
; KNL: ## BB#0:
|
||||
; KNL-NEXT: ## kill: %EDI<def> %EDI<kill> %RDI<def>
|
||||
; KNL-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
|
||||
; KNL-NEXT: andl $7, %edi
|
||||
; KNL-NEXT: movzwl -24(%rsp,%rdi,2), %eax
|
||||
; KNL-NEXT: retq
|
||||
;
|
||||
; SKX-LABEL: test_extractelement_variable_v8i16:
|
||||
; SKX: ## BB#0:
|
||||
; SKX-NEXT: ## kill: %EDI<def> %EDI<kill> %RDI<def>
|
||||
; SKX-NEXT: vmovdqu %xmm0, -{{[0-9]+}}(%rsp)
|
||||
; SKX-NEXT: andl $7, %edi
|
||||
; SKX-NEXT: movzwl -24(%rsp,%rdi,2), %eax
|
||||
; SKX-NEXT: retq
|
||||
; CHECK-LABEL: test_extractelement_variable_v8i16:
|
||||
; CHECK: ## BB#0:
|
||||
; CHECK-NEXT: ## kill: %EDI<def> %EDI<kill> %RDI<def>
|
||||
; CHECK-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
|
||||
; CHECK-NEXT: andl $7, %edi
|
||||
; CHECK-NEXT: movzwl -24(%rsp,%rdi,2), %eax
|
||||
; CHECK-NEXT: retq
|
||||
%t2 = extractelement <8 x i16> %t1, i32 %index
|
||||
ret i16 %t2
|
||||
}
|
||||
|
|
@ -1892,7 +1884,7 @@ define i16 @test_extractelement_variable_v16i16(<16 x i16> %t1, i32 %index) {
|
|||
; SKX-NEXT: andq $-32, %rsp
|
||||
; SKX-NEXT: subq $64, %rsp
|
||||
; SKX-NEXT: ## kill: %EDI<def> %EDI<kill> %RDI<def>
|
||||
; SKX-NEXT: vmovdqu %ymm0, (%rsp)
|
||||
; SKX-NEXT: vmovaps %ymm0, (%rsp)
|
||||
; SKX-NEXT: andl $15, %edi
|
||||
; SKX-NEXT: movzwl (%rsp,%rdi,2), %eax
|
||||
; SKX-NEXT: movq %rbp, %rsp
|
||||
|
|
@ -1938,7 +1930,7 @@ define i16 @test_extractelement_variable_v32i16(<32 x i16> %t1, i32 %index) {
|
|||
; SKX-NEXT: andq $-64, %rsp
|
||||
; SKX-NEXT: subq $128, %rsp
|
||||
; SKX-NEXT: ## kill: %EDI<def> %EDI<kill> %RDI<def>
|
||||
; SKX-NEXT: vmovdqu16 %zmm0, (%rsp)
|
||||
; SKX-NEXT: vmovaps %zmm0, (%rsp)
|
||||
; SKX-NEXT: andl $31, %edi
|
||||
; SKX-NEXT: movzwl (%rsp,%rdi,2), %eax
|
||||
; SKX-NEXT: movq %rbp, %rsp
|
||||
|
|
@ -1950,23 +1942,14 @@ define i16 @test_extractelement_variable_v32i16(<32 x i16> %t1, i32 %index) {
|
|||
}
|
||||
|
||||
define i8 @test_extractelement_variable_v16i8(<16 x i8> %t1, i32 %index) {
|
||||
; KNL-LABEL: test_extractelement_variable_v16i8:
|
||||
; KNL: ## BB#0:
|
||||
; KNL-NEXT: ## kill: %EDI<def> %EDI<kill> %RDI<def>
|
||||
; KNL-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
|
||||
; KNL-NEXT: andl $15, %edi
|
||||
; KNL-NEXT: leaq -{{[0-9]+}}(%rsp), %rax
|
||||
; KNL-NEXT: movb (%rdi,%rax), %al
|
||||
; KNL-NEXT: retq
|
||||
;
|
||||
; SKX-LABEL: test_extractelement_variable_v16i8:
|
||||
; SKX: ## BB#0:
|
||||
; SKX-NEXT: ## kill: %EDI<def> %EDI<kill> %RDI<def>
|
||||
; SKX-NEXT: vmovdqu %xmm0, -{{[0-9]+}}(%rsp)
|
||||
; SKX-NEXT: andl $15, %edi
|
||||
; SKX-NEXT: leaq -{{[0-9]+}}(%rsp), %rax
|
||||
; SKX-NEXT: movb (%rdi,%rax), %al
|
||||
; SKX-NEXT: retq
|
||||
; CHECK-LABEL: test_extractelement_variable_v16i8:
|
||||
; CHECK: ## BB#0:
|
||||
; CHECK-NEXT: ## kill: %EDI<def> %EDI<kill> %RDI<def>
|
||||
; CHECK-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
|
||||
; CHECK-NEXT: andl $15, %edi
|
||||
; CHECK-NEXT: leaq -{{[0-9]+}}(%rsp), %rax
|
||||
; CHECK-NEXT: movb (%rdi,%rax), %al
|
||||
; CHECK-NEXT: retq
|
||||
%t2 = extractelement <16 x i8> %t1, i32 %index
|
||||
ret i8 %t2
|
||||
}
|
||||
|
|
@ -2006,7 +1989,7 @@ define i8 @test_extractelement_variable_v32i8(<32 x i8> %t1, i32 %index) {
|
|||
; SKX-NEXT: andq $-32, %rsp
|
||||
; SKX-NEXT: subq $64, %rsp
|
||||
; SKX-NEXT: ## kill: %EDI<def> %EDI<kill> %RDI<def>
|
||||
; SKX-NEXT: vmovdqu %ymm0, (%rsp)
|
||||
; SKX-NEXT: vmovaps %ymm0, (%rsp)
|
||||
; SKX-NEXT: andl $31, %edi
|
||||
; SKX-NEXT: movq %rsp, %rax
|
||||
; SKX-NEXT: movb (%rdi,%rax), %al
|
||||
|
|
@ -2055,7 +2038,7 @@ define i8 @test_extractelement_variable_v64i8(<64 x i8> %t1, i32 %index) {
|
|||
; SKX-NEXT: andq $-64, %rsp
|
||||
; SKX-NEXT: subq $128, %rsp
|
||||
; SKX-NEXT: ## kill: %EDI<def> %EDI<kill> %RDI<def>
|
||||
; SKX-NEXT: vmovdqu8 %zmm0, (%rsp)
|
||||
; SKX-NEXT: vmovaps %zmm0, (%rsp)
|
||||
; SKX-NEXT: andl $63, %edi
|
||||
; SKX-NEXT: movq %rsp, %rax
|
||||
; SKX-NEXT: movb (%rdi,%rax), %al
|
||||
|
|
@ -2105,7 +2088,7 @@ define i8 @test_extractelement_variable_v64i8_indexi8(<64 x i8> %t1, i8 %index)
|
|||
; SKX-NEXT: andq $-64, %rsp
|
||||
; SKX-NEXT: subq $128, %rsp
|
||||
; SKX-NEXT: addb %dil, %dil
|
||||
; SKX-NEXT: vmovdqu8 %zmm0, (%rsp)
|
||||
; SKX-NEXT: vmovaps %zmm0, (%rsp)
|
||||
; SKX-NEXT: movzbl %dil, %eax
|
||||
; SKX-NEXT: andl $63, %eax
|
||||
; SKX-NEXT: movq %rsp, %rcx
|
||||
|
|
@ -2330,7 +2313,7 @@ define zeroext i8 @test_extractelement_varible_v32i1(<32 x i8> %a, <32 x i8> %b,
|
|||
; SKX-NEXT: ## kill: %EDI<def> %EDI<kill> %RDI<def>
|
||||
; SKX-NEXT: vpcmpnleub %ymm1, %ymm0, %k0
|
||||
; SKX-NEXT: vpmovm2w %k0, %zmm0
|
||||
; SKX-NEXT: vmovdqu16 %zmm0, (%rsp)
|
||||
; SKX-NEXT: vmovdqa32 %zmm0, (%rsp)
|
||||
; SKX-NEXT: andl $31, %edi
|
||||
; SKX-NEXT: movzbl (%rsp,%rdi,2), %eax
|
||||
; SKX-NEXT: andl $1, %eax
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ define zeroext i8 @test_extractelement_varible_v64i1(<64 x i8> %a, <64 x i8> %b,
|
|||
; SKX-NEXT: ## kill: %EDI<def> %EDI<kill> %RDI<def>
|
||||
; SKX-NEXT: vpcmpnleub %zmm1, %zmm0, %k0
|
||||
; SKX-NEXT: vpmovm2b %k0, %zmm0
|
||||
; SKX-NEXT: vmovdqu8 %zmm0, (%rsp)
|
||||
; SKX-NEXT: vmovdqa32 %zmm0, (%rsp)
|
||||
; SKX-NEXT: andl $63, %edi
|
||||
; SKX-NEXT: movq %rsp, %rax
|
||||
; SKX-NEXT: movzbl (%rdi,%rax), %eax
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ define void@test_int_x86_avx512_mask_storeu_b_512(i8* %ptr1, i8* %ptr2, <64 x i8
|
|||
; AVX512BW: ## BB#0:
|
||||
; AVX512BW-NEXT: kmovq %rdx, %k1
|
||||
; AVX512BW-NEXT: vmovdqu8 %zmm0, (%rdi) {%k1}
|
||||
; AVX512BW-NEXT: vmovdqu8 %zmm0, (%rsi)
|
||||
; AVX512BW-NEXT: vmovdqu32 %zmm0, (%rsi)
|
||||
; AVX512BW-NEXT: retq
|
||||
;
|
||||
; AVX512F-32-LABEL: test_int_x86_avx512_mask_storeu_b_512:
|
||||
|
|
@ -18,7 +18,7 @@ define void@test_int_x86_avx512_mask_storeu_b_512(i8* %ptr1, i8* %ptr2, <64 x i8
|
|||
; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
||||
; AVX512F-32-NEXT: kmovq {{[0-9]+}}(%esp), %k1
|
||||
; AVX512F-32-NEXT: vmovdqu8 %zmm0, (%ecx) {%k1}
|
||||
; AVX512F-32-NEXT: vmovdqu8 %zmm0, (%eax)
|
||||
; AVX512F-32-NEXT: vmovdqu32 %zmm0, (%eax)
|
||||
; AVX512F-32-NEXT: retl
|
||||
call void @llvm.x86.avx512.mask.storeu.b.512(i8* %ptr1, <64 x i8> %x1, i64 %x2)
|
||||
call void @llvm.x86.avx512.mask.storeu.b.512(i8* %ptr2, <64 x i8> %x1, i64 -1)
|
||||
|
|
@ -32,7 +32,7 @@ define void@test_int_x86_avx512_mask_storeu_w_512(i8* %ptr1, i8* %ptr2, <32 x i1
|
|||
; AVX512BW: ## BB#0:
|
||||
; AVX512BW-NEXT: kmovd %edx, %k1
|
||||
; AVX512BW-NEXT: vmovdqu16 %zmm0, (%rdi) {%k1}
|
||||
; AVX512BW-NEXT: vmovdqu16 %zmm0, (%rsi)
|
||||
; AVX512BW-NEXT: vmovdqu32 %zmm0, (%rsi)
|
||||
; AVX512BW-NEXT: retq
|
||||
;
|
||||
; AVX512F-32-LABEL: test_int_x86_avx512_mask_storeu_w_512:
|
||||
|
|
@ -41,7 +41,7 @@ define void@test_int_x86_avx512_mask_storeu_w_512(i8* %ptr1, i8* %ptr2, <32 x i1
|
|||
; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
||||
; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1
|
||||
; AVX512F-32-NEXT: vmovdqu16 %zmm0, (%ecx) {%k1}
|
||||
; AVX512F-32-NEXT: vmovdqu16 %zmm0, (%eax)
|
||||
; AVX512F-32-NEXT: vmovdqu32 %zmm0, (%eax)
|
||||
; AVX512F-32-NEXT: retl
|
||||
call void @llvm.x86.avx512.mask.storeu.w.512(i8* %ptr1, <32 x i16> %x1, i32 %x2)
|
||||
call void @llvm.x86.avx512.mask.storeu.w.512(i8* %ptr2, <32 x i16> %x1, i32 -1)
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ define <64 x i8> @test1(i8 * %addr) {
|
|||
define void @test2(i8 * %addr, <64 x i8> %data) {
|
||||
; CHECK-LABEL: test2:
|
||||
; CHECK: ## BB#0:
|
||||
; CHECK-NEXT: vmovdqu8 %zmm0, (%rdi)
|
||||
; CHECK-NEXT: vmovups %zmm0, (%rdi)
|
||||
; CHECK-NEXT: retq
|
||||
%vaddr = bitcast i8* %addr to <64 x i8>*
|
||||
store <64 x i8>%data, <64 x i8>* %vaddr, align 1
|
||||
|
|
@ -62,7 +62,7 @@ define <32 x i16> @test5(i8 * %addr) {
|
|||
define void @test6(i8 * %addr, <32 x i16> %data) {
|
||||
; CHECK-LABEL: test6:
|
||||
; CHECK: ## BB#0:
|
||||
; CHECK-NEXT: vmovdqu16 %zmm0, (%rdi)
|
||||
; CHECK-NEXT: vmovups %zmm0, (%rdi)
|
||||
; CHECK-NEXT: retq
|
||||
%vaddr = bitcast i8* %addr to <32 x i16>*
|
||||
store <32 x i16>%data, <32 x i16>* %vaddr, align 1
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ define <32 x i8> @test_256_1(i8 * %addr) {
|
|||
define void @test_256_2(i8 * %addr, <32 x i8> %data) {
|
||||
; CHECK-LABEL: test_256_2:
|
||||
; CHECK: ## BB#0:
|
||||
; CHECK-NEXT: vmovdqu %ymm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfe,0x7f,0x07]
|
||||
; CHECK-NEXT: vmovups %ymm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfc,0x11,0x07]
|
||||
; CHECK-NEXT: retq ## encoding: [0xc3]
|
||||
%vaddr = bitcast i8* %addr to <32 x i8>*
|
||||
store <32 x i8>%data, <32 x i8>* %vaddr, align 1
|
||||
|
|
@ -62,7 +62,7 @@ define <16 x i16> @test_256_5(i8 * %addr) {
|
|||
define void @test_256_6(i8 * %addr, <16 x i16> %data) {
|
||||
; CHECK-LABEL: test_256_6:
|
||||
; CHECK: ## BB#0:
|
||||
; CHECK-NEXT: vmovdqu %ymm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfe,0x7f,0x07]
|
||||
; CHECK-NEXT: vmovups %ymm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfc,0x11,0x07]
|
||||
; CHECK-NEXT: retq ## encoding: [0xc3]
|
||||
%vaddr = bitcast i8* %addr to <16 x i16>*
|
||||
store <16 x i16>%data, <16 x i16>* %vaddr, align 1
|
||||
|
|
@ -110,7 +110,7 @@ define <16 x i8> @test_128_1(i8 * %addr) {
|
|||
define void @test_128_2(i8 * %addr, <16 x i8> %data) {
|
||||
; CHECK-LABEL: test_128_2:
|
||||
; CHECK: ## BB#0:
|
||||
; CHECK-NEXT: vmovdqu %xmm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7f,0x07]
|
||||
; CHECK-NEXT: vmovups %xmm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x11,0x07]
|
||||
; CHECK-NEXT: retq ## encoding: [0xc3]
|
||||
%vaddr = bitcast i8* %addr to <16 x i8>*
|
||||
store <16 x i8>%data, <16 x i8>* %vaddr, align 1
|
||||
|
|
@ -158,7 +158,7 @@ define <8 x i16> @test_128_5(i8 * %addr) {
|
|||
define void @test_128_6(i8 * %addr, <8 x i16> %data) {
|
||||
; CHECK-LABEL: test_128_6:
|
||||
; CHECK: ## BB#0:
|
||||
; CHECK-NEXT: vmovdqu %xmm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7f,0x07]
|
||||
; CHECK-NEXT: vmovups %xmm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x11,0x07]
|
||||
; CHECK-NEXT: retq ## encoding: [0xc3]
|
||||
%vaddr = bitcast i8* %addr to <8 x i16>*
|
||||
store <8 x i16>%data, <8 x i16>* %vaddr, align 1
|
||||
|
|
|
|||
|
|
@ -894,69 +894,21 @@ define <8 x i32> @test_broadcast_4i32_8i32_reuse(<4 x i32>* %p0, <4 x i32>* %p1)
|
|||
}
|
||||
|
||||
define <16 x i16> @test_broadcast_8i16_16i16_reuse(<8 x i16> *%p0, <8 x i16> *%p1) nounwind {
|
||||
; X32-AVX-LABEL: test_broadcast_8i16_16i16_reuse:
|
||||
; X32-AVX: ## BB#0:
|
||||
; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
|
||||
; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
||||
; X32-AVX-NEXT: vmovaps (%ecx), %xmm0
|
||||
; X32-AVX-NEXT: vmovaps %xmm0, (%eax)
|
||||
; X32-AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X32-AVX-NEXT: retl
|
||||
; X32-LABEL: test_broadcast_8i16_16i16_reuse:
|
||||
; X32: ## BB#0:
|
||||
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
|
||||
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
||||
; X32-NEXT: vmovaps (%ecx), %xmm0
|
||||
; X32-NEXT: vmovaps %xmm0, (%eax)
|
||||
; X32-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X32-NEXT: retl
|
||||
;
|
||||
; X32-AVX512F-LABEL: test_broadcast_8i16_16i16_reuse:
|
||||
; X32-AVX512F: ## BB#0:
|
||||
; X32-AVX512F-NEXT: movl {{[0-9]+}}(%esp), %eax
|
||||
; X32-AVX512F-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
||||
; X32-AVX512F-NEXT: vmovaps (%ecx), %xmm0
|
||||
; X32-AVX512F-NEXT: vmovaps %xmm0, (%eax)
|
||||
; X32-AVX512F-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X32-AVX512F-NEXT: retl
|
||||
;
|
||||
; X32-AVX512BW-LABEL: test_broadcast_8i16_16i16_reuse:
|
||||
; X32-AVX512BW: ## BB#0:
|
||||
; X32-AVX512BW-NEXT: movl {{[0-9]+}}(%esp), %eax
|
||||
; X32-AVX512BW-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
||||
; X32-AVX512BW-NEXT: vmovdqa (%ecx), %xmm0
|
||||
; X32-AVX512BW-NEXT: vmovdqu %xmm0, (%eax)
|
||||
; X32-AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X32-AVX512BW-NEXT: retl
|
||||
;
|
||||
; X32-AVX512DQ-LABEL: test_broadcast_8i16_16i16_reuse:
|
||||
; X32-AVX512DQ: ## BB#0:
|
||||
; X32-AVX512DQ-NEXT: movl {{[0-9]+}}(%esp), %eax
|
||||
; X32-AVX512DQ-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
||||
; X32-AVX512DQ-NEXT: vmovaps (%ecx), %xmm0
|
||||
; X32-AVX512DQ-NEXT: vmovaps %xmm0, (%eax)
|
||||
; X32-AVX512DQ-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X32-AVX512DQ-NEXT: retl
|
||||
;
|
||||
; X64-AVX-LABEL: test_broadcast_8i16_16i16_reuse:
|
||||
; X64-AVX: ## BB#0:
|
||||
; X64-AVX-NEXT: vmovaps (%rdi), %xmm0
|
||||
; X64-AVX-NEXT: vmovaps %xmm0, (%rsi)
|
||||
; X64-AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X64-AVX-NEXT: retq
|
||||
;
|
||||
; X64-AVX512F-LABEL: test_broadcast_8i16_16i16_reuse:
|
||||
; X64-AVX512F: ## BB#0:
|
||||
; X64-AVX512F-NEXT: vmovaps (%rdi), %xmm0
|
||||
; X64-AVX512F-NEXT: vmovaps %xmm0, (%rsi)
|
||||
; X64-AVX512F-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X64-AVX512F-NEXT: retq
|
||||
;
|
||||
; X64-AVX512BW-LABEL: test_broadcast_8i16_16i16_reuse:
|
||||
; X64-AVX512BW: ## BB#0:
|
||||
; X64-AVX512BW-NEXT: vmovdqa (%rdi), %xmm0
|
||||
; X64-AVX512BW-NEXT: vmovdqu %xmm0, (%rsi)
|
||||
; X64-AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X64-AVX512BW-NEXT: retq
|
||||
;
|
||||
; X64-AVX512DQ-LABEL: test_broadcast_8i16_16i16_reuse:
|
||||
; X64-AVX512DQ: ## BB#0:
|
||||
; X64-AVX512DQ-NEXT: vmovaps (%rdi), %xmm0
|
||||
; X64-AVX512DQ-NEXT: vmovaps %xmm0, (%rsi)
|
||||
; X64-AVX512DQ-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X64-AVX512DQ-NEXT: retq
|
||||
; X64-LABEL: test_broadcast_8i16_16i16_reuse:
|
||||
; X64: ## BB#0:
|
||||
; X64-NEXT: vmovaps (%rdi), %xmm0
|
||||
; X64-NEXT: vmovaps %xmm0, (%rsi)
|
||||
; X64-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X64-NEXT: retq
|
||||
%1 = load <8 x i16>, <8 x i16> *%p0
|
||||
store <8 x i16> %1, <8 x i16>* %p1
|
||||
%2 = shufflevector <8 x i16> %1, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
||||
|
|
@ -964,69 +916,21 @@ define <16 x i16> @test_broadcast_8i16_16i16_reuse(<8 x i16> *%p0, <8 x i16> *%p
|
|||
}
|
||||
|
||||
define <32 x i8> @test_broadcast_16i8_32i8_reuse(<16 x i8> *%p0, <16 x i8> *%p1) nounwind {
|
||||
; X32-AVX-LABEL: test_broadcast_16i8_32i8_reuse:
|
||||
; X32-AVX: ## BB#0:
|
||||
; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
|
||||
; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
||||
; X32-AVX-NEXT: vmovaps (%ecx), %xmm0
|
||||
; X32-AVX-NEXT: vmovaps %xmm0, (%eax)
|
||||
; X32-AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X32-AVX-NEXT: retl
|
||||
; X32-LABEL: test_broadcast_16i8_32i8_reuse:
|
||||
; X32: ## BB#0:
|
||||
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
|
||||
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
||||
; X32-NEXT: vmovaps (%ecx), %xmm0
|
||||
; X32-NEXT: vmovaps %xmm0, (%eax)
|
||||
; X32-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X32-NEXT: retl
|
||||
;
|
||||
; X32-AVX512F-LABEL: test_broadcast_16i8_32i8_reuse:
|
||||
; X32-AVX512F: ## BB#0:
|
||||
; X32-AVX512F-NEXT: movl {{[0-9]+}}(%esp), %eax
|
||||
; X32-AVX512F-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
||||
; X32-AVX512F-NEXT: vmovaps (%ecx), %xmm0
|
||||
; X32-AVX512F-NEXT: vmovaps %xmm0, (%eax)
|
||||
; X32-AVX512F-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X32-AVX512F-NEXT: retl
|
||||
;
|
||||
; X32-AVX512BW-LABEL: test_broadcast_16i8_32i8_reuse:
|
||||
; X32-AVX512BW: ## BB#0:
|
||||
; X32-AVX512BW-NEXT: movl {{[0-9]+}}(%esp), %eax
|
||||
; X32-AVX512BW-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
||||
; X32-AVX512BW-NEXT: vmovdqa (%ecx), %xmm0
|
||||
; X32-AVX512BW-NEXT: vmovdqu %xmm0, (%eax)
|
||||
; X32-AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X32-AVX512BW-NEXT: retl
|
||||
;
|
||||
; X32-AVX512DQ-LABEL: test_broadcast_16i8_32i8_reuse:
|
||||
; X32-AVX512DQ: ## BB#0:
|
||||
; X32-AVX512DQ-NEXT: movl {{[0-9]+}}(%esp), %eax
|
||||
; X32-AVX512DQ-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
||||
; X32-AVX512DQ-NEXT: vmovaps (%ecx), %xmm0
|
||||
; X32-AVX512DQ-NEXT: vmovaps %xmm0, (%eax)
|
||||
; X32-AVX512DQ-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X32-AVX512DQ-NEXT: retl
|
||||
;
|
||||
; X64-AVX-LABEL: test_broadcast_16i8_32i8_reuse:
|
||||
; X64-AVX: ## BB#0:
|
||||
; X64-AVX-NEXT: vmovaps (%rdi), %xmm0
|
||||
; X64-AVX-NEXT: vmovaps %xmm0, (%rsi)
|
||||
; X64-AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X64-AVX-NEXT: retq
|
||||
;
|
||||
; X64-AVX512F-LABEL: test_broadcast_16i8_32i8_reuse:
|
||||
; X64-AVX512F: ## BB#0:
|
||||
; X64-AVX512F-NEXT: vmovaps (%rdi), %xmm0
|
||||
; X64-AVX512F-NEXT: vmovaps %xmm0, (%rsi)
|
||||
; X64-AVX512F-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X64-AVX512F-NEXT: retq
|
||||
;
|
||||
; X64-AVX512BW-LABEL: test_broadcast_16i8_32i8_reuse:
|
||||
; X64-AVX512BW: ## BB#0:
|
||||
; X64-AVX512BW-NEXT: vmovdqa (%rdi), %xmm0
|
||||
; X64-AVX512BW-NEXT: vmovdqu %xmm0, (%rsi)
|
||||
; X64-AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X64-AVX512BW-NEXT: retq
|
||||
;
|
||||
; X64-AVX512DQ-LABEL: test_broadcast_16i8_32i8_reuse:
|
||||
; X64-AVX512DQ: ## BB#0:
|
||||
; X64-AVX512DQ-NEXT: vmovaps (%rdi), %xmm0
|
||||
; X64-AVX512DQ-NEXT: vmovaps %xmm0, (%rsi)
|
||||
; X64-AVX512DQ-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X64-AVX512DQ-NEXT: retq
|
||||
; X64-LABEL: test_broadcast_16i8_32i8_reuse:
|
||||
; X64: ## BB#0:
|
||||
; X64-NEXT: vmovaps (%rdi), %xmm0
|
||||
; X64-NEXT: vmovaps %xmm0, (%rsi)
|
||||
; X64-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
||||
; X64-NEXT: retq
|
||||
%1 = load <16 x i8>, <16 x i8> *%p0
|
||||
store <16 x i8> %1, <16 x i8>* %p1
|
||||
%2 = shufflevector <16 x i8> %1, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
|
|
|
|||
|
|
@ -361,8 +361,8 @@ define void @interleaved_store_vf32_i8_stride4(<32 x i8> %x1, <32 x i8> %x2, <32
|
|||
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm5[2,3]
|
||||
; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm2, %zmm2
|
||||
; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
|
||||
; AVX512-NEXT: vmovdqu8 %zmm0, 64(%rdi)
|
||||
; AVX512-NEXT: vmovdqu8 %zmm2, (%rdi)
|
||||
; AVX512-NEXT: vmovdqa32 %zmm0, 64(%rdi)
|
||||
; AVX512-NEXT: vmovdqa32 %zmm2, (%rdi)
|
||||
; AVX512-NEXT: vzeroupper
|
||||
; AVX512-NEXT: retq
|
||||
%v1 = shufflevector <32 x i8> %x1, <32 x i8> %x2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
|
||||
|
|
@ -444,7 +444,7 @@ define void @interleaved_store_vf16_i8_stride4(<16 x i8> %x1, <16 x i8> %x2, <16
|
|||
; AVX512-NEXT: vpblendvb %ymm8, %ymm0, %ymm3, %ymm0
|
||||
; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
|
||||
; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
|
||||
; AVX512-NEXT: vmovdqu8 %zmm0, (%rdi)
|
||||
; AVX512-NEXT: vmovdqa32 %zmm0, (%rdi)
|
||||
; AVX512-NEXT: vzeroupper
|
||||
; AVX512-NEXT: retq
|
||||
%v1 = shufflevector <16 x i8> %x1, <16 x i8> %x2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
|
||||
|
|
|
|||
Loading…
Reference in New Issue