diff --git a/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll b/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll index 7487cd98e0aa..2a4ede05acd6 100644 --- a/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll +++ b/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s @@ -5,12 +6,23 @@ declare i32 @llvm.amdgcn.workitem.id.x() #0 @lds.obj = addrspace(3) global [256 x i32] undef, align 4 -; GCN-LABEL: {{^}}write_ds_sub0_offset0_global: -; GCN: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], 2, v0 -; GCN: v_sub_{{[iu]}}32_e32 [[BASEPTR:v[0-9]+]], {{(vcc, )?}}lds.obj@abs32@lo, [[SHL]] -; GCN: v_mov_b32_e32 [[VAL:v[0-9]+]], 0x7b -; GCN: ds_write_b32 [[BASEPTR]], [[VAL]] offset:12 define amdgpu_kernel void @write_ds_sub0_offset0_global() #0 { +; CI-LABEL: write_ds_sub0_offset0_global: +; CI: ; %bb.0: ; %entry +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_sub_i32_e32 v0, vcc, lds.obj@abs32@lo, v0 +; CI-NEXT: v_mov_b32_e32 v1, 0x7b +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_write_b32 v0, v1 offset:12 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: write_ds_sub0_offset0_global: +; GFX9: ; %bb.0: ; %entry +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: v_sub_u32_e32 v0, lds.obj@abs32@lo, v0 +; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b +; GFX9-NEXT: ds_write_b32 v0, v1 offset:12 +; GFX9-NEXT: s_endpgm entry: %x.i = call i32 @llvm.amdgcn.workitem.id.x() #1 %sub1 = sub i32 0, %x.i @@ -20,10 +32,41 @@ entry: ret void } -; GFX9-LABEL: {{^}}write_ds_sub0_offset0_global_clamp_bit: -; GFX9: v_sub_u32 -; GFX9: s_endpgm define amdgpu_kernel void @write_ds_sub0_offset0_global_clamp_bit(float %dummy.val) #0 { +; CI-LABEL: write_ds_sub0_offset0_global_clamp_bit: +; CI: ; %bb.0: ; %entry +; CI-NEXT: s_load_dword s0, s[0:1], 0x9 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_sub_i32_e32 v0, vcc, lds.obj@abs32@lo, v0 +; CI-NEXT: s_mov_b64 vcc, 0 +; CI-NEXT: v_mov_b32_e32 v2, 0x7b +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_mov_b32_e32 v1, s0 +; CI-NEXT: s_mov_b32 s0, 0 +; CI-NEXT: v_div_fmas_f32 v1, v1, v1, v1 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, -1 +; CI-NEXT: s_mov_b32 s1, s0 +; CI-NEXT: ds_write_b32 v0, v2 offset:12 +; CI-NEXT: buffer_store_dword v1, off, s[0:3], 0 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: write_ds_sub0_offset0_global_clamp_bit: +; GFX9: ; %bb.0: ; %entry +; GFX9-NEXT: s_load_dword s0, s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 vcc, 0 +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: v_sub_u32_e32 v0, lds.obj@abs32@lo, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: v_div_fmas_f32 v2, v1, v1, v1 +; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b +; GFX9-NEXT: ds_write_b32 v0, v1 offset:12 +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: global_store_dword v[0:1], v2, off +; GFX9-NEXT: s_endpgm entry: %x.i = call i32 @llvm.amdgcn.workitem.id.x() #1 %sub1 = sub i32 0, %x.i @@ -35,13 +78,23 @@ entry: ret void } -; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_max_offset: -; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0 -; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]] -; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SCALED]] -; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 13 -; GCN: ds_write_b8 [[NEG]], [[K]] offset:65535 define amdgpu_kernel void @add_x_shl_neg_to_sub_max_offset() #1 { +; CI-LABEL: add_x_shl_neg_to_sub_max_offset: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_sub_i32_e32 v0, vcc, 0, v0 +; CI-NEXT: v_mov_b32_e32 v1, 13 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_write_b8 v0, v1 offset:65535 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: add_x_shl_neg_to_sub_max_offset: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: v_sub_u32_e32 v0, 0, v0 +; GFX9-NEXT: v_mov_b32_e32 v1, 13 +; GFX9-NEXT: ds_write_b8 v0, v1 offset:65535 +; GFX9-NEXT: s_endpgm %x.i = call i32 @llvm.amdgcn.workitem.id.x() #0 %neg = sub i32 0, %x.i %shl = shl i32 %neg, 2 @@ -51,13 +104,23 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_max_offset() #1 { ret void } -; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_max_offset_p1: -; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0 -; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0x10000, [[SCALED]] -; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0x10000, [[SCALED]] -; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 13 -; GCN: ds_write_b8 [[NEG]], [[K]]{{$}} define amdgpu_kernel void @add_x_shl_neg_to_sub_max_offset_p1() #1 { +; CI-LABEL: add_x_shl_neg_to_sub_max_offset_p1: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_sub_i32_e32 v0, vcc, 0x10000, v0 +; CI-NEXT: v_mov_b32_e32 v1, 13 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_write_b8 v0, v1 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: add_x_shl_neg_to_sub_max_offset_p1: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: v_sub_u32_e32 v0, 0x10000, v0 +; GFX9-NEXT: v_mov_b32_e32 v1, 13 +; GFX9-NEXT: ds_write_b8 v0, v1 +; GFX9-NEXT: s_endpgm %x.i = call i32 @llvm.amdgcn.workitem.id.x() #0 %neg = sub i32 0, %x.i %shl = shl i32 %neg, 2 @@ -67,17 +130,25 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_max_offset_p1() #1 { ret void } -; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_multi_use: -; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0 -; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]] -; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SCALED]] -; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 13 -; GCN-NOT: v_sub -; GCN: ds_write_b32 [[NEG]], [[K]] offset:123{{$}} -; GCN-NOT: v_sub -; GCN: ds_write_b32 [[NEG]], [[K]] offset:456{{$}} -; GCN: s_endpgm define amdgpu_kernel void @add_x_shl_neg_to_sub_multi_use() #1 { +; CI-LABEL: add_x_shl_neg_to_sub_multi_use: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_sub_i32_e32 v0, vcc, 0, v0 +; CI-NEXT: v_mov_b32_e32 v1, 13 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_write_b32 v0, v1 offset:123 +; CI-NEXT: ds_write_b32 v0, v1 offset:456 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: add_x_shl_neg_to_sub_multi_use: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: v_sub_u32_e32 v0, 0, v0 +; GFX9-NEXT: v_mov_b32_e32 v1, 13 +; GFX9-NEXT: ds_write_b32 v0, v1 offset:123 +; GFX9-NEXT: ds_write_b32 v0, v1 offset:456 +; GFX9-NEXT: s_endpgm %x.i = call i32 @llvm.amdgcn.workitem.id.x() #0 %neg = sub i32 0, %x.i %shl = shl i32 %neg, 2 @@ -90,17 +161,25 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_multi_use() #1 { ret void } -; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_multi_use_same_offset: -; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0 -; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]] -; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SCALED]] -; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 13 -; GCN-NOT: v_sub -; GCN: ds_write_b32 [[NEG]], [[K]] offset:123{{$}} -; GCN-NOT: v_sub -; GCN: ds_write_b32 [[NEG]], [[K]] offset:123{{$}} -; GCN: s_endpgm define amdgpu_kernel void @add_x_shl_neg_to_sub_multi_use_same_offset() #1 { +; CI-LABEL: add_x_shl_neg_to_sub_multi_use_same_offset: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_sub_i32_e32 v0, vcc, 0, v0 +; CI-NEXT: v_mov_b32_e32 v1, 13 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_write_b32 v0, v1 offset:123 +; CI-NEXT: ds_write_b32 v0, v1 offset:123 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: add_x_shl_neg_to_sub_multi_use_same_offset: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: v_sub_u32_e32 v0, 0, v0 +; GFX9-NEXT: v_mov_b32_e32 v1, 13 +; GFX9-NEXT: ds_write_b32 v0, v1 offset:123 +; GFX9-NEXT: ds_write_b32 v0, v1 offset:123 +; GFX9-NEXT: s_endpgm %x.i = call i32 @llvm.amdgcn.workitem.id.x() #0 %neg = sub i32 0, %x.i %shl = shl i32 %neg, 2 @@ -111,12 +190,25 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_multi_use_same_offset() #1 { ret void } -; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_misaligned_i64_max_offset: -; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0 -; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]] -; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SCALED]] -; GCN: ds_write2_b32 [[NEG]], {{v[0-9]+}}, {{v[0-9]+}} offset0:254 offset1:255 define amdgpu_kernel void @add_x_shl_neg_to_sub_misaligned_i64_max_offset() #1 { +; CI-LABEL: add_x_shl_neg_to_sub_misaligned_i64_max_offset: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_sub_i32_e32 v0, vcc, 0, v0 +; CI-NEXT: v_mov_b32_e32 v1, 0x7b +; CI-NEXT: v_mov_b32_e32 v2, 0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_write2_b32 v0, v1, v2 offset0:254 offset1:255 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: add_x_shl_neg_to_sub_misaligned_i64_max_offset: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: v_sub_u32_e32 v0, 0, v0 +; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b +; GFX9-NEXT: v_mov_b32_e32 v2, 0 +; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset0:254 offset1:255 +; GFX9-NEXT: s_endpgm %x.i = call i32 @llvm.amdgcn.workitem.id.x() #0 %neg = sub i32 0, %x.i %shl = shl i32 %neg, 2 @@ -126,10 +218,43 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_misaligned_i64_max_offset() #1 { ret void } -; GFX9-LABEL: {{^}}add_x_shl_neg_to_sub_misaligned_i64_max_offset_clamp_bit: -; GFX9: v_sub_u32 -; GFX9: s_endpgm define amdgpu_kernel void @add_x_shl_neg_to_sub_misaligned_i64_max_offset_clamp_bit(float %dummy.val) #1 { +; CI-LABEL: add_x_shl_neg_to_sub_misaligned_i64_max_offset_clamp_bit: +; CI: ; %bb.0: +; CI-NEXT: s_load_dword s0, s[0:1], 0x9 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_sub_i32_e32 v0, vcc, 0, v0 +; CI-NEXT: s_mov_b64 vcc, 0 +; CI-NEXT: v_mov_b32_e32 v2, 0x7b +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_mov_b32_e32 v1, s0 +; CI-NEXT: s_mov_b32 s0, 0 +; CI-NEXT: v_div_fmas_f32 v1, v1, v1, v1 +; CI-NEXT: v_mov_b32_e32 v3, 0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, -1 +; CI-NEXT: s_mov_b32 s1, s0 +; CI-NEXT: ds_write2_b32 v0, v2, v3 offset0:254 offset1:255 +; CI-NEXT: buffer_store_dword v1, off, s[0:3], 0 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: add_x_shl_neg_to_sub_misaligned_i64_max_offset_clamp_bit: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dword s0, s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 vcc, 0 +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: v_sub_u32_e32 v0, 0, v0 +; GFX9-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: v_div_fmas_f32 v2, v1, v1, v1 +; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b +; GFX9-NEXT: ds_write2_b32 v0, v1, v3 offset0:254 offset1:255 +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: global_store_dword v[0:1], v2, off +; GFX9-NEXT: s_endpgm %x.i = call i32 @llvm.amdgcn.workitem.id.x() #0 %neg = sub i32 0, %x.i %shl = shl i32 %neg, 2 @@ -141,12 +266,25 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_misaligned_i64_max_offset_clamp_ ret void } -; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_misaligned_i64_max_offset_p1: -; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0 -; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0x3fc, [[SCALED]] -; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0x3fc, [[SCALED]] -; GCN: ds_write2_b32 [[NEG]], {{v[0-9]+}}, {{v[0-9]+}} offset1:1{{$}} define amdgpu_kernel void @add_x_shl_neg_to_sub_misaligned_i64_max_offset_p1() #1 { +; CI-LABEL: add_x_shl_neg_to_sub_misaligned_i64_max_offset_p1: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_sub_i32_e32 v0, vcc, 0x3fc, v0 +; CI-NEXT: v_mov_b32_e32 v1, 0x7b +; CI-NEXT: v_mov_b32_e32 v2, 0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: add_x_shl_neg_to_sub_misaligned_i64_max_offset_p1: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: v_sub_u32_e32 v0, 0x3fc, v0 +; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b +; GFX9-NEXT: v_mov_b32_e32 v2, 0 +; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; GFX9-NEXT: s_endpgm %x.i = call i32 @llvm.amdgcn.workitem.id.x() #0 %neg = sub i32 0, %x.i %shl = shl i32 %neg, 2 diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2.ll b/llvm/test/CodeGen/AMDGPU/ds_read2.ll index d01d41955449..eb445a3d095a 100644 --- a/llvm/test/CodeGen/AMDGPU/ds_read2.ll +++ b/llvm/test/CodeGen/AMDGPU/ds_read2.ll @@ -1,6 +1,7 @@ -; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,CI %s -; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global,-unaligned-access-mode < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9,GFX9-ALIGNED %s -; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global,+unaligned-access-mode < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9,GFX9-UNALIGNED %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s +; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,-unaligned-access-mode < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-ALIGNED %s +; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+unaligned-access-mode < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-UNALIGNED %s ; FIXME: We don't get cases where the address was an SGPR because we ; get a copy to the address register for each one. @@ -8,17 +9,32 @@ @lds = addrspace(3) global [512 x float] undef, align 4 @lds.f64 = addrspace(3) global [512 x double] undef, align 8 -; GCN-LABEL: {{^}}simple_read2_f32: -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:8 -; GCN: s_waitcnt lgkmcnt(0) -; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]] -; CI: buffer_store_dword [[RESULT]] -; GFX9: global_store_dword v{{[0-9]+}}, [[RESULT]], s{{\[[0-9]+:[0-9]+\]}} -; GCN: s_endpgm define amdgpu_kernel void @simple_read2_f32(float addrspace(1)* %out) #0 { +; CI-LABEL: simple_read2_f32: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_add_i32_e32 v1, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read2_b32 v[1:2], v1 offset1:8 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v2, v1, v2 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_read2_f32: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0 +; GFX9-NEXT: v_add_u32_e32 v0, lds@abs32@lo, v2 +; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:8 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-NEXT: global_store_dword v2, v0, s[0:1] +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i %val0 = load float, float addrspace(3)* %arrayidx0, align 4 @@ -31,17 +47,32 @@ define amdgpu_kernel void @simple_read2_f32(float addrspace(1)* %out) #0 { ret void } -; GCN-LABEL: {{^}}simple_read2_f32_max_offset: -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:255 -; GCN: s_waitcnt lgkmcnt(0) -; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]] - -; CI: buffer_store_dword [[RESULT]] -; GFX9: global_store_dword v{{[0-9]+}}, [[RESULT]], s{{\[[0-9]+:[0-9]+\]}} define amdgpu_kernel void @simple_read2_f32_max_offset(float addrspace(1)* %out) #0 { +; CI-LABEL: simple_read2_f32_max_offset: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_add_i32_e32 v1, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read2_b32 v[1:2], v1 offset1:255 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v2, v1, v2 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_read2_f32_max_offset: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0 +; GFX9-NEXT: v_add_u32_e32 v0, lds@abs32@lo, v2 +; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:255 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-NEXT: global_store_dword v2, v0, s[0:1] +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i %val0 = load float, float addrspace(3)* %arrayidx0, align 4 @@ -54,15 +85,34 @@ define amdgpu_kernel void @simple_read2_f32_max_offset(float addrspace(1)* %out) ret void } -; GCN-LABEL: @simple_read2_f32_too_far -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-NOT: ds_read2_b32 -; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} -; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:1028 -; GCN: s_endpgm define amdgpu_kernel void @simple_read2_f32_too_far(float addrspace(1)* %out) #0 { +; CI-LABEL: simple_read2_f32_too_far: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_add_i32_e32 v1, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read_b32 v2, v1 +; CI-NEXT: ds_read_b32 v1, v1 offset:1028 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v2, v2, v1 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_read2_f32_too_far: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: v_add_u32_e32 v1, lds@abs32@lo, v0 +; GFX9-NEXT: ds_read_b32 v2, v1 +; GFX9-NEXT: ds_read_b32 v1, v1 offset:1028 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v1, v2, v1 +; GFX9-NEXT: global_store_dword v0, v1, s[0:1] +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i %val0 = load float, float addrspace(3)* %arrayidx0, align 4 @@ -75,14 +125,39 @@ define amdgpu_kernel void @simple_read2_f32_too_far(float addrspace(1)* %out) #0 ret void } -; GCN-LABEL: @simple_read2_f32_x2 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset1:8 -; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27 -; GCN: s_endpgm define amdgpu_kernel void @simple_read2_f32_x2(float addrspace(1)* %out) #0 { +; CI-LABEL: simple_read2_f32_x2: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_add_i32_e32 v3, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read2_b32 v[1:2], v3 offset1:8 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v4, v1, v2 +; CI-NEXT: ds_read2_b32 v[1:2], v3 offset0:11 offset1:27 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v1, v1, v2 +; CI-NEXT: v_add_f32_e32 v2, v4, v1 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_read2_f32_x2: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 2, v0 +; GFX9-NEXT: v_add_u32_e32 v2, lds@abs32@lo, v4 +; GFX9-NEXT: ds_read2_b32 v[0:1], v2 offset1:8 +; GFX9-NEXT: ds_read2_b32 v[2:3], v2 offset0:11 offset1:27 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-NEXT: v_add_f32_e32 v1, v2, v3 +; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-NEXT: global_store_dword v4, v0, s[0:1] +; GFX9-NEXT: s_endpgm %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %idx.0 = add nsw i32 %tid.x, 0 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0 @@ -109,15 +184,42 @@ define amdgpu_kernel void @simple_read2_f32_x2(float addrspace(1)* %out) #0 { } ; Make sure there is an instruction between the two sets of reads. -; GCN-LABEL: @simple_read2_f32_x2_barrier -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset1:8 -; GCN: s_barrier -; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27 -; GCN: s_endpgm define amdgpu_kernel void @simple_read2_f32_x2_barrier(float addrspace(1)* %out) #0 { +; CI-LABEL: simple_read2_f32_x2_barrier: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_add_i32_e32 v3, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read2_b32 v[1:2], v3 offset1:8 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: s_barrier +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_add_f32_e32 v4, v1, v2 +; CI-NEXT: ds_read2_b32 v[1:2], v3 offset0:11 offset1:27 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v1, v1, v2 +; CI-NEXT: v_add_f32_e32 v2, v4, v1 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_read2_f32_x2_barrier: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 2, v0 +; GFX9-NEXT: v_add_u32_e32 v2, lds@abs32@lo, v4 +; GFX9-NEXT: ds_read2_b32 v[0:1], v2 offset1:8 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_barrier +; GFX9-NEXT: ds_read2_b32 v[2:3], v2 offset0:11 offset1:27 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v1, v2, v3 +; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-NEXT: global_store_dword v4, v0, s[0:1] +; GFX9-NEXT: s_endpgm %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %idx.0 = add nsw i32 %tid.x, 0 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0 @@ -147,15 +249,39 @@ define amdgpu_kernel void @simple_read2_f32_x2_barrier(float addrspace(1)* %out) ; For some reason adding something to the base address for the first ; element results in only folding the inner pair. - -; GCN-LABEL: @simple_read2_f32_x2_nonzero_base -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset0:2 offset1:8 -; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27 -; GCN: s_endpgm define amdgpu_kernel void @simple_read2_f32_x2_nonzero_base(float addrspace(1)* %out) #0 { +; CI-LABEL: simple_read2_f32_x2_nonzero_base: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_add_i32_e32 v3, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read2_b32 v[1:2], v3 offset0:2 offset1:8 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v4, v1, v2 +; CI-NEXT: ds_read2_b32 v[1:2], v3 offset0:11 offset1:27 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v1, v1, v2 +; CI-NEXT: v_add_f32_e32 v2, v4, v1 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:8 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_read2_f32_x2_nonzero_base: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 2, v0 +; GFX9-NEXT: v_add_u32_e32 v2, lds@abs32@lo, v4 +; GFX9-NEXT: ds_read2_b32 v[0:1], v2 offset0:2 offset1:8 +; GFX9-NEXT: ds_read2_b32 v[2:3], v2 offset0:11 offset1:27 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-NEXT: v_add_f32_e32 v1, v2, v3 +; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-NEXT: global_store_dword v4, v0, s[0:1] offset:8 +; GFX9-NEXT: s_endpgm %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %idx.0 = add nsw i32 %tid.x, 2 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0 @@ -186,16 +312,40 @@ define amdgpu_kernel void @simple_read2_f32_x2_nonzero_base(float addrspace(1)* ; merge. ; Base pointers come from different subregister of same super ; register. We can't safely merge this. - -; GCN-LABEL: @read2_ptr_is_subreg_arg_f32 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-NOT: ds_read2_b32 -; GCN: ds_read_b32 -; GCN: ds_read_b32 -; GCN: s_endpgm define amdgpu_kernel void @read2_ptr_is_subreg_arg_f32(float addrspace(1)* %out, <2 x float addrspace(3)*> %lds.ptr) #0 { +; CI-LABEL: read2_ptr_is_subreg_arg_f32: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_mov_b32 s7, 0xf000 +; CI-NEXT: s_mov_b32 s6, 0 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_mov_b32_e32 v1, s0 +; CI-NEXT: v_mov_b32_e32 v2, s1 +; CI-NEXT: ds_read_b32 v1, v1 offset:32 +; CI-NEXT: ds_read_b32 v2, v2 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v2, v1, v2 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: read2_ptr_is_subreg_arg_f32: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: ds_read_b32 v1, v1 offset:32 +; GFX9-NEXT: ds_read_b32 v2, v2 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX9-NEXT: global_store_dword v0, v1, s[2:3] +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0 %index.1 = insertelement <2 x i32> %index.0, i32 8, i32 0 @@ -215,16 +365,40 @@ define amdgpu_kernel void @read2_ptr_is_subreg_arg_f32(float addrspace(1)* %out, ; are rejecting merges that have the same, constant 0 offset, so make ; sure we are really rejecting it because of the different ; subregisters. - -; GCN-LABEL: @read2_ptr_is_subreg_arg_offset_f32 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-NOT: ds_read2_b32 -; GCN: ds_read_b32 -; GCN: ds_read_b32 -; GCN: s_endpgm define amdgpu_kernel void @read2_ptr_is_subreg_arg_offset_f32(float addrspace(1)* %out, <2 x float addrspace(3)*> %lds.ptr) #0 { +; CI-LABEL: read2_ptr_is_subreg_arg_offset_f32: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_mov_b32 s7, 0xf000 +; CI-NEXT: s_mov_b32 s6, 0 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_mov_b32_e32 v1, s0 +; CI-NEXT: v_mov_b32_e32 v2, s1 +; CI-NEXT: ds_read_b32 v1, v1 offset:32 +; CI-NEXT: ds_read_b32 v2, v2 offset:32 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v2, v1, v2 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: read2_ptr_is_subreg_arg_offset_f32: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: ds_read_b32 v1, v1 offset:32 +; GFX9-NEXT: ds_read_b32 v2, v2 offset:32 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX9-NEXT: global_store_dword v0, v1, s[2:3] +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0 %index.1 = insertelement <2 x i32> %index.0, i32 8, i32 0 @@ -244,13 +418,32 @@ define amdgpu_kernel void @read2_ptr_is_subreg_arg_offset_f32(float addrspace(1) ret void } -; GCN-LABEL: {{^}}read2_ptr_is_subreg_f32: -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:8{{$}} -; GCN: s_endpgm define amdgpu_kernel void @read2_ptr_is_subreg_f32(float addrspace(1)* %out) #0 { +; CI-LABEL: read2_ptr_is_subreg_f32: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_add_i32_e32 v1, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read2_b32 v[1:2], v1 offset1:8 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v2, v1, v2 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: read2_ptr_is_subreg_f32: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0 +; GFX9-NEXT: v_add_u32_e32 v0, lds@abs32@lo, v2 +; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:8 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-NEXT: global_store_dword v2, v0, s[0:1] +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %ptr.0 = insertelement <2 x [512 x float] addrspace(3)*> undef, [512 x float] addrspace(3)* @lds, i32 0 %ptr.1 = insertelement <2 x [512 x float] addrspace(3)*> %ptr.0, [512 x float] addrspace(3)* @lds, i32 1 @@ -269,15 +462,34 @@ define amdgpu_kernel void @read2_ptr_is_subreg_f32(float addrspace(1)* %out) #0 ret void } -; GCN-LABEL: @simple_read2_f32_volatile_0 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-NOT: ds_read2_b32 -; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} -; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32 -; GCN: s_endpgm define amdgpu_kernel void @simple_read2_f32_volatile_0(float addrspace(1)* %out) #0 { +; CI-LABEL: simple_read2_f32_volatile_0: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_add_i32_e32 v1, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read_b32 v2, v1 +; CI-NEXT: ds_read_b32 v1, v1 offset:32 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v2, v2, v1 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_read2_f32_volatile_0: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: v_add_u32_e32 v1, lds@abs32@lo, v0 +; GFX9-NEXT: ds_read_b32 v2, v1 +; GFX9-NEXT: ds_read_b32 v1, v1 offset:32 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v1, v2, v1 +; GFX9-NEXT: global_store_dword v0, v1, s[0:1] +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i %val0 = load volatile float, float addrspace(3)* %arrayidx0, align 4 @@ -290,15 +502,34 @@ define amdgpu_kernel void @simple_read2_f32_volatile_0(float addrspace(1)* %out) ret void } -; GCN-LABEL: @simple_read2_f32_volatile_1 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-NOT: ds_read2_b32 -; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} -; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32 -; GCN: s_endpgm define amdgpu_kernel void @simple_read2_f32_volatile_1(float addrspace(1)* %out) #0 { +; CI-LABEL: simple_read2_f32_volatile_1: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_add_i32_e32 v1, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read_b32 v2, v1 +; CI-NEXT: ds_read_b32 v1, v1 offset:32 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v2, v2, v1 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_read2_f32_volatile_1: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: v_add_u32_e32 v1, lds@abs32@lo, v0 +; GFX9-NEXT: ds_read_b32 v2, v1 +; GFX9-NEXT: ds_read_b32 v1, v1 offset:32 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v1, v2, v1 +; GFX9-NEXT: global_store_dword v0, v1, s[0:1] +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i %val0 = load float, float addrspace(3)* %arrayidx0, align 4 @@ -312,15 +543,88 @@ define amdgpu_kernel void @simple_read2_f32_volatile_1(float addrspace(1)* %out) } ; Can't fold since not correctly aligned. -; GCN-LABEL: @unaligned_read2_f32 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; CI-COUNT-4: ds_read_u8 -; GFX9-ALIGNED-4: ds_read_u8 -; GFX9-UNALIGNED-4: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}} -; GCN: s_endpgm define amdgpu_kernel void @unaligned_read2_f32(float addrspace(1)* %out, float addrspace(3)* %lds) #0 { +; CI-LABEL: unaligned_read2_f32: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; CI-NEXT: s_load_dword s0, s[0:1], 0xb +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_mov_b32 s7, 0xf000 +; CI-NEXT: s_mov_b32 s6, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_i32_e32 v1, vcc, s0, v0 +; CI-NEXT: ds_read_u8 v2, v1 offset:35 +; CI-NEXT: ds_read_u8 v3, v1 offset:34 +; CI-NEXT: ds_read_u8 v4, v1 offset:33 +; CI-NEXT: ds_read_u8 v5, v1 offset:32 +; CI-NEXT: ds_read_u8 v6, v1 offset:3 +; CI-NEXT: ds_read_u8 v7, v1 offset:2 +; CI-NEXT: ds_read_u8 v8, v1 offset:1 +; CI-NEXT: ds_read_u8 v1, v1 +; CI-NEXT: s_waitcnt lgkmcnt(7) +; CI-NEXT: v_lshlrev_b32_e32 v2, 8, v2 +; CI-NEXT: s_waitcnt lgkmcnt(3) +; CI-NEXT: v_lshlrev_b32_e32 v6, 8, v6 +; CI-NEXT: s_waitcnt lgkmcnt(2) +; CI-NEXT: v_or_b32_e32 v6, v6, v7 +; CI-NEXT: v_lshlrev_b32_e32 v4, 8, v4 +; CI-NEXT: v_or_b32_e32 v2, v2, v3 +; CI-NEXT: s_waitcnt lgkmcnt(1) +; CI-NEXT: v_lshlrev_b32_e32 v8, 8, v8 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_or_b32_e32 v1, v8, v1 +; CI-NEXT: v_lshlrev_b32_e32 v6, 16, v6 +; CI-NEXT: v_or_b32_e32 v4, v4, v5 +; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; CI-NEXT: v_or_b32_e32 v1, v6, v1 +; CI-NEXT: v_or_b32_e32 v2, v2, v4 +; CI-NEXT: v_add_f32_e32 v2, v1, v2 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-ALIGNED-LABEL: unaligned_read2_f32: +; GFX9-ALIGNED: ; %bb.0: +; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-ALIGNED-NEXT: s_load_dword s0, s[0:1], 0x2c +; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-ALIGNED-NEXT: v_add_u32_e32 v1, s0, v0 +; GFX9-ALIGNED-NEXT: ds_read_u8 v2, v1 +; GFX9-ALIGNED-NEXT: ds_read_u8 v3, v1 offset:1 +; GFX9-ALIGNED-NEXT: ds_read_u8 v4, v1 offset:2 +; GFX9-ALIGNED-NEXT: ds_read_u8 v5, v1 offset:3 +; GFX9-ALIGNED-NEXT: ds_read_u8 v6, v1 offset:32 +; GFX9-ALIGNED-NEXT: ds_read_u8 v7, v1 offset:33 +; GFX9-ALIGNED-NEXT: ds_read_u8 v8, v1 offset:34 +; GFX9-ALIGNED-NEXT: ds_read_u8 v1, v1 offset:35 +; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(6) +; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v2, v3, 8, v2 +; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(4) +; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v3, v5, 8, v4 +; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v2, v3, 16, v2 +; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(2) +; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v6 +; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 8, v8 +; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 16, v3 +; GFX9-ALIGNED-NEXT: v_add_f32_e32 v1, v2, v1 +; GFX9-ALIGNED-NEXT: global_store_dword v0, v1, s[2:3] +; GFX9-ALIGNED-NEXT: s_endpgm +; +; GFX9-UNALIGNED-LABEL: unaligned_read2_f32: +; GFX9-UNALIGNED: ; %bb.0: +; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-UNALIGNED-NEXT: s_load_dword s0, s[0:1], 0x2c +; GFX9-UNALIGNED-NEXT: v_lshlrev_b32_e32 v2, 2, v0 +; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-UNALIGNED-NEXT: v_add_u32_e32 v0, s0, v2 +; GFX9-UNALIGNED-NEXT: ds_read2_b32 v[0:1], v0 offset1:8 +; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-UNALIGNED-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-UNALIGNED-NEXT: global_store_dword v2, v0, s[2:3] +; GFX9-UNALIGNED-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %arrayidx0 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %x.i %val0 = load float, float addrspace(3)* %arrayidx0, align 1 @@ -333,15 +637,65 @@ define amdgpu_kernel void @unaligned_read2_f32(float addrspace(1)* %out, float a ret void } -; GCN-LABEL: @misaligned_2_simple_read2_f32 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; CI-COUNT-2: ds_read_u16 -; GFX9-ALIGNED-2: ds_read_u16 -; GFX9-UNALIGNED-4: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}} -; GCN: s_endpgm define amdgpu_kernel void @misaligned_2_simple_read2_f32(float addrspace(1)* %out, float addrspace(3)* %lds) #0 { +; CI-LABEL: misaligned_2_simple_read2_f32: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; CI-NEXT: s_load_dword s0, s[0:1], 0xb +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_mov_b32 s7, 0xf000 +; CI-NEXT: s_mov_b32 s6, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_i32_e32 v1, vcc, s0, v0 +; CI-NEXT: ds_read_u16 v2, v1 offset:34 +; CI-NEXT: ds_read_u16 v3, v1 offset:32 +; CI-NEXT: ds_read_u16 v4, v1 offset:2 +; CI-NEXT: ds_read_u16 v1, v1 +; CI-NEXT: s_waitcnt lgkmcnt(3) +; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; CI-NEXT: s_waitcnt lgkmcnt(2) +; CI-NEXT: v_or_b32_e32 v2, v2, v3 +; CI-NEXT: s_waitcnt lgkmcnt(1) +; CI-NEXT: v_lshlrev_b32_e32 v4, 16, v4 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_or_b32_e32 v1, v4, v1 +; CI-NEXT: v_add_f32_e32 v2, v1, v2 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-ALIGNED-LABEL: misaligned_2_simple_read2_f32: +; GFX9-ALIGNED: ; %bb.0: +; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-ALIGNED-NEXT: s_load_dword s0, s[0:1], 0x2c +; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-ALIGNED-NEXT: v_add_u32_e32 v1, s0, v0 +; GFX9-ALIGNED-NEXT: ds_read_u16 v2, v1 +; GFX9-ALIGNED-NEXT: ds_read_u16 v3, v1 offset:2 +; GFX9-ALIGNED-NEXT: ds_read_u16 v4, v1 offset:32 +; GFX9-ALIGNED-NEXT: ds_read_u16 v1, v1 offset:34 +; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(2) +; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v2, v3, 16, v2 +; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 16, v4 +; GFX9-ALIGNED-NEXT: v_add_f32_e32 v1, v2, v1 +; GFX9-ALIGNED-NEXT: global_store_dword v0, v1, s[2:3] +; GFX9-ALIGNED-NEXT: s_endpgm +; +; GFX9-UNALIGNED-LABEL: misaligned_2_simple_read2_f32: +; GFX9-UNALIGNED: ; %bb.0: +; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-UNALIGNED-NEXT: s_load_dword s0, s[0:1], 0x2c +; GFX9-UNALIGNED-NEXT: v_lshlrev_b32_e32 v2, 2, v0 +; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-UNALIGNED-NEXT: v_add_u32_e32 v0, s0, v2 +; GFX9-UNALIGNED-NEXT: ds_read2_b32 v[0:1], v0 offset1:8 +; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-UNALIGNED-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-UNALIGNED-NEXT: global_store_dword v2, v0, s[2:3] +; GFX9-UNALIGNED-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %arrayidx0 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %x.i %val0 = load float, float addrspace(3)* %arrayidx0, align 2 @@ -354,18 +708,32 @@ define amdgpu_kernel void @misaligned_2_simple_read2_f32(float addrspace(1)* %ou ret void } -; GCN-LABEL: @simple_read2_f64 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-DAG: v_lshlrev_b32_e32 [[VOFS:v[0-9]+]], 3, {{v[0-9]+}} -; GCN-DAG: v_add_{{[iu]}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}lds.f64@abs32@lo, [[VOFS]] -; GCN: ds_read2_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, [[VPTR]] offset1:8 -; GCN: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO_VREG]]:{{[0-9]+\]}}, v{{\[[0-9]+}}:[[HI_VREG]]{{\]}} - -; CI: buffer_store_dwordx2 [[RESULT]] -; GFX9: global_store_dwordx2 v{{[0-9]+}}, [[RESULT]], s{{\[[0-9]+:[0-9]+\]}} define amdgpu_kernel void @simple_read2_f64(double addrspace(1)* %out) #0 { +; CI-LABEL: simple_read2_f64: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds.f64@abs32@lo, v4 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read2_b64 v[0:3], v0 offset1:8 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_mov_b32_e32 v5, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; CI-NEXT: buffer_store_dwordx2 v[0:1], v[4:5], s[0:3], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_read2_f64: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; GFX9-NEXT: v_add_u32_e32 v0, lds.f64@abs32@lo, v4 +; GFX9-NEXT: ds_read2_b64 v[0:3], v0 offset1:8 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1] +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i %val0 = load double, double addrspace(3)* %arrayidx0, align 8 @@ -378,13 +746,32 @@ define amdgpu_kernel void @simple_read2_f64(double addrspace(1)* %out) #0 { ret void } -; GCN-LABEL: @simple_read2_f64_max_offset -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN: ds_read2_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:255 -; GCN: s_endpgm define amdgpu_kernel void @simple_read2_f64_max_offset(double addrspace(1)* %out) #0 { +; CI-LABEL: simple_read2_f64_max_offset: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds.f64@abs32@lo, v4 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read2_b64 v[0:3], v0 offset1:255 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_mov_b32_e32 v5, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; CI-NEXT: buffer_store_dwordx2 v[0:1], v[4:5], s[0:3], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_read2_f64_max_offset: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; GFX9-NEXT: v_add_u32_e32 v0, lds.f64@abs32@lo, v4 +; GFX9-NEXT: ds_read2_b64 v[0:3], v0 offset1:255 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1] +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i %val0 = load double, double addrspace(3)* %arrayidx0, align 8 @@ -397,15 +784,34 @@ define amdgpu_kernel void @simple_read2_f64_max_offset(double addrspace(1)* %out ret void } -; GCN-LABEL: @simple_read2_f64_too_far -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-NOT: ds_read2_b64 -; GCN: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} -; GCN: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:2056 -; GCN: s_endpgm define amdgpu_kernel void @simple_read2_f64_too_far(double addrspace(1)* %out) #0 { +; CI-LABEL: simple_read2_f64_too_far: +; CI: ; %bb.0: +; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0 +; CI-NEXT: v_add_i32_e32 v3, vcc, lds.f64@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read_b64 v[1:2], v3 +; CI-NEXT: ds_read_b64 v[3:4], v3 offset:2056 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f64 v[2:3], v[1:2], v[3:4] +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_read2_f64_too_far: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; GFX9-NEXT: v_add_u32_e32 v2, lds.f64@abs32@lo, v4 +; GFX9-NEXT: ds_read_b64 v[0:1], v2 +; GFX9-NEXT: ds_read_b64 v[2:3], v2 offset:2056 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1] +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i %val0 = load double, double addrspace(3)* %arrayidx0, align 8 @@ -419,14 +825,38 @@ define amdgpu_kernel void @simple_read2_f64_too_far(double addrspace(1)* %out) # } ; Alignment only 4 -; GCN-LABEL: @misaligned_read2_f64 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:1 -; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:14 offset1:15 -; GCN: s_endpgm define amdgpu_kernel void @misaligned_read2_f64(double addrspace(1)* %out, double addrspace(3)* %lds) #0 { +; CI-LABEL: misaligned_read2_f64: +; CI: ; %bb.0: +; CI-NEXT: s_load_dword s2, s[0:1], 0xb +; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_i32_e32 v3, vcc, s2, v0 +; CI-NEXT: ds_read2_b32 v[1:2], v3 offset1:1 +; CI-NEXT: ds_read2_b32 v[3:4], v3 offset0:14 offset1:15 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f64 v[2:3], v[1:2], v[3:4] +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: misaligned_read2_f64: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dword s2, s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_u32_e32 v2, s2, v4 +; GFX9-NEXT: ds_read2_b32 v[0:1], v2 offset1:1 +; GFX9-NEXT: ds_read2_b32 v[2:3], v2 offset0:14 offset1:15 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1] +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %arrayidx0 = getelementptr inbounds double, double addrspace(3)* %lds, i32 %x.i %val0 = load double, double addrspace(3)* %arrayidx0, align 4 @@ -441,13 +871,31 @@ define amdgpu_kernel void @misaligned_read2_f64(double addrspace(1)* %out, doubl @foo = addrspace(3) global [4 x i32] undef, align 4 -; GCN-LABEL: @load_constant_adjacent_offsets -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], foo@abs32@lo{{$}} -; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:1 define amdgpu_kernel void @load_constant_adjacent_offsets(i32 addrspace(1)* %out) { +; CI-LABEL: load_constant_adjacent_offsets: +; CI: ; %bb.0: +; CI-NEXT: v_mov_b32_e32 v0, foo@abs32@lo +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read2_b32 v[0:1], v0 offset1:1 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, -1 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_i32_e32 v0, vcc, v1, v0 +; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: load_constant_adjacent_offsets: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_mov_b32_e32 v0, foo@abs32@lo +; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:1 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_u32_e32 v2, v0, v1 +; GFX9-NEXT: v_mov_b32_e32 v0, s0 +; GFX9-NEXT: v_mov_b32_e32 v1, s1 +; GFX9-NEXT: global_store_dword v[0:1], v2, off +; GFX9-NEXT: s_endpgm %val0 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4 %val1 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 1), align 4 %sum = add i32 %val0, %val1 @@ -455,13 +903,31 @@ define amdgpu_kernel void @load_constant_adjacent_offsets(i32 addrspace(1)* %out ret void } -; GCN-LABEL: @load_constant_disjoint_offsets -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], foo@abs32@lo{{$}} -; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:2 define amdgpu_kernel void @load_constant_disjoint_offsets(i32 addrspace(1)* %out) { +; CI-LABEL: load_constant_disjoint_offsets: +; CI: ; %bb.0: +; CI-NEXT: v_mov_b32_e32 v0, foo@abs32@lo +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read2_b32 v[0:1], v0 offset1:2 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, -1 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_i32_e32 v0, vcc, v1, v0 +; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: load_constant_disjoint_offsets: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_mov_b32_e32 v0, foo@abs32@lo +; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:2 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_u32_e32 v2, v0, v1 +; GFX9-NEXT: v_mov_b32_e32 v0, s0 +; GFX9-NEXT: v_mov_b32_e32 v1, s1 +; GFX9-NEXT: global_store_dword v[0:1], v2, off +; GFX9-NEXT: s_endpgm %val0 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4 %val1 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 2), align 4 %sum = add i32 %val0, %val1 @@ -471,19 +937,48 @@ define amdgpu_kernel void @load_constant_disjoint_offsets(i32 addrspace(1)* %out @bar = addrspace(3) global [4 x i64] undef, align 4 -; GCN-LABEL: @load_misaligned64_constant_offsets -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], bar@abs32@lo{{$}} -; CI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:1 -; CI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset0:2 offset1:3 - -; GFX9-ALIGNED-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:1 -; GFX9-ALIGNED-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset0:2 offset1:3 - -; GFX9-UNALIGNED: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] define amdgpu_kernel void @load_misaligned64_constant_offsets(i64 addrspace(1)* %out) { +; CI-LABEL: load_misaligned64_constant_offsets: +; CI: ; %bb.0: +; CI-NEXT: v_mov_b32_e32 v2, bar@abs32@lo +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read2_b32 v[0:1], v2 offset1:1 +; CI-NEXT: ds_read2_b32 v[2:3], v2 offset0:2 offset1:3 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, -1 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; CI-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; CI-NEXT: s_endpgm +; +; GFX9-ALIGNED-LABEL: load_misaligned64_constant_offsets: +; GFX9-ALIGNED: ; %bb.0: +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v2, bar@abs32@lo +; GFX9-ALIGNED-NEXT: ds_read2_b32 v[0:1], v2 offset1:1 +; GFX9-ALIGNED-NEXT: ds_read2_b32 v[2:3], v2 offset0:2 offset1:3 +; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-ALIGNED-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2 +; GFX9-ALIGNED-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v3, s1 +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v2, s0 +; GFX9-ALIGNED-NEXT: global_store_dwordx2 v[2:3], v[0:1], off +; GFX9-ALIGNED-NEXT: s_endpgm +; +; GFX9-UNALIGNED-LABEL: load_misaligned64_constant_offsets: +; GFX9-UNALIGNED: ; %bb.0: +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, bar@abs32@lo +; GFX9-UNALIGNED-NEXT: ds_read_b128 v[0:3], v0 +; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-UNALIGNED-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2 +; GFX9-UNALIGNED-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v3, s1 +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v2, s0 +; GFX9-UNALIGNED-NEXT: global_store_dwordx2 v[2:3], v[0:1], off +; GFX9-UNALIGNED-NEXT: s_endpgm %val0 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 0), align 4 %val1 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 1), align 4 %sum = add i64 %val0, %val1 @@ -493,19 +988,43 @@ define amdgpu_kernel void @load_misaligned64_constant_offsets(i64 addrspace(1)* @bar.large = addrspace(3) global [4096 x i64] undef, align 4 -; GCN-LABEL: @load_misaligned64_constant_large_offsets -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-DAG: s_mov_b32 [[SBASE0:s[0-9]+]], bar.large@abs32@lo -; GCN-DAG: s_add_i32 [[SBASE1:s[0-9]+]], [[SBASE0]], 0x4000{{$}} -; GCN-DAG: s_addk_i32 [[SBASE0]], 0x7ff8{{$}} -; GCN-DAG: v_mov_b32_e32 [[VBASE0:v[0-9]+]], [[SBASE0]] -; GCN-DAG: v_mov_b32_e32 [[VBASE1:v[0-9]+]], [[SBASE1]] -; GCN-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[VBASE0]] offset1:1 -; GCN-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[VBASE1]] offset1:1 -; GCN: s_endpgm define amdgpu_kernel void @load_misaligned64_constant_large_offsets(i64 addrspace(1)* %out) { +; CI-LABEL: load_misaligned64_constant_large_offsets: +; CI: ; %bb.0: +; CI-NEXT: s_mov_b32 s4, bar.large@abs32@lo +; CI-NEXT: s_add_i32 s5, s4, 0x4000 +; CI-NEXT: s_addk_i32 s4, 0x7ff8 +; CI-NEXT: v_mov_b32_e32 v0, s5 +; CI-NEXT: v_mov_b32_e32 v2, s4 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read2_b32 v[0:1], v0 offset1:1 +; CI-NEXT: ds_read2_b32 v[2:3], v2 offset1:1 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, -1 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; CI-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: load_misaligned64_constant_large_offsets: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s2, bar.large@abs32@lo +; GFX9-NEXT: s_add_i32 s3, s2, 0x4000 +; GFX9-NEXT: s_addk_i32 s2, 0x7ff8 +; GFX9-NEXT: v_mov_b32_e32 v0, s3 +; GFX9-NEXT: v_mov_b32_e32 v2, s2 +; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:1 +; GFX9-NEXT: ds_read2_b32 v[2:3], v2 offset1:1 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9-NEXT: v_mov_b32_e32 v3, s1 +; GFX9-NEXT: v_mov_b32_e32 v2, s0 +; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off +; GFX9-NEXT: s_endpgm %val0 = load i64, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 2048), align 4 %val1 = load i64, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 4095), align 4 %sum = add i64 %val0, %val1 @@ -516,11 +1035,65 @@ define amdgpu_kernel void @load_misaligned64_constant_large_offsets(i64 addrspac @sgemm.lA = internal unnamed_addr addrspace(3) global [264 x float] undef, align 4 @sgemm.lB = internal unnamed_addr addrspace(3) global [776 x float] undef, align 4 -; GCN-LABEL: {{^}}sgemm_inner_loop_read2_sequence: -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(float addrspace(1)* %C, i32 %lda, i32 %ldb) #0 { +; CI-LABEL: sgemm_inner_loop_read2_sequence: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; CI-NEXT: s_lshl_b32 s0, s2, 2 +; CI-NEXT: s_add_i32 s1, s0, 0xc20 +; CI-NEXT: s_addk_i32 s0, 0xc60 +; CI-NEXT: v_mov_b32_e32 v0, s1 +; CI-NEXT: v_mov_b32_e32 v4, s0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read2_b32 v[2:3], v0 offset1:1 +; CI-NEXT: ds_read2_b32 v[4:5], v4 offset1:1 +; CI-NEXT: v_lshlrev_b32_e32 v8, 2, v1 +; CI-NEXT: ds_read2_b32 v[0:1], v8 offset1:1 +; CI-NEXT: ds_read2_b32 v[6:7], v8 offset0:32 offset1:33 +; CI-NEXT: ds_read2_b32 v[8:9], v8 offset0:64 offset1:65 +; CI-NEXT: s_mov_b32 s7, 0xf000 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v2, v2, v3 +; CI-NEXT: v_add_f32_e32 v2, v2, v4 +; CI-NEXT: v_add_f32_e32 v2, v2, v5 +; CI-NEXT: v_add_f32_e32 v0, v2, v0 +; CI-NEXT: v_add_f32_e32 v0, v0, v1 +; CI-NEXT: v_add_f32_e32 v0, v0, v6 +; CI-NEXT: v_add_f32_e32 v0, v0, v7 +; CI-NEXT: v_add_f32_e32 v0, v0, v8 +; CI-NEXT: s_mov_b32 s6, -1 +; CI-NEXT: v_add_f32_e32 v0, v0, v9 +; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: sgemm_inner_loop_read2_sequence: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_lshl_b32 s2, s2, 2 +; GFX9-NEXT: s_add_i32 s3, s2, 0xc20 +; GFX9-NEXT: s_addk_i32 s2, 0xc60 +; GFX9-NEXT: v_mov_b32_e32 v0, s3 +; GFX9-NEXT: v_mov_b32_e32 v4, s2 +; GFX9-NEXT: ds_read2_b32 v[2:3], v0 offset1:1 +; GFX9-NEXT: ds_read2_b32 v[4:5], v4 offset1:1 +; GFX9-NEXT: v_lshlrev_b32_e32 v8, 2, v1 +; GFX9-NEXT: ds_read2_b32 v[0:1], v8 offset1:1 +; GFX9-NEXT: ds_read2_b32 v[6:7], v8 offset0:32 offset1:33 +; GFX9-NEXT: ds_read2_b32 v[8:9], v8 offset0:64 offset1:65 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v2, v2, v3 +; GFX9-NEXT: v_add_f32_e32 v2, v2, v4 +; GFX9-NEXT: v_add_f32_e32 v2, v2, v5 +; GFX9-NEXT: v_add_f32_e32 v0, v2, v0 +; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-NEXT: v_add_f32_e32 v0, v0, v6 +; GFX9-NEXT: v_add_f32_e32 v0, v0, v7 +; GFX9-NEXT: v_add_f32_e32 v0, v0, v8 +; GFX9-NEXT: v_add_f32_e32 v2, v0, v9 +; GFX9-NEXT: v_mov_b32_e32 v0, s0 +; GFX9-NEXT: v_mov_b32_e32 v1, s1 +; GFX9-NEXT: global_store_dword v[0:1], v2, off +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workgroup.id.x() #1 %y.i = tail call i32 @llvm.amdgcn.workitem.id.y() #1 %arrayidx44 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %x.i @@ -564,30 +1137,131 @@ define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(float addrspace(1)* % ret void } -; GCN-LABEL: {{^}}misaligned_read2_v2i32: -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 define amdgpu_kernel void @misaligned_read2_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(3)* %in) #0 { +; CI-LABEL: misaligned_read2_v2i32: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; CI-NEXT: s_load_dword s0, s[0:1], 0xb +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_mov_b32 s7, 0xf000 +; CI-NEXT: s_mov_b32 s6, -1 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_mov_b32_e32 v0, s0 +; CI-NEXT: ds_read2_b32 v[0:1], v0 offset1:1 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: misaligned_read2_v2i32: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-NEXT: s_load_dword s0, s[0:1], 0x2c +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s2 +; GFX9-NEXT: v_mov_b32_e32 v0, s0 +; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:1 +; GFX9-NEXT: v_mov_b32_e32 v3, s3 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off +; GFX9-NEXT: s_endpgm %load = load <2 x i32>, <2 x i32> addrspace(3)* %in, align 4 store <2 x i32> %load, <2 x i32> addrspace(1)* %out, align 8 ret void } -; GCN-LABEL: {{^}}misaligned_read2_i64: -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 define amdgpu_kernel void @misaligned_read2_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %in) #0 { +; CI-LABEL: misaligned_read2_i64: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; CI-NEXT: s_load_dword s0, s[0:1], 0xb +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_mov_b32 s7, 0xf000 +; CI-NEXT: s_mov_b32 s6, -1 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_mov_b32_e32 v0, s0 +; CI-NEXT: ds_read2_b32 v[0:1], v0 offset1:1 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: misaligned_read2_i64: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-NEXT: s_load_dword s0, s[0:1], 0x2c +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s2 +; GFX9-NEXT: v_mov_b32_e32 v0, s0 +; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:1 +; GFX9-NEXT: v_mov_b32_e32 v3, s3 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off +; GFX9-NEXT: s_endpgm %load = load i64, i64 addrspace(3)* %in, align 4 store i64 %load, i64 addrspace(1)* %out, align 8 ret void } -; GCN-LABEL: ds_read_diff_base_interleaving -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-NOT: ds_read_b32 define amdgpu_kernel void @ds_read_diff_base_interleaving( +; CI-LABEL: ds_read_diff_base_interleaving: +; CI: ; %bb.0: ; %bb +; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xb +; CI-NEXT: v_lshlrev_b32_e32 v1, 4, v1 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_mov_b32 s7, 0xf000 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_i32_e32 v2, vcc, s0, v1 +; CI-NEXT: v_add_i32_e32 v4, vcc, s1, v0 +; CI-NEXT: v_add_i32_e32 v3, vcc, s2, v1 +; CI-NEXT: v_add_i32_e32 v6, vcc, s3, v0 +; CI-NEXT: ds_read2_b32 v[0:1], v2 offset1:1 +; CI-NEXT: ds_read2_b32 v[2:3], v3 offset1:1 +; CI-NEXT: ds_read2_b32 v[4:5], v4 offset1:4 +; CI-NEXT: s_mov_b32 s6, -1 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_mul_f32_e32 v0, v0, v4 +; CI-NEXT: v_add_f32_e32 v4, 2.0, v0 +; CI-NEXT: v_mul_f32_e32 v5, v1, v5 +; CI-NEXT: ds_read2_b32 v[0:1], v6 offset1:4 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_mul_f32_e32 v0, v2, v0 +; CI-NEXT: v_sub_f32_e32 v0, v4, v0 +; CI-NEXT: v_sub_f32_e32 v0, v0, v5 +; CI-NEXT: v_mul_f32_e32 v1, v3, v1 +; CI-NEXT: v_sub_f32_e32 v0, v0, v1 +; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0 offset:40 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: ds_read_diff_base_interleaving: +; GFX9: ; %bb.0: ; %bb +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 4, v1 +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_u32_e32 v2, s0, v1 +; GFX9-NEXT: v_add_u32_e32 v3, s1, v0 +; GFX9-NEXT: v_add_u32_e32 v4, s2, v1 +; GFX9-NEXT: v_add_u32_e32 v6, s3, v0 +; GFX9-NEXT: ds_read2_b32 v[0:1], v2 offset1:1 +; GFX9-NEXT: ds_read2_b32 v[2:3], v3 offset1:4 +; GFX9-NEXT: ds_read2_b32 v[4:5], v4 offset1:1 +; GFX9-NEXT: ds_read2_b32 v[6:7], v6 offset1:4 +; GFX9-NEXT: s_waitcnt lgkmcnt(2) +; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2 +; GFX9-NEXT: v_add_f32_e32 v0, 2.0, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mul_f32_e32 v2, v4, v6 +; GFX9-NEXT: v_sub_f32_e32 v0, v0, v2 +; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3 +; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX9-NEXT: v_mul_f32_e32 v1, v5, v7 +; GFX9-NEXT: v_sub_f32_e32 v2, v0, v1 +; GFX9-NEXT: v_mov_b32_e32 v0, s4 +; GFX9-NEXT: v_mov_b32_e32 v1, s5 +; GFX9-NEXT: global_store_dword v[0:1], v2, off offset:40 +; GFX9-NEXT: s_endpgm float addrspace(1)* nocapture %arg, [4 x [4 x float]] addrspace(3)* %arg1, [4 x [4 x float]] addrspace(3)* %arg2, @@ -625,11 +1299,68 @@ bb: ret void } -; GCN-LABEL: ds_read_call_read: -; GCN: ds_read_b32 -; GCN: s_swappc_b64 -; GCN: ds_read_b32 define amdgpu_kernel void @ds_read_call_read(i32 addrspace(1)* %out, i32 addrspace(3)* %arg) { +; CI-LABEL: ds_read_call_read: +; CI: ; %bb.0: +; CI-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; CI-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; CI-NEXT: s_load_dwordx2 s[36:37], s[0:1], 0x9 +; CI-NEXT: s_load_dword s0, s[0:1], 0xb +; CI-NEXT: s_mov_b32 s42, -1 +; CI-NEXT: s_mov_b32 s43, 0xe8f000 +; CI-NEXT: s_add_u32 s40, s40, s3 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: s_addc_u32 s41, s41, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_i32_e32 v40, vcc, s0, v0 +; CI-NEXT: s_getpc_b64 s[0:1] +; CI-NEXT: s_add_u32 s0, s0, void_func_void@gotpcrel32@lo+4 +; CI-NEXT: s_addc_u32 s1, s1, void_func_void@gotpcrel32@hi+12 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; CI-NEXT: ds_read_b32 v41, v40 +; CI-NEXT: s_mov_b64 s[0:1], s[40:41] +; CI-NEXT: s_mov_b64 s[2:3], s[42:43] +; CI-NEXT: s_mov_b32 s32, 0 +; CI-NEXT: s_mov_b32 s39, 0xf000 +; CI-NEXT: s_mov_b32 s38, -1 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: s_swappc_b64 s[30:31], s[4:5] +; CI-NEXT: ds_read_b32 v0, v40 offset:4 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_i32_e32 v0, vcc, v41, v0 +; CI-NEXT: buffer_store_dword v0, off, s[36:39], 0 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: ds_read_call_read: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[0:1], 0x24 +; GFX9-NEXT: s_load_dword s2, s[0:1], 0x2c +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s3 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, void_func_void@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, void_func_void@gotpcrel32@hi+12 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_lshl_add_u32 v40, v0, 2, s2 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: ds_read_b32 v41, v40 +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: ds_read_b32 v0, v40 offset:4 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_u32_e32 v2, v41, v0 +; GFX9-NEXT: v_mov_b32_e32 v0, s34 +; GFX9-NEXT: v_mov_b32_e32 v1, s35 +; GFX9-NEXT: global_store_dword v[0:1], v2, off +; GFX9-NEXT: s_endpgm %x = call i32 @llvm.amdgcn.workitem.id.x() %arrayidx0 = getelementptr i32, i32 addrspace(3)* %arg, i32 %x %arrayidx1 = getelementptr i32, i32 addrspace(3)* %arrayidx0, i32 1 @@ -641,17 +1372,29 @@ define amdgpu_kernel void @ds_read_call_read(i32 addrspace(1)* %out, i32 addrspa ret void } -; GCN-LABEL: {{^}}ds_read_interp_read: -; CI: s_mov_b32 m0, -1 -; CI: ds_read_b32 -; CI: s_mov_b32 m0, s0 -; CI: v_interp_mov_f32 -; CI: s_mov_b32 m0, -1 -; CI: ds_read_b32 -; GFX9: ds_read2_b32 v[0:1], v0 offset1:4 -; GFX9: s_mov_b32 m0, s0 -; GFX9: v_interp_mov_f32 define amdgpu_ps <2 x float> @ds_read_interp_read(i32 inreg %prims, float addrspace(3)* %inptr) { +; CI-LABEL: ds_read_interp_read: +; CI: ; %bb.0: +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read_b32 v2, v0 +; CI-NEXT: s_mov_b32 m0, s0 +; CI-NEXT: v_interp_mov_f32 v1, p10, attr0.x +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read_b32 v0, v0 offset:16 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_f32_e32 v1, v0, v1 +; CI-NEXT: v_mov_b32_e32 v0, v2 +; CI-NEXT: ; return to shader part epilog +; +; GFX9-LABEL: ds_read_interp_read: +; GFX9: ; %bb.0: +; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:4 +; GFX9-NEXT: s_mov_b32 m0, s0 +; GFX9-NEXT: s_nop 0 +; GFX9-NEXT: v_interp_mov_f32_e32 v2, p10, attr0.x +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX9-NEXT: ; return to shader part epilog %v0 = load float, float addrspace(3)* %inptr, align 4 %intrp = call float @llvm.amdgcn.interp.mov(i32 0, i32 0, i32 0, i32 %prims) %ptr1 = getelementptr float, float addrspace(3)* %inptr, i32 4 @@ -664,14 +1407,80 @@ define amdgpu_ps <2 x float> @ds_read_interp_read(i32 inreg %prims, float addrsp @v2i32_align1 = internal addrspace(3) global [100 x <2 x i32>] undef, align 1 -; GCN-LABEL: {{^}}read2_v2i32_align1_odd_offset: -; CI-COUNT-8: ds_read_u8 - -; GFX9-ALIGNED-COUNT-8: ds_read_u8 - -; GFX9-UNALIGNED: v_mov_b32_e32 [[BASE_ADDR:v[0-9]+]], 0x41{{$}} -; GFX9-UNALIGNED: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASE_ADDR]] offset1:1{{$}} define amdgpu_kernel void @read2_v2i32_align1_odd_offset(<2 x i32> addrspace(1)* %out) { +; CI-LABEL: read2_v2i32_align1_odd_offset: +; CI: ; %bb.0: ; %entry +; CI-NEXT: v_mov_b32_e32 v0, 0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_read_u8 v1, v0 offset:72 +; CI-NEXT: ds_read_u8 v2, v0 offset:71 +; CI-NEXT: ds_read_u8 v3, v0 offset:70 +; CI-NEXT: ds_read_u8 v4, v0 offset:69 +; CI-NEXT: ds_read_u8 v5, v0 offset:68 +; CI-NEXT: s_waitcnt lgkmcnt(4) +; CI-NEXT: v_lshlrev_b32_e32 v1, 8, v1 +; CI-NEXT: s_waitcnt lgkmcnt(3) +; CI-NEXT: v_or_b32_e32 v1, v1, v2 +; CI-NEXT: s_waitcnt lgkmcnt(2) +; CI-NEXT: v_lshlrev_b32_e32 v3, 8, v3 +; CI-NEXT: s_waitcnt lgkmcnt(1) +; CI-NEXT: v_or_b32_e32 v3, v3, v4 +; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; CI-NEXT: v_or_b32_e32 v1, v1, v3 +; CI-NEXT: ds_read_u8 v2, v0 offset:67 +; CI-NEXT: ds_read_u8 v3, v0 offset:66 +; CI-NEXT: ds_read_u8 v0, v0 offset:65 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, -1 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_lshlrev_b32_e32 v3, 8, v3 +; CI-NEXT: v_or_b32_e32 v0, v3, v0 +; CI-NEXT: v_lshlrev_b32_e32 v3, 8, v5 +; CI-NEXT: v_or_b32_e32 v2, v3, v2 +; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; CI-NEXT: v_or_b32_e32 v0, v2, v0 +; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; CI-NEXT: s_endpgm +; +; GFX9-ALIGNED-LABEL: read2_v2i32_align1_odd_offset: +; GFX9-ALIGNED: ; %bb.0: ; %entry +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-ALIGNED-NEXT: ds_read_u8 v2, v0 offset:65 +; GFX9-ALIGNED-NEXT: ds_read_u8 v3, v0 offset:66 +; GFX9-ALIGNED-NEXT: ds_read_u8 v4, v0 offset:67 +; GFX9-ALIGNED-NEXT: ds_read_u8 v5, v0 offset:68 +; GFX9-ALIGNED-NEXT: ds_read_u8 v1, v0 offset:69 +; GFX9-ALIGNED-NEXT: ds_read_u8 v6, v0 offset:70 +; GFX9-ALIGNED-NEXT: ds_read_u8 v7, v0 offset:71 +; GFX9-ALIGNED-NEXT: ds_read_u8 v0, v0 offset:72 +; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v6, 8, v6 +; GFX9-ALIGNED-NEXT: v_or_b32_e32 v1, v6, v1 +; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v0, 8, v0 +; GFX9-ALIGNED-NEXT: v_or_b32_sdwa v0, v0, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX9-ALIGNED-NEXT: v_or_b32_e32 v1, v0, v1 +; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v0, 8, v3 +; GFX9-ALIGNED-NEXT: v_or_b32_e32 v0, v0, v2 +; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v2, 8, v5 +; GFX9-ALIGNED-NEXT: v_or_b32_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX9-ALIGNED-NEXT: v_or_b32_e32 v0, v2, v0 +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v3, s1 +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v2, s0 +; GFX9-ALIGNED-NEXT: global_store_dwordx2 v[2:3], v[0:1], off +; GFX9-ALIGNED-NEXT: s_endpgm +; +; GFX9-UNALIGNED-LABEL: read2_v2i32_align1_odd_offset: +; GFX9-UNALIGNED: ; %bb.0: ; %entry +; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, 0x41 +; GFX9-UNALIGNED-NEXT: ds_read2_b32 v[0:1], v0 offset1:1 +; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v3, s1 +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v2, s0 +; GFX9-UNALIGNED-NEXT: global_store_dwordx2 v[2:3], v[0:1], off +; GFX9-UNALIGNED-NEXT: s_endpgm entry: %load = load <2 x i32>, <2 x i32> addrspace(3)* bitcast (i8 addrspace(3)* getelementptr (i8, i8 addrspace(3)* bitcast ([100 x <2 x i32>] addrspace(3)* @v2i32_align1 to i8 addrspace(3)*), i32 65) to <2 x i32> addrspace(3)*), align 1 store <2 x i32> %load, <2 x i32> addrspace(1)* %out diff --git a/llvm/test/CodeGen/AMDGPU/ds_write2.ll b/llvm/test/CodeGen/AMDGPU/ds_write2.ll index a4a5d47c9e05..f4657490d34d 100644 --- a/llvm/test/CodeGen/AMDGPU/ds_write2.ll +++ b/llvm/test/CodeGen/AMDGPU/ds_write2.ll @@ -1,20 +1,37 @@ -; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,CI %s -; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global,-unaligned-access-mode < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9,GFX9-ALIGNED %s -; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global,+unaligned-access-mode < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9,GFX9-UNALIGNED %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s +; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,-unaligned-access-mode < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-ALIGNED %s +; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+unaligned-access-mode < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-UNALIGNED %s @lds = addrspace(3) global [512 x float] undef, align 4 @lds.f64 = addrspace(3) global [512 x double] undef, align 8 -; GCN-LABEL: {{^}}simple_write2_one_val_f32: -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-DAG: {{buffer|flat|global}}_load_dword [[VAL:v[0-9]+]] -; GCN-DAG: v_lshlrev_b32_e32 [[VBASE:v[0-9]+]], 2, v{{[0-9]+}} -; GCN-DAG: v_add_{{[ui]}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}lds@abs32@lo, [[VBASE]] -; GCN: ds_write2_b32 [[VPTR]], [[VAL]], [[VAL]] offset1:8 -; GCN: s_endpgm define amdgpu_kernel void @simple_write2_one_val_f32(float addrspace(1)* %C, float addrspace(1)* %in) #0 { +; CI-LABEL: simple_write2_one_val_f32: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: buffer_load_dword v1, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write2_b32 v0, v1, v1 offset1:8 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_write2_one_val_f32: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dword v1, v0, s[0:1] +; GFX9-NEXT: v_add_u32_e32 v0, lds@abs32@lo, v0 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: ds_write2_b32 v0, v1, v1 offset1:8 +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in.gep = getelementptr float, float addrspace(1)* %in, i32 %x.i %val = load float, float addrspace(1)* %in.gep, align 4 @@ -26,21 +43,34 @@ define amdgpu_kernel void @simple_write2_one_val_f32(float addrspace(1)* %C, flo ret void } -; GCN-LABEL: {{^}}simple_write2_two_val_f32: -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; CI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; CI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 - -; GFX9-DAG: global_load_dword [[VAL0:v[0-9]+]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}{{$}} -; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} offset:4{{$}} - -; GCN-DAG: v_lshlrev_b32_e32 [[VBASE:v[0-9]+]], 2, v{{[0-9]+}} -; GCN-DAG: v_add_{{[ui]}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}lds@abs32@lo, [[VBASE]] -; GCN: ds_write2_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:8 -; GCN: s_endpgm define amdgpu_kernel void @simple_write2_two_val_f32(float addrspace(1)* %C, float addrspace(1)* %in) #0 { +; CI-LABEL: simple_write2_two_val_f32: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: buffer_load_dword v1, v[0:1], s[0:3], 0 addr64 offset:4 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write2_b32 v0, v2, v1 offset1:8 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_write2_two_val_f32: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dword v1, v0, s[0:1] +; GFX9-NEXT: global_load_dword v2, v0, s[0:1] offset:4 +; GFX9-NEXT: v_add_u32_e32 v0, lds@abs32@lo, v0 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:8 +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in.gep.0 = getelementptr float, float addrspace(1)* %in, i32 %x.i %in.gep.1 = getelementptr float, float addrspace(1)* %in.gep.0, i32 1 @@ -54,15 +84,41 @@ define amdgpu_kernel void @simple_write2_two_val_f32(float addrspace(1)* %C, flo ret void } -; GCN-LABEL: @simple_write2_two_val_f32_volatile_0 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-NOT: ds_write2_b32 -; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} -; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:32 -; GCN: s_endpgm define amdgpu_kernel void @simple_write2_two_val_f32_volatile_0(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 { +; CI-LABEL: simple_write2_two_val_f32_volatile_0: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: s_mov_b64 s[0:1], s[4:5] +; CI-NEXT: s_mov_b64 s[4:5], s[6:7] +; CI-NEXT: s_mov_b64 s[6:7], s[2:3] +; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: buffer_load_dword v1, v[0:1], s[4:7], 0 addr64 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt vmcnt(1) +; CI-NEXT: ds_write_b32 v0, v2 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write_b32 v0, v1 offset:32 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_write2_two_val_f32_volatile_0: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dword v1, v0, s[0:1] +; GFX9-NEXT: global_load_dword v2, v0, s[2:3] +; GFX9-NEXT: v_add_u32_e32 v0, lds@abs32@lo, v0 +; GFX9-NEXT: s_waitcnt vmcnt(1) +; GFX9-NEXT: ds_write_b32 v0, v1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: ds_write_b32 v0, v2 offset:32 +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i %in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %x.i @@ -76,15 +132,41 @@ define amdgpu_kernel void @simple_write2_two_val_f32_volatile_0(float addrspace( ret void } -; GCN-LABEL: @simple_write2_two_val_f32_volatile_1 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-NOT: ds_write2_b32 -; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} -; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:32 -; GCN: s_endpgm define amdgpu_kernel void @simple_write2_two_val_f32_volatile_1(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 { +; CI-LABEL: simple_write2_two_val_f32_volatile_1: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: s_mov_b64 s[0:1], s[4:5] +; CI-NEXT: s_mov_b64 s[4:5], s[6:7] +; CI-NEXT: s_mov_b64 s[6:7], s[2:3] +; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: buffer_load_dword v1, v[0:1], s[4:7], 0 addr64 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt vmcnt(1) +; CI-NEXT: ds_write_b32 v0, v2 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write_b32 v0, v1 offset:32 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_write2_two_val_f32_volatile_1: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dword v1, v0, s[0:1] +; GFX9-NEXT: global_load_dword v2, v0, s[2:3] +; GFX9-NEXT: v_add_u32_e32 v0, lds@abs32@lo, v0 +; GFX9-NEXT: s_waitcnt vmcnt(1) +; GFX9-NEXT: ds_write_b32 v0, v1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: ds_write_b32 v0, v2 offset:32 +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i %in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %x.i @@ -99,28 +181,41 @@ define amdgpu_kernel void @simple_write2_two_val_f32_volatile_1(float addrspace( } ; 2 data subregisters from different super registers. -; GCN-LABEL: {{^}}simple_write2_two_val_subreg2_mixed_f32: -; GFX9-NOT: m0 - -; CI: buffer_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:{{[0-9]+\]}} -; CI: buffer_load_dwordx2 v{{\[[0-9]+}}:[[VAL1:[0-9]+]]{{\]}} -; CI-DAG: s_mov_b32 m0 - -; CI-DAG: v_lshlrev_b32_e32 [[VOFS:v[0-9]+]], 2, v{{[0-9]+}} -; CI-DAG: v_add_i32_e32 [[VPTR:v[0-9]+]], vcc, lds@abs32@lo, [[VOFS]] -; -; TODO: This should be an s_mov_b32. The v_mov_b32 gets introduced by an +; TODO: GFX9 has v_mov_b32_e32 v2, lds@abs32@lo +; This should be an s_mov_b32. The v_mov_b32 gets introduced by an ; early legalization of the constant bus constraint on the v_lshl_add_u32, ; and then SIFoldOperands folds in an unlucky order. -; GFX9-DAG: v_mov_b32_e32 [[VBASE:v[0-9]+]], lds@abs32@lo -; GFX9-DAG: v_lshl_add_u32 [[VPTR:v[0-9]+]], {{v[0-9]+}}, 2, [[VBASE]] - -; GFX9-DAG: global_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:{{[0-9]+\]}} -; GFX9-DAG: global_load_dwordx2 v{{\[[0-9]+}}:[[VAL1:[0-9]+]]{{\]}} - -; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8 -; GCN: s_endpgm define amdgpu_kernel void @simple_write2_two_val_subreg2_mixed_f32(float addrspace(1)* %C, <2 x float> addrspace(1)* %in) #0 { +; CI-LABEL: simple_write2_two_val_subreg2_mixed_f32: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb +; CI-NEXT: v_lshlrev_b32_e32 v1, 3, v0 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_mov_b32_e32 v2, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: buffer_load_dwordx2 v[3:4], v[1:2], s[0:3], 0 addr64 +; CI-NEXT: buffer_load_dwordx2 v[1:2], v[1:2], s[0:3], 0 addr64 offset:8 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write2_b32 v0, v3, v2 offset1:8 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_write2_two_val_subreg2_mixed_f32: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dwordx2 v[1:2], v3, s[0:1] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: global_load_dwordx2 v[2:3], v3, s[0:1] offset:8 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, lds@abs32@lo +; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v2 +; GFX9-NEXT: ds_write2_b32 v0, v1, v3 offset1:8 +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in.gep.0 = getelementptr <2 x float>, <2 x float> addrspace(1)* %in, i32 %x.i %in.gep.1 = getelementptr <2 x float>, <2 x float> addrspace(1)* %in.gep.0, i32 1 @@ -136,20 +231,34 @@ define amdgpu_kernel void @simple_write2_two_val_subreg2_mixed_f32(float addrspa ret void } -; GCN-LABEL: @simple_write2_two_val_subreg2_f32 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-DAG: {{buffer|global}}_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}} - -; CI-DAG: v_lshlrev_b32_e32 [[VOFS:v[0-9]+]], 2, v{{[0-9]+}} -; CI-DAG: v_add_i32_e32 [[VPTR:v[0-9]+]], vcc, lds@abs32@lo, [[VOFS]] -; GFX9-DAG: v_mov_b32_e32 [[VBASE:v[0-9]+]], lds@abs32@lo -; GFX9-DAG: v_lshl_add_u32 [[VPTR:v[0-9]+]], v{{[0-9]+}}, 2, [[VBASE]] - -; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8 -; GCN: s_endpgm define amdgpu_kernel void @simple_write2_two_val_subreg2_f32(float addrspace(1)* %C, <2 x float> addrspace(1)* %in) #0 { +; CI-LABEL: simple_write2_two_val_subreg2_f32: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb +; CI-NEXT: v_lshlrev_b32_e32 v1, 3, v0 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_mov_b32_e32 v2, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: buffer_load_dwordx2 v[1:2], v[1:2], s[0:3], 0 addr64 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:8 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_write2_two_val_subreg2_f32: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 3, v0 +; GFX9-NEXT: v_mov_b32_e32 v3, lds@abs32@lo +; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v3 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dwordx2 v[1:2], v1, s[0:1] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:8 +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in.gep = getelementptr <2 x float>, <2 x float> addrspace(1)* %in, i32 %x.i %val = load <2 x float>, <2 x float> addrspace(1)* %in.gep, align 8 @@ -163,20 +272,34 @@ define amdgpu_kernel void @simple_write2_two_val_subreg2_f32(float addrspace(1)* ret void } -; GCN-LABEL: @simple_write2_two_val_subreg4_f32 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-DAG: {{buffer|global}}_load_dwordx4 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}} - -; CI-DAG: v_lshlrev_b32_e32 [[VOFS:v[0-9]+]], 2, v{{[0-9]+}} -; CI-DAG: v_add_i32_e32 [[VPTR:v[0-9]+]], vcc, lds@abs32@lo, [[VOFS]] -; GFX9-DAG: v_mov_b32_e32 [[VBASE:v[0-9]+]], lds@abs32@lo -; GFX9-DAG: v_lshl_add_u32 [[VPTR:v[0-9]+]], v{{[0-9]+}}, 2, [[VBASE]] - -; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8 -; GCN: s_endpgm define amdgpu_kernel void @simple_write2_two_val_subreg4_f32(float addrspace(1)* %C, <4 x float> addrspace(1)* %in) #0 { +; CI-LABEL: simple_write2_two_val_subreg4_f32: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb +; CI-NEXT: v_lshlrev_b32_e32 v1, 4, v0 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_mov_b32_e32 v2, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: buffer_load_dwordx4 v[1:4], v[1:2], s[0:3], 0 addr64 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write2_b32 v0, v1, v4 offset1:8 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_write2_two_val_subreg4_f32: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 4, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dwordx4 v[1:4], v1, s[0:1] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, lds@abs32@lo +; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v2 +; GFX9-NEXT: ds_write2_b32 v0, v1, v4 offset1:8 +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in.gep = getelementptr <4 x float>, <4 x float> addrspace(1)* %in, i32 %x.i %val = load <4 x float>, <4 x float> addrspace(1)* %in.gep, align 16 @@ -190,22 +313,34 @@ define amdgpu_kernel void @simple_write2_two_val_subreg4_f32(float addrspace(1)* ret void } -; GCN-LABEL: @simple_write2_two_val_max_offset_f32 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; CI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; CI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 - -; GFX9-DAG: global_load_dword [[VAL0:v[0-9]+]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}{{$}} -; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} offset:4{{$}} - -; GCN-DAG: v_lshlrev_b32_e32 [[VBASE:v[0-9]+]], 2, v{{[0-9]+}} -; GCN-DAG: v_add_{{[ui]}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}lds@abs32@lo, [[VBASE]] - -; GCN: ds_write2_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:255 -; GCN: s_endpgm define amdgpu_kernel void @simple_write2_two_val_max_offset_f32(float addrspace(1)* %C, float addrspace(1)* %in) #0 { +; CI-LABEL: simple_write2_two_val_max_offset_f32: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: buffer_load_dword v1, v[0:1], s[0:3], 0 addr64 offset:4 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write2_b32 v0, v2, v1 offset1:255 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_write2_two_val_max_offset_f32: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dword v1, v0, s[0:1] +; GFX9-NEXT: global_load_dword v2, v0, s[0:1] offset:4 +; GFX9-NEXT: v_add_u32_e32 v0, lds@abs32@lo, v0 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:255 +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in.gep.0 = getelementptr float, float addrspace(1)* %in, i32 %x.i %in.gep.1 = getelementptr float, float addrspace(1)* %in.gep.0, i32 1 @@ -219,14 +354,41 @@ define amdgpu_kernel void @simple_write2_two_val_max_offset_f32(float addrspace( ret void } -; GCN-LABEL: @simple_write2_two_val_too_far_f32 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}} -; GCN: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:1028 -; GCN: s_endpgm define amdgpu_kernel void @simple_write2_two_val_too_far_f32(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 { +; CI-LABEL: simple_write2_two_val_too_far_f32: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: s_mov_b64 s[0:1], s[4:5] +; CI-NEXT: s_mov_b64 s[4:5], s[6:7] +; CI-NEXT: s_mov_b64 s[6:7], s[2:3] +; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: buffer_load_dword v1, v[0:1], s[4:7], 0 addr64 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt vmcnt(1) +; CI-NEXT: ds_write_b32 v0, v2 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write_b32 v0, v1 offset:1028 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_write2_two_val_too_far_f32: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dword v1, v0, s[0:1] +; GFX9-NEXT: global_load_dword v2, v0, s[2:3] +; GFX9-NEXT: v_add_u32_e32 v0, lds@abs32@lo, v0 +; GFX9-NEXT: s_waitcnt vmcnt(1) +; GFX9-NEXT: ds_write_b32 v0, v1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: ds_write_b32 v0, v2 offset:1028 +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i %in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %x.i @@ -240,14 +402,39 @@ define amdgpu_kernel void @simple_write2_two_val_too_far_f32(float addrspace(1)* ret void } -; GCN-LABEL: @simple_write2_two_val_f32_x2 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0:v[0-9]+]], [[VAL1:v[0-9]+]] offset1:8 -; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0]], [[VAL1]] offset0:11 offset1:27 -; GCN: s_endpgm define amdgpu_kernel void @simple_write2_two_val_f32_x2(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 { +; CI-LABEL: simple_write2_two_val_f32_x2: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: s_mov_b64 s[0:1], s[4:5] +; CI-NEXT: s_mov_b64 s[4:5], s[6:7] +; CI-NEXT: s_mov_b64 s[6:7], s[2:3] +; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: buffer_load_dword v1, v[0:1], s[4:7], 0 addr64 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write2_b32 v0, v2, v1 offset1:8 +; CI-NEXT: ds_write2_b32 v0, v2, v1 offset0:11 offset1:27 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_write2_two_val_f32_x2: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dword v1, v0, s[0:1] +; GFX9-NEXT: global_load_dword v2, v0, s[2:3] +; GFX9-NEXT: v_add_u32_e32 v0, lds@abs32@lo, v0 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:8 +; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset0:11 offset1:27 +; GFX9-NEXT: s_endpgm %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %tid.x %in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %tid.x @@ -273,14 +460,39 @@ define amdgpu_kernel void @simple_write2_two_val_f32_x2(float addrspace(1)* %C, ret void } -; GCN-LABEL: @simple_write2_two_val_f32_x2_nonzero_base -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0:v[0-9]+]], [[VAL1:v[0-9]+]] offset0:3 offset1:8 -; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0]], [[VAL1]] offset0:11 offset1:27 -; GCN: s_endpgm define amdgpu_kernel void @simple_write2_two_val_f32_x2_nonzero_base(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 { +; CI-LABEL: simple_write2_two_val_f32_x2_nonzero_base: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: s_mov_b64 s[0:1], s[4:5] +; CI-NEXT: s_mov_b64 s[4:5], s[6:7] +; CI-NEXT: s_mov_b64 s[6:7], s[2:3] +; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: buffer_load_dword v1, v[0:1], s[4:7], 0 addr64 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write2_b32 v0, v2, v1 offset0:3 offset1:8 +; CI-NEXT: ds_write2_b32 v0, v2, v1 offset0:11 offset1:27 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_write2_two_val_f32_x2_nonzero_base: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dword v1, v0, s[0:1] +; GFX9-NEXT: global_load_dword v2, v0, s[2:3] +; GFX9-NEXT: v_add_u32_e32 v0, lds@abs32@lo, v0 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset0:3 offset1:8 +; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset0:11 offset1:27 +; GFX9-NEXT: s_endpgm %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %tid.x %in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %tid.x @@ -306,15 +518,45 @@ define amdgpu_kernel void @simple_write2_two_val_f32_x2_nonzero_base(float addrs ret void } -; GCN-LABEL: @write2_ptr_subreg_arg_two_val_f32 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-NOT: ds_write2_b32 -; GCN: ds_write_b32 -; GCN: ds_write_b32 -; GCN: s_endpgm define amdgpu_kernel void @write2_ptr_subreg_arg_two_val_f32(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1, <2 x float addrspace(3)*> %lds.ptr) #0 { +; CI-LABEL: write2_ptr_subreg_arg_two_val_f32: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb +; CI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xf +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: s_mov_b64 s[0:1], s[4:5] +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: s_mov_b64 s[4:5], s[6:7] +; CI-NEXT: s_mov_b64 s[6:7], s[2:3] +; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 +; CI-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64 +; CI-NEXT: v_mov_b32_e32 v1, s8 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: v_mov_b32_e32 v3, s9 +; CI-NEXT: s_waitcnt vmcnt(1) +; CI-NEXT: ds_write_b32 v1, v2 offset:32 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write_b32 v3, v0 offset:32 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: write2_ptr_subreg_arg_two_val_f32: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x3c +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dword v1, v0, s[4:5] +; GFX9-NEXT: global_load_dword v0, v0, s[6:7] +; GFX9-NEXT: v_mov_b32_e32 v2, s0 +; GFX9-NEXT: v_mov_b32_e32 v3, s1 +; GFX9-NEXT: s_waitcnt vmcnt(1) +; GFX9-NEXT: ds_write_b32 v2, v1 offset:32 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: ds_write_b32 v3, v0 offset:32 +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i %in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %x.i @@ -336,15 +578,32 @@ define amdgpu_kernel void @write2_ptr_subreg_arg_two_val_f32(float addrspace(1)* ret void } -; GCN-LABEL: @simple_write2_one_val_f64 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-DAG: {{buffer|global}}_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]], -; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, v{{[0-9]+}} -; GCN: ds_write2_b64 [[VPTR]], [[VAL]], [[VAL]] offset1:8 -; GCN: s_endpgm define amdgpu_kernel void @simple_write2_one_val_f64(double addrspace(1)* %C, double addrspace(1)* %in) #0 { +; CI-LABEL: simple_write2_one_val_f64: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: buffer_load_dwordx2 v[1:2], v[0:1], s[0:3], 0 addr64 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds.f64@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write2_b64 v0, v[1:2], v[1:2] offset1:8 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_write2_one_val_f64: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 3, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1] +; GFX9-NEXT: v_add_u32_e32 v2, lds.f64@abs32@lo, v2 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: ds_write2_b64 v2, v[0:1], v[0:1] offset1:8 +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in.gep = getelementptr double, double addrspace(1)* %in, i32 %x.i %val = load double, double addrspace(1)* %in.gep, align 8 @@ -356,16 +615,36 @@ define amdgpu_kernel void @simple_write2_one_val_f64(double addrspace(1)* %C, do ret void } -; GCN-LABEL: @misaligned_simple_write2_one_val_f64 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-DAG: {{buffer|global}}_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}} -; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, v{{[0-9]+}} -; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:1 -; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset0:14 offset1:15 -; GCN: s_endpgm define amdgpu_kernel void @misaligned_simple_write2_one_val_f64(double addrspace(1)* %C, double addrspace(1)* %in, double addrspace(3)* %lds) #0 { +; CI-LABEL: misaligned_simple_write2_one_val_f64: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xb +; CI-NEXT: s_load_dword s0, s[0:1], 0xd +; CI-NEXT: s_mov_b32 s7, 0xf000 +; CI-NEXT: s_mov_b32 s6, 0 +; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: buffer_load_dwordx2 v[1:2], v[0:1], s[4:7], 0 addr64 +; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; CI-NEXT: ds_write2_b32 v0, v1, v2 offset0:14 offset1:15 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: misaligned_simple_write2_one_val_f64: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c +; GFX9-NEXT: s_load_dword s0, s[0:1], 0x34 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 3, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3] +; GFX9-NEXT: v_add_u32_e32 v2, s0, v2 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: ds_write2_b32 v2, v0, v1 offset1:1 +; GFX9-NEXT: ds_write2_b32 v2, v0, v1 offset0:14 offset1:15 +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in.gep = getelementptr double, double addrspace(1)* %in, i32 %x.i %val = load double, double addrspace(1)* %in.gep, align 8 @@ -377,21 +656,34 @@ define amdgpu_kernel void @misaligned_simple_write2_one_val_f64(double addrspace ret void } -; GCN-LABEL: @simple_write2_two_val_f64 -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; CI-DAG: buffer_load_dwordx2 [[VAL0:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; CI-DAG: buffer_load_dwordx2 [[VAL1:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 - -; GFX9-DAG: global_load_dwordx2 [[VAL0:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]$}} -; GFX9-DAG: global_load_dwordx2 [[VAL1:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} offset:8 - -; GCN-DAG: v_lshlrev_b32_e32 [[VBASE:v[0-9]+]], 3, v{{[0-9]+}} -; GCN-DAG: v_add_{{[ui]}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}lds.f64@abs32@lo, [[VBASE]] -; GCN: ds_write2_b64 [[VPTR]], [[VAL0]], [[VAL1]] offset1:8 -; GCN: s_endpgm define amdgpu_kernel void @simple_write2_two_val_f64(double addrspace(1)* %C, double addrspace(1)* %in) #0 { +; CI-LABEL: simple_write2_two_val_f64: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, 0 +; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0 +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64 +; CI-NEXT: buffer_load_dwordx2 v[4:5], v[0:1], s[0:3], 0 addr64 offset:8 +; CI-NEXT: v_add_i32_e32 v0, vcc, lds.f64@abs32@lo, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset1:8 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: simple_write2_two_val_f64: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[0:1] offset:8 +; GFX9-NEXT: v_add_u32_e32 v4, lds.f64@abs32@lo, v4 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: ds_write2_b64 v4, v[0:1], v[2:3] offset1:8 +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in.gep.0 = getelementptr double, double addrspace(1)* %in, i32 %x.i %in.gep.1 = getelementptr double, double addrspace(1)* %in.gep.0, i32 1 @@ -407,26 +699,45 @@ define amdgpu_kernel void @simple_write2_two_val_f64(double addrspace(1)* %C, do @foo = addrspace(3) global [4 x i32] undef, align 4 -; GCN-LABEL: @store_constant_adjacent_offsets -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], foo@abs32@lo{{$}} -; GCN: ds_write2_b32 [[PTR]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1 define amdgpu_kernel void @store_constant_adjacent_offsets() { +; CI-LABEL: store_constant_adjacent_offsets: +; CI: ; %bb.0: +; CI-NEXT: s_movk_i32 s0, 0x7b +; CI-NEXT: v_mov_b32_e32 v0, foo@abs32@lo +; CI-NEXT: v_mov_b32_e32 v1, s0 +; CI-NEXT: v_mov_b32_e32 v2, s0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: store_constant_adjacent_offsets: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_movk_i32 s0, 0x7b +; GFX9-NEXT: v_mov_b32_e32 v0, foo@abs32@lo +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: v_mov_b32_e32 v2, s0 +; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; GFX9-NEXT: s_endpgm store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4 store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 1), align 4 ret void } -; GCN-LABEL: @store_constant_disjoint_offsets -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-DAG: v_mov_b32_e32 [[VAL:v[0-9]+]], 0x7b{{$}} -; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], foo@abs32@lo{{$}} -; GCN: ds_write2_b32 [[PTR]], [[VAL]], [[VAL]] offset1:2 define amdgpu_kernel void @store_constant_disjoint_offsets() { +; CI-LABEL: store_constant_disjoint_offsets: +; CI: ; %bb.0: +; CI-NEXT: v_mov_b32_e32 v0, foo@abs32@lo +; CI-NEXT: v_mov_b32_e32 v1, 0x7b +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_write2_b32 v0, v1, v1 offset1:2 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: store_constant_disjoint_offsets: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_mov_b32_e32 v0, foo@abs32@lo +; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b +; GFX9-NEXT: ds_write2_b32 v0, v1, v1 offset1:2 +; GFX9-NEXT: s_endpgm store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4 store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 2), align 4 ret void @@ -434,21 +745,35 @@ define amdgpu_kernel void @store_constant_disjoint_offsets() { @bar = addrspace(3) global [4 x i64] undef, align 4 -; GCN-LABEL: @store_misaligned64_constant_offsets -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], bar@abs32@lo{{$}} -; CI-DAG: ds_write2_b32 [[PTR]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1 -; CI-DAG: ds_write2_b32 [[PTR]], v{{[0-9]+}}, v{{[0-9]+}} offset0:2 offset1:3 - -; GFX9-ALIGNED-DAG: ds_write2_b32 [[PTR]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1 -; GFX9-ALIGNED-DAG: ds_write2_b32 [[PTR]], v{{[0-9]+}}, v{{[0-9]+}} offset0:2 offset1:3 - -; GFX9-UNALIGNED: ds_write_b128 [[PTR]], {{v\[[0-9]+:[0-9]+\]}} - -; GCN: s_endpgm define amdgpu_kernel void @store_misaligned64_constant_offsets() { +; CI-LABEL: store_misaligned64_constant_offsets: +; CI: ; %bb.0: +; CI-NEXT: v_mov_b32_e32 v0, bar@abs32@lo +; CI-NEXT: v_mov_b32_e32 v1, 0x7b +; CI-NEXT: v_mov_b32_e32 v2, 0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; CI-NEXT: ds_write2_b32 v0, v1, v2 offset0:2 offset1:3 +; CI-NEXT: s_endpgm +; +; GFX9-ALIGNED-LABEL: store_misaligned64_constant_offsets: +; GFX9-ALIGNED: ; %bb.0: +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v0, bar@abs32@lo +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v1, 0x7b +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v2, 0 +; GFX9-ALIGNED-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; GFX9-ALIGNED-NEXT: ds_write2_b32 v0, v1, v2 offset0:2 offset1:3 +; GFX9-ALIGNED-NEXT: s_endpgm +; +; GFX9-UNALIGNED-LABEL: store_misaligned64_constant_offsets: +; GFX9-UNALIGNED: ; %bb.0: +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, 0x7b +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v4, bar@abs32@lo +; GFX9-UNALIGNED-NEXT: ds_write_b128 v4, v[0:3] +; GFX9-UNALIGNED-NEXT: s_endpgm store i64 123, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 0), align 4 store i64 123, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 1), align 4 ret void @@ -456,19 +781,33 @@ define amdgpu_kernel void @store_misaligned64_constant_offsets() { @bar.large = addrspace(3) global [4096 x i64] undef, align 4 -; GCN-LABEL: @store_misaligned64_constant_large_offsets -; CI-DAG: s_mov_b32 m0 -; GFX9-NOT: m0 - -; GCN-DAG: s_mov_b32 [[SBASE0:s[0-9]+]], bar.large@abs32@lo -; GCN-DAG: s_add_i32 [[SBASE1:s[0-9]+]], [[SBASE0]], 0x4000{{$}} -; GCN-DAG: s_addk_i32 [[SBASE0]], 0x7ff8{{$}} -; GCN-DAG: v_mov_b32_e32 [[VBASE0:v[0-9]+]], [[SBASE0]]{{$}} -; GCN-DAG: v_mov_b32_e32 [[VBASE1:v[0-9]+]], [[SBASE1]]{{$}} -; GCN-DAG: ds_write2_b32 [[VBASE0]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1 -; GCN-DAG: ds_write2_b32 [[VBASE1]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1 -; GCN: s_endpgm define amdgpu_kernel void @store_misaligned64_constant_large_offsets() { +; CI-LABEL: store_misaligned64_constant_large_offsets: +; CI: ; %bb.0: +; CI-NEXT: s_mov_b32 s0, bar.large@abs32@lo +; CI-NEXT: s_add_i32 s1, s0, 0x4000 +; CI-NEXT: v_mov_b32_e32 v0, s1 +; CI-NEXT: v_mov_b32_e32 v1, 0x7b +; CI-NEXT: v_mov_b32_e32 v2, 0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_addk_i32 s0, 0x7ff8 +; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; CI-NEXT: v_mov_b32_e32 v0, s0 +; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: store_misaligned64_constant_large_offsets: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s0, bar.large@abs32@lo +; GFX9-NEXT: s_add_i32 s1, s0, 0x4000 +; GFX9-NEXT: v_mov_b32_e32 v0, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b +; GFX9-NEXT: v_mov_b32_e32 v2, 0 +; GFX9-NEXT: s_addk_i32 s0, 0x7ff8 +; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; GFX9-NEXT: v_mov_b32_e32 v0, s0 +; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; GFX9-NEXT: s_endpgm store i64 123, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 2048), align 4 store i64 123, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 4095), align 4 ret void @@ -478,6 +817,48 @@ define amdgpu_kernel void @store_misaligned64_constant_large_offsets() { @sgemm.lB = internal unnamed_addr addrspace(3) global [776 x float] undef, align 4 define amdgpu_kernel void @write2_sgemm_sequence(float addrspace(1)* %C, i32 %lda, i32 %ldb, float addrspace(1)* %in) #0 { +; CI-LABEL: write2_sgemm_sequence: +; CI: ; %bb.0: +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd +; CI-NEXT: s_lshl_b32 s2, s2, 2 +; CI-NEXT: s_add_i32 s3, s2, 0xc20 +; CI-NEXT: v_mov_b32_e32 v0, s3 +; CI-NEXT: s_addk_i32 s2, 0xc60 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: s_load_dword s0, s[0:1], 0x0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_mov_b32_e32 v2, s0 +; CI-NEXT: v_mov_b32_e32 v3, s0 +; CI-NEXT: ds_write2_b32 v0, v2, v3 offset1:1 +; CI-NEXT: v_mov_b32_e32 v0, s2 +; CI-NEXT: ds_write2_b32 v0, v2, v3 offset1:1 +; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v1 +; CI-NEXT: ds_write2_b32 v0, v2, v3 offset1:1 +; CI-NEXT: ds_write2_b32 v0, v2, v3 offset0:32 offset1:33 +; CI-NEXT: ds_write2_b32 v0, v2, v3 offset0:64 offset1:65 +; CI-NEXT: s_endpgm +; +; GFX9-LABEL: write2_sgemm_sequence: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; GFX9-NEXT: s_lshl_b32 s2, s2, 2 +; GFX9-NEXT: s_add_i32 s3, s2, 0xc20 +; GFX9-NEXT: s_addk_i32 s2, 0xc60 +; GFX9-NEXT: v_mov_b32_e32 v0, s3 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dword s0, s[0:1], 0x0 +; GFX9-NEXT: v_mov_b32_e32 v2, s2 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v3, s0 +; GFX9-NEXT: v_mov_b32_e32 v4, s0 +; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset1:1 +; GFX9-NEXT: ds_write2_b32 v2, v3, v4 offset1:1 +; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v1 +; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset1:1 +; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset0:32 offset1:33 +; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset0:64 offset1:65 +; GFX9-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workgroup.id.x() #1 %y.i = tail call i32 @llvm.amdgcn.workitem.id.y() #1 %val = load float, float addrspace(1)* %in @@ -512,18 +893,55 @@ define amdgpu_kernel void @write2_sgemm_sequence(float addrspace(1)* %C, i32 %ld ret void } -; GCN-LABEL: {{^}}simple_write2_v4f32_superreg_align4: -; CI: s_mov_b32 m0 -; GFX9-NOT: m0 - -; CI: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset1:1{{$}} -; CI: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset0:2 offset1:3{{$}} - -; GFX9-ALIGNED-DAG: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset1:1{{$}} -; GFX9-ALIGNED-DAG: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset0:2 offset1:3{{$}} - -; GFX9-UNALIGNED: ds_write_b128 {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}} define amdgpu_kernel void @simple_write2_v4f32_superreg_align4(<4 x float> addrspace(3)* %out, <4 x float> addrspace(1)* %in) #0 { +; CI-LABEL: simple_write2_v4f32_superreg_align4: +; CI: ; %bb.0: +; CI-NEXT: s_load_dword s4, s[0:1], 0x9 +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb +; CI-NEXT: v_lshlrev_b32_e32 v0, 4, v0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_add_i32_e32 v0, vcc, s4, v0 +; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; CI-NEXT: s_waitcnt lgkmcnt(0) +; CI-NEXT: v_mov_b32_e32 v1, s0 +; CI-NEXT: v_mov_b32_e32 v2, s1 +; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; CI-NEXT: v_mov_b32_e32 v3, s2 +; CI-NEXT: v_mov_b32_e32 v1, s3 +; CI-NEXT: ds_write2_b32 v0, v3, v1 offset0:2 offset1:3 +; CI-NEXT: s_endpgm +; +; GFX9-ALIGNED-LABEL: simple_write2_v4f32_superreg_align4: +; GFX9-ALIGNED: ; %bb.0: +; GFX9-ALIGNED-NEXT: s_load_dword s4, s[0:1], 0x24 +; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c +; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-ALIGNED-NEXT: v_lshl_add_u32 v0, v0, 4, s4 +; GFX9-ALIGNED-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v3, s2 +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v4, s3 +; GFX9-ALIGNED-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; GFX9-ALIGNED-NEXT: ds_write2_b32 v0, v3, v4 offset0:2 offset1:3 +; GFX9-ALIGNED-NEXT: s_endpgm +; +; GFX9-UNALIGNED-LABEL: simple_write2_v4f32_superreg_align4: +; GFX9-UNALIGNED: ; %bb.0: +; GFX9-UNALIGNED-NEXT: s_load_dword s4, s[0:1], 0x24 +; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c +; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-UNALIGNED-NEXT: v_lshl_add_u32 v4, v0, 4, s4 +; GFX9-UNALIGNED-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, s0 +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v1, s1 +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v2, s2 +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v3, s3 +; GFX9-UNALIGNED-NEXT: ds_write_b128 v4, v[0:3] +; GFX9-UNALIGNED-NEXT: s_endpgm %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %in.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %in %val0 = load <4 x float>, <4 x float> addrspace(1)* %in.gep, align 4 @@ -534,14 +952,47 @@ define amdgpu_kernel void @simple_write2_v4f32_superreg_align4(<4 x float> addrs @v2i32_align1 = internal addrspace(3) global [100 x <2 x i32>] undef, align 1 -; GCN-LABEL: {{^}}write2_v2i32_align1_odd_offset: -; CI-COUNT-8: ds_write_b8 - -; GFX9-ALIGNED-COUNT-8: ds_write_b8 - -; GFX9-UNALIGNED: v_mov_b32_e32 [[BASE_ADDR:v[0-9]+]], 0x41{{$}} -; GFX9-UNALIGNED: ds_write2_b32 [[BASE_ADDR]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1{{$}} define amdgpu_kernel void @write2_v2i32_align1_odd_offset() { +; CI-LABEL: write2_v2i32_align1_odd_offset: +; CI: ; %bb.0: ; %entry +; CI-NEXT: v_mov_b32_e32 v0, 0x7b +; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: s_mov_b32 m0, -1 +; CI-NEXT: ds_write_b8 v1, v0 offset:65 +; CI-NEXT: v_mov_b32_e32 v0, 1 +; CI-NEXT: ds_write_b8 v1, v0 offset:70 +; CI-NEXT: v_mov_b32_e32 v0, 0xc8 +; CI-NEXT: ds_write_b8 v1, v0 offset:69 +; CI-NEXT: ds_write_b8 v1, v1 offset:68 +; CI-NEXT: ds_write_b8 v1, v1 offset:67 +; CI-NEXT: ds_write_b8 v1, v1 offset:66 +; CI-NEXT: ds_write_b8 v1, v1 offset:72 +; CI-NEXT: ds_write_b8 v1, v1 offset:71 +; CI-NEXT: s_endpgm +; +; GFX9-ALIGNED-LABEL: write2_v2i32_align1_odd_offset: +; GFX9-ALIGNED: ; %bb.0: ; %entry +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v0, 0x7b +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v0 offset:65 +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v0, 1 +; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v0 offset:70 +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v0, 0xc8 +; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v0 offset:69 +; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v1 offset:68 +; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v1 offset:67 +; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v1 offset:66 +; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v1 offset:72 +; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v1 offset:71 +; GFX9-ALIGNED-NEXT: s_endpgm +; +; GFX9-UNALIGNED-LABEL: write2_v2i32_align1_odd_offset: +; GFX9-UNALIGNED: ; %bb.0: ; %entry +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, 0x41 +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v1, 0x7b +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v2, 0x1c8 +; GFX9-UNALIGNED-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; GFX9-UNALIGNED-NEXT: s_endpgm entry: store <2 x i32> , <2 x i32> addrspace(3)* bitcast (i8 addrspace(3)* getelementptr (i8, i8 addrspace(3)* bitcast ([100 x <2 x i32>] addrspace(3)* @v2i32_align1 to i8 addrspace(3)*), i32 65) to <2 x i32> addrspace(3)*), align 1 ret void