From c6c9dc877301ed41566344664ef9502c95f8a411 Mon Sep 17 00:00:00 2001 From: Farhana Aleen Date: Fri, 16 Mar 2018 18:12:00 +0000 Subject: [PATCH] [AMDGPU] Supported ds_write_b128 generation. Summary: This is a follow-on patch of https://reviews.llvm.org/D44210 Author: FarhanaAleen Reviewed By: msearles Subscribers: llvm-commits, AMDGPU Differential Revision: https://reviews.llvm.org/D44319 llvm-svn: 327726 --- llvm/lib/Target/AMDGPU/AMDGPUInstructions.td | 3 +++ llvm/lib/Target/AMDGPU/DSInstructions.td | 2 ++ llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 12 ++++++------ llvm/lib/Target/AMDGPU/SIInstrInfo.td | 5 +++++ llvm/test/CodeGen/AMDGPU/load-local-f32.ll | 11 ++++++++--- llvm/test/CodeGen/AMDGPU/load-local-f64.ll | 5 ++++- llvm/test/CodeGen/AMDGPU/load-local-i16.ll | 11 ++++++++--- llvm/test/CodeGen/AMDGPU/load-local-i32.ll | 11 ++++++++--- llvm/test/CodeGen/AMDGPU/load-local-i64.ll | 11 +++++++---- llvm/test/CodeGen/AMDGPU/load-local-i8.ll | 11 ++++++++--- 10 files changed, 59 insertions(+), 23 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td index e719933b052d..4d7da21a591c 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -383,6 +383,9 @@ def store_align8_local : Aligned8Bytes < (ops node:$val, node:$ptr), (store_local node:$val, node:$ptr) >; +def store_align16_local : Aligned16Bytes < + (ops node:$val, node:$ptr), (store_local node:$val, node:$ptr) +>; def load_flat : FlatLoad ; def az_extloadi8_flat : FlatLoad ; diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td index 88484c06218a..5af8b9ace43d 100644 --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -719,6 +719,8 @@ def : DS64Bit4ByteAlignedWritePat; let AddedComplexity = 100 in { defm : DSWritePat_mc ; +defm : DSWritePat_mc ; + } // End AddedComplexity = 100 class DSAtomicRetPat : GCNPat < (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$value), diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 8463b22d60a7..702b059e9221 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -5833,14 +5833,14 @@ SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { llvm_unreachable("unsupported private_element_size"); } } else if (AS == AMDGPUASI.LOCAL_ADDRESS) { + // Use ds_write_b128 if possible. + if (Subtarget->useDS128(EnableDS128) && Store->getAlignment() >= 16 && + VT.getStoreSize() == 16) + return SDValue(); + if (NumElements > 2) return SplitVectorStore(Op, DAG); - - if (NumElements == 2) - return Op; - - // If properly aligned, if we split we might be able to use ds_write_b64. - return SplitVectorStore(Op, DAG); + return SDValue(); } else { llvm_unreachable("unhandled address space"); } diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td index 7428f4d5f52f..777bb0ac9052 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -457,11 +457,16 @@ def store_glue_align8 : Aligned8Bytes < (ops node:$value, node:$ptr), (store_glue node:$value, node:$ptr) >; +def store_glue_align16 : Aligned16Bytes < + (ops node:$value, node:$ptr), (store_glue node:$value, node:$ptr) +>; + def store_local_m0 : StoreFrag, LocalAddress; def truncstorei8_local_m0 : StoreFrag, LocalAddress; def truncstorei16_local_m0 : StoreFrag, LocalAddress; def store_align8_local_m0 : StoreFrag, LocalAddress; +def store_align16_local_m0 : StoreFrag, LocalAddress; def si_setcc_uniform : PatFrag < (ops node:$lhs, node:$rhs, node:$cond), diff --git a/llvm/test/CodeGen/AMDGPU/load-local-f32.ll b/llvm/test/CodeGen/AMDGPU/load-local-f32.ll index 3d8720888c79..c2722717fb36 100644 --- a/llvm/test/CodeGen/AMDGPU/load-local-f32.ll +++ b/llvm/test/CodeGen/AMDGPU/load-local-f32.ll @@ -2,7 +2,7 @@ ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s -; Testing for ds_read_128 +; Testing for ds_read/write_128 ; RUN: llc -march=amdgcn -mcpu=tahiti -amdgpu-ds128 < %s | FileCheck -check-prefixes=SI,FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s ; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s @@ -127,17 +127,22 @@ entry: ret void } -; Tests if ds_read_b128 gets generated for the 16 byte aligned load. +; Tests if ds_read/write_b128 gets generated for the 16 byte aligned load. ; FUNC-LABEL: {{^}}local_v4f32_to_128: + ; SI-NOT: ds_read_b128 +; SI-NOT: ds_write_b128 + ; CIVI: ds_read_b128 +; CIVI: ds_write_b128 + ; EG: LDS_READ_RET ; EG: LDS_READ_RET ; EG: LDS_READ_RET ; EG: LDS_READ_RET define amdgpu_kernel void @local_v4f32_to_128(<4 x float> addrspace(3)* %out, <4 x float> addrspace(3)* %in) { %ld = load <4 x float>, <4 x float> addrspace(3)* %in, align 16 - store <4 x float> %ld, <4 x float> addrspace(3)* %out + store <4 x float> %ld, <4 x float> addrspace(3)* %out, align 16 ret void } diff --git a/llvm/test/CodeGen/AMDGPU/load-local-f64.ll b/llvm/test/CodeGen/AMDGPU/load-local-f64.ll index 14c31e67d7f2..f4040db37973 100644 --- a/llvm/test/CodeGen/AMDGPU/load-local-f64.ll +++ b/llvm/test/CodeGen/AMDGPU/load-local-f64.ll @@ -176,7 +176,10 @@ entry: ; Tests if ds_read_b128 gets generated for the 16 byte aligned load. ; FUNC-LABEL: {{^}}local_load_v2f64_to_128: + ; CIVI: ds_read_b128 +; CIVI: ds_write_b128 + ; EG: LDS_READ_RET ; EG: LDS_READ_RET ; EG: LDS_READ_RET @@ -184,7 +187,7 @@ entry: define amdgpu_kernel void @local_load_v2f64_to_128(<2 x double> addrspace(3)* %out, <2 x double> addrspace(3)* %in) { entry: %ld = load <2 x double>, <2 x double> addrspace(3)* %in, align 16 - store <2 x double> %ld, <2 x double> addrspace(3)* %out + store <2 x double> %ld, <2 x double> addrspace(3)* %out, align 16 ret void } diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll index 7438fd2681dc..83cf85b51260 100644 --- a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll +++ b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll @@ -3,7 +3,7 @@ ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,GFX89,FUNC %s ; RUN: llc -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s -; Testing for ds_read_b128 +; Testing for ds_read/write_b128 ; RUN: llc -march=amdgcn -mcpu=tonga -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s ; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s @@ -939,17 +939,22 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(<32 x i64> addrspace( ; ret void ; } -; Tests if ds_read_b128 gets generated for the 16 byte aligned load. +; Tests if ds_read/write_b128 gets generated for the 16 byte aligned load. ; FUNC-LABEL: {{^}}local_v8i16_to_128: + ; SI-NOT: ds_read_b128 +; SI-NOT: ds_write_b128 + ; CIVI: ds_read_b128 +; CIVI: ds_write_b128 + ; EG: LDS_READ_RET ; EG: LDS_READ_RET ; EG: LDS_READ_RET ; EG: LDS_READ_RET define amdgpu_kernel void @local_v8i16_to_128(<8 x i16> addrspace(3)* %out, <8 x i16> addrspace(3)* %in) { %ld = load <8 x i16>, <8 x i16> addrspace(3)* %in, align 16 - store <8 x i16> %ld, <8 x i16> addrspace(3)* %out + store <8 x i16> %ld, <8 x i16> addrspace(3)* %out, align 16 ret void } diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i32.ll b/llvm/test/CodeGen/AMDGPU/load-local-i32.ll index 1dd7daf95ab0..2d0e989b6861 100644 --- a/llvm/test/CodeGen/AMDGPU/load-local-i32.ll +++ b/llvm/test/CodeGen/AMDGPU/load-local-i32.ll @@ -3,7 +3,7 @@ ; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s -; Testing for ds_read_128 +; Testing for ds_read/write_128 ; RUN: llc -march=amdgcn -mcpu=tahiti -amdgpu-ds128 < %s | FileCheck -check-prefixes=SI,FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s ; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s @@ -180,17 +180,22 @@ define amdgpu_kernel void @local_sextload_v4i32_to_v4i64(<4 x i64> addrspace(3)* ret void } -; Tests if ds_read_b128 gets generated for the 16 byte aligned load. +; Tests if ds_read/write_b128 gets generated for the 16 byte aligned load. ; FUNC-LABEL: {{^}}local_v4i32_to_128: + ; SI-NOT: ds_read_b128 +; SI-NOT: ds_write_b128 + ; CIVI: ds_read_b128 +; CIVI: ds_write_b128 + ; EG: LDS_READ_RET ; EG: LDS_READ_RET ; EG: LDS_READ_RET ; EG: LDS_READ_RET define amdgpu_kernel void @local_v4i32_to_128(<4 x i32> addrspace(3)* %out, <4 x i32> addrspace(3)* %in) { %ld = load <4 x i32>, <4 x i32> addrspace(3)* %in, align 16 - store <4 x i32> %ld, <4 x i32> addrspace(3)* %out + store <4 x i32> %ld, <4 x i32> addrspace(3)* %out, align 16 ret void } diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i64.ll b/llvm/test/CodeGen/AMDGPU/load-local-i64.ll index 359fbb42f40a..697f474f10c4 100644 --- a/llvm/test/CodeGen/AMDGPU/load-local-i64.ll +++ b/llvm/test/CodeGen/AMDGPU/load-local-i64.ll @@ -4,7 +4,7 @@ ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s -; Testing for ds_read_b128 +; Testing for ds_read/write_b128 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s @@ -40,13 +40,16 @@ entry: ret void } -; Tests if ds_read_b128 gets generated for the 16 byte aligned load. +; Tests if ds_read/write_b128 gets generated for the 16 byte aligned load. ; FUNC-LABEL: {{^}}local_load_v2i64_to_128: + ; CIVI: ds_read_b128 +; CIVI: ds_write_b128 + define amdgpu_kernel void @local_load_v2i64_to_128(<2 x i64> addrspace(3)* %out, <2 x i64> addrspace(3)* %in) { entry: - %ld = load <2 x i64>, <2 x i64> addrspace(3)* %in - store <2 x i64> %ld, <2 x i64> addrspace(3)* %out + %ld = load <2 x i64>, <2 x i64> addrspace(3)* %in, align 16 + store <2 x i64> %ld, <2 x i64> addrspace(3)* %out, align 16 ret void } diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i8.ll b/llvm/test/CodeGen/AMDGPU/load-local-i8.ll index e1931af042bc..898d35d60a21 100644 --- a/llvm/test/CodeGen/AMDGPU/load-local-i8.ll +++ b/llvm/test/CodeGen/AMDGPU/load-local-i8.ll @@ -3,7 +3,7 @@ ; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s ; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s -; Testing for ds_read_b128 +; Testing for ds_read/write_b128 ; RUN: llc -march=amdgcn -mcpu=tonga -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s ; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s @@ -1024,17 +1024,22 @@ define amdgpu_kernel void @local_sextload_v32i8_to_v32i16(<32 x i16> addrspace(3 ; ret void ; } -; Tests if ds_read_b128 gets generated for the 16 byte aligned load. +; Tests if ds_read/write_b128 gets generated for the 16 byte aligned load. ; FUNC-LABEL: {{^}}local_v16i8_to_128: + ; SI-NOT: ds_read_b128 +; SI-NOT: ds_write_b128 + ; CIVI: ds_read_b128 +; CIVI: ds_write_b128 + ; EG: LDS_READ_RET ; EG: LDS_READ_RET ; EG: LDS_READ_RET ; EG: LDS_READ_RET define amdgpu_kernel void @local_v16i8_to_128(<16 x i8> addrspace(3)* %out, <16 x i8> addrspace(3)* %in) { %ld = load <16 x i8>, <16 x i8> addrspace(3)* %in, align 16 - store <16 x i8> %ld, <16 x i8> addrspace(3)* %out + store <16 x i8> %ld, <16 x i8> addrspace(3)* %out, align 16 ret void }