2019-05-22 05:49:14 +08:00
|
|
|
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
|
|
|
; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX8,VARIANT0 %s
|
|
|
|
|
; RUN: llc -march=amdgcn -mattr=+auto-waitcnt-before-barrier -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX8,VARIANT1 %s
|
|
|
|
|
; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX9,VARIANT2 %s
|
|
|
|
|
; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+auto-waitcnt-before-barrier -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX9,VARIANT3 %s
|
2016-01-23 05:30:43 +08:00
|
|
|
|
2017-05-18 05:38:21 +08:00
|
|
|
define amdgpu_kernel void @test_barrier(i32 addrspace(1)* %out, i32 %size) #0 {
|
2019-05-22 05:49:14 +08:00
|
|
|
; VARIANT0-LABEL: test_barrier:
|
|
|
|
|
; VARIANT0: ; %bb.0: ; %entry
|
|
|
|
|
; VARIANT0-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9
|
|
|
|
|
; VARIANT0-NEXT: s_load_dword s2, s[0:1], 0xb
|
|
|
|
|
; VARIANT0-NEXT: s_mov_b32 s7, 0xf000
|
|
|
|
|
; VARIANT0-NEXT: s_mov_b32 s6, 0
|
|
|
|
|
; VARIANT0-NEXT: v_lshlrev_b32_e32 v1, 2, v0
|
|
|
|
|
; VARIANT0-NEXT: v_mov_b32_e32 v2, 0
|
[AMDGPU] Remove dubious logic in bidirectional list scheduler
Summary:
pickNodeBidirectional tried to compare the best top candidate and the
best bottom candidate by examining TopCand.Reason and BotCand.Reason.
This is unsound because, after calling pickNodeFromQueue, Cand.Reason
does not reflect the most important reason why Cand was chosen. Rather
it reflects the most recent reason why it beat some other potential
candidate, which could have been for some low priority tie breaker
reason.
I have seen this cause problems where TopCand is a good candidate, but
because TopCand.Reason is ORDER (which is very low priority) it is
repeatedly ignored in favour of a mediocre BotCand. This is not how
bidirectional scheduling is supposed to work.
To fix this I changed the code to always compare TopCand and BotCand
directly, like the generic implementation of pickNodeBidirectional does.
This removes some uncommented AMDGPU-specific logic; if this logic turns
out to be important then perhaps it could be moved into an override of
tryCandidate instead.
Graphics shader benchmarking on gfx10 shows a lot more positive than
negative effects from this change.
Reviewers: arsenm, tstellar, rampitec, kzhuravl, vpykhtin, dstuttard, tpr, atrick, MatzeB
Subscribers: jvesely, wdng, nhaehnle, yaxunl, t-tye, hiraditya, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D68338
2019-10-07 22:33:59 +08:00
|
|
|
; VARIANT0-NEXT: v_not_b32_e32 v3, v0
|
2019-05-22 05:49:14 +08:00
|
|
|
; VARIANT0-NEXT: s_waitcnt lgkmcnt(0)
|
|
|
|
|
; VARIANT0-NEXT: buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
|
[DAGCombine][X86][AArch64][AMDGPU] (x - y) + -1 -> add (xor y, -1), x fold. Try 3
Summary:
This prevents regressions in next patch,
and somewhat recovers from the regression to AMDGPU test in D62223.
It is indeed not great that we leave vector decrement,
don't transform it into vector add all-ones..
https://rise4fun.com/Alive/ZRl
This is a recommit, originally committed in rL361852, but reverted
to investigate test-suite compile-time hangs, and then reverted in
rL362109 to fix missing constant folds that were causing
endless combine loops.
Reviewers: RKSimon, craig.topper, spatel, arsenm
Reviewed By: RKSimon, arsenm
Subscribers: kzhuravl, jvesely, wdng, nhaehnle, yaxunl, javed.absar, dstuttard, tpr, t-tye, kristof.beyls, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D62263
llvm-svn: 362144
2019-05-31 04:37:29 +08:00
|
|
|
; VARIANT0-NEXT: s_waitcnt vmcnt(0) expcnt(0)
|
2019-05-22 05:49:14 +08:00
|
|
|
; VARIANT0-NEXT: s_barrier
|
[AMDGPU] Mark s_barrier as having side effects but not accessing memory.
Summary:
This fixes poor scheduling in a function containing a barrier and a few
load instructions.
Without this fix, ScheduleDAGInstrs::buildSchedGraph adds an artificial
edge in the dependency graph from the barrier instruction to the exit
node representing live-out latency, with a latency of about 500 cycles.
Because of this it thinks the critical path through the graph also has
a latency of about 500 cycles. And because of that it does not think
that any of the load instructions are on the critical path, so it
schedules them with no regard for their (80 cycle) latency, which gives
poor results.
Reviewers: arsenm, dstuttard, tpr, nhaehnle
Subscribers: kzhuravl, jvesely, wdng, yaxunl, t-tye, hiraditya, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D67218
llvm-svn: 371192
2019-09-06 18:07:28 +08:00
|
|
|
; VARIANT0-NEXT: v_add_i32_e32 v3, vcc, s2, v3
|
2019-05-22 05:49:14 +08:00
|
|
|
; VARIANT0-NEXT: v_ashrrev_i32_e32 v4, 31, v3
|
|
|
|
|
; VARIANT0-NEXT: v_lshl_b64 v[3:4], v[3:4], 2
|
|
|
|
|
; VARIANT0-NEXT: buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
|
|
|
|
|
; VARIANT0-NEXT: s_waitcnt vmcnt(0)
|
|
|
|
|
; VARIANT0-NEXT: buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
|
|
|
|
|
; VARIANT0-NEXT: s_endpgm
|
|
|
|
|
;
|
|
|
|
|
; VARIANT1-LABEL: test_barrier:
|
|
|
|
|
; VARIANT1: ; %bb.0: ; %entry
|
|
|
|
|
; VARIANT1-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9
|
|
|
|
|
; VARIANT1-NEXT: s_load_dword s2, s[0:1], 0xb
|
|
|
|
|
; VARIANT1-NEXT: s_mov_b32 s7, 0xf000
|
|
|
|
|
; VARIANT1-NEXT: s_mov_b32 s6, 0
|
|
|
|
|
; VARIANT1-NEXT: v_lshlrev_b32_e32 v1, 2, v0
|
|
|
|
|
; VARIANT1-NEXT: v_mov_b32_e32 v2, 0
|
[AMDGPU] Remove dubious logic in bidirectional list scheduler
Summary:
pickNodeBidirectional tried to compare the best top candidate and the
best bottom candidate by examining TopCand.Reason and BotCand.Reason.
This is unsound because, after calling pickNodeFromQueue, Cand.Reason
does not reflect the most important reason why Cand was chosen. Rather
it reflects the most recent reason why it beat some other potential
candidate, which could have been for some low priority tie breaker
reason.
I have seen this cause problems where TopCand is a good candidate, but
because TopCand.Reason is ORDER (which is very low priority) it is
repeatedly ignored in favour of a mediocre BotCand. This is not how
bidirectional scheduling is supposed to work.
To fix this I changed the code to always compare TopCand and BotCand
directly, like the generic implementation of pickNodeBidirectional does.
This removes some uncommented AMDGPU-specific logic; if this logic turns
out to be important then perhaps it could be moved into an override of
tryCandidate instead.
Graphics shader benchmarking on gfx10 shows a lot more positive than
negative effects from this change.
Reviewers: arsenm, tstellar, rampitec, kzhuravl, vpykhtin, dstuttard, tpr, atrick, MatzeB
Subscribers: jvesely, wdng, nhaehnle, yaxunl, t-tye, hiraditya, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D68338
2019-10-07 22:33:59 +08:00
|
|
|
; VARIANT1-NEXT: v_not_b32_e32 v3, v0
|
2019-05-22 05:49:14 +08:00
|
|
|
; VARIANT1-NEXT: s_waitcnt lgkmcnt(0)
|
|
|
|
|
; VARIANT1-NEXT: buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
|
|
|
|
|
; VARIANT1-NEXT: s_barrier
|
[AMDGPU] Mark s_barrier as having side effects but not accessing memory.
Summary:
This fixes poor scheduling in a function containing a barrier and a few
load instructions.
Without this fix, ScheduleDAGInstrs::buildSchedGraph adds an artificial
edge in the dependency graph from the barrier instruction to the exit
node representing live-out latency, with a latency of about 500 cycles.
Because of this it thinks the critical path through the graph also has
a latency of about 500 cycles. And because of that it does not think
that any of the load instructions are on the critical path, so it
schedules them with no regard for their (80 cycle) latency, which gives
poor results.
Reviewers: arsenm, dstuttard, tpr, nhaehnle
Subscribers: kzhuravl, jvesely, wdng, yaxunl, t-tye, hiraditya, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D67218
llvm-svn: 371192
2019-09-06 18:07:28 +08:00
|
|
|
; VARIANT1-NEXT: v_add_i32_e32 v3, vcc, s2, v3
|
2019-05-22 05:49:14 +08:00
|
|
|
; VARIANT1-NEXT: v_ashrrev_i32_e32 v4, 31, v3
|
|
|
|
|
; VARIANT1-NEXT: v_lshl_b64 v[3:4], v[3:4], 2
|
[DAGCombine][X86][AArch64][AMDGPU] (x - y) + -1 -> add (xor y, -1), x fold. Try 3
Summary:
This prevents regressions in next patch,
and somewhat recovers from the regression to AMDGPU test in D62223.
It is indeed not great that we leave vector decrement,
don't transform it into vector add all-ones..
https://rise4fun.com/Alive/ZRl
This is a recommit, originally committed in rL361852, but reverted
to investigate test-suite compile-time hangs, and then reverted in
rL362109 to fix missing constant folds that were causing
endless combine loops.
Reviewers: RKSimon, craig.topper, spatel, arsenm
Reviewed By: RKSimon, arsenm
Subscribers: kzhuravl, jvesely, wdng, nhaehnle, yaxunl, javed.absar, dstuttard, tpr, t-tye, kristof.beyls, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D62263
llvm-svn: 362144
2019-05-31 04:37:29 +08:00
|
|
|
; VARIANT1-NEXT: s_waitcnt expcnt(0)
|
2019-05-22 05:49:14 +08:00
|
|
|
; VARIANT1-NEXT: buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
|
|
|
|
|
; VARIANT1-NEXT: s_waitcnt vmcnt(0)
|
|
|
|
|
; VARIANT1-NEXT: buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
|
|
|
|
|
; VARIANT1-NEXT: s_endpgm
|
|
|
|
|
;
|
|
|
|
|
; VARIANT2-LABEL: test_barrier:
|
|
|
|
|
; VARIANT2: ; %bb.0: ; %entry
|
|
|
|
|
; VARIANT2-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
|
|
|
|
|
; VARIANT2-NEXT: s_load_dword s0, s[0:1], 0x2c
|
2020-08-14 04:51:07 +08:00
|
|
|
; VARIANT2-NEXT: v_lshlrev_b32_e32 v2, 2, v0
|
2019-05-22 05:49:14 +08:00
|
|
|
; VARIANT2-NEXT: s_waitcnt lgkmcnt(0)
|
2020-08-14 04:51:07 +08:00
|
|
|
; VARIANT2-NEXT: global_store_dword v2, v0, s[2:3]
|
|
|
|
|
; VARIANT2-NEXT: v_xad_u32 v0, v0, -1, s0
|
|
|
|
|
; VARIANT2-NEXT: v_ashrrev_i32_e32 v1, 31, v0
|
|
|
|
|
; VARIANT2-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
|
|
|
|
|
; VARIANT2-NEXT: v_mov_b32_e32 v3, s3
|
|
|
|
|
; VARIANT2-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
|
|
|
|
|
; VARIANT2-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
|
2019-05-22 05:49:14 +08:00
|
|
|
; VARIANT2-NEXT: s_waitcnt vmcnt(0)
|
|
|
|
|
; VARIANT2-NEXT: s_barrier
|
2020-08-14 04:51:07 +08:00
|
|
|
; VARIANT2-NEXT: global_load_dword v0, v[0:1], off
|
2019-05-22 05:49:14 +08:00
|
|
|
; VARIANT2-NEXT: s_waitcnt vmcnt(0)
|
2020-08-14 04:51:07 +08:00
|
|
|
; VARIANT2-NEXT: global_store_dword v2, v0, s[2:3]
|
2019-05-22 05:49:14 +08:00
|
|
|
; VARIANT2-NEXT: s_endpgm
|
|
|
|
|
;
|
|
|
|
|
; VARIANT3-LABEL: test_barrier:
|
|
|
|
|
; VARIANT3: ; %bb.0: ; %entry
|
|
|
|
|
; VARIANT3-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
|
|
|
|
|
; VARIANT3-NEXT: s_load_dword s0, s[0:1], 0x2c
|
2020-08-14 04:51:07 +08:00
|
|
|
; VARIANT3-NEXT: v_lshlrev_b32_e32 v2, 2, v0
|
2019-05-22 05:49:14 +08:00
|
|
|
; VARIANT3-NEXT: s_waitcnt lgkmcnt(0)
|
2020-08-14 04:51:07 +08:00
|
|
|
; VARIANT3-NEXT: global_store_dword v2, v0, s[2:3]
|
|
|
|
|
; VARIANT3-NEXT: v_xad_u32 v0, v0, -1, s0
|
|
|
|
|
; VARIANT3-NEXT: v_ashrrev_i32_e32 v1, 31, v0
|
|
|
|
|
; VARIANT3-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
|
|
|
|
|
; VARIANT3-NEXT: v_mov_b32_e32 v3, s3
|
|
|
|
|
; VARIANT3-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
|
|
|
|
|
; VARIANT3-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
|
[AMDGPU] Mark s_barrier as having side effects but not accessing memory.
Summary:
This fixes poor scheduling in a function containing a barrier and a few
load instructions.
Without this fix, ScheduleDAGInstrs::buildSchedGraph adds an artificial
edge in the dependency graph from the barrier instruction to the exit
node representing live-out latency, with a latency of about 500 cycles.
Because of this it thinks the critical path through the graph also has
a latency of about 500 cycles. And because of that it does not think
that any of the load instructions are on the critical path, so it
schedules them with no regard for their (80 cycle) latency, which gives
poor results.
Reviewers: arsenm, dstuttard, tpr, nhaehnle
Subscribers: kzhuravl, jvesely, wdng, yaxunl, t-tye, hiraditya, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D67218
llvm-svn: 371192
2019-09-06 18:07:28 +08:00
|
|
|
; VARIANT3-NEXT: s_barrier
|
2020-08-14 04:51:07 +08:00
|
|
|
; VARIANT3-NEXT: global_load_dword v0, v[0:1], off
|
2019-05-22 05:49:14 +08:00
|
|
|
; VARIANT3-NEXT: s_waitcnt vmcnt(0)
|
2020-08-14 04:51:07 +08:00
|
|
|
; VARIANT3-NEXT: global_store_dword v2, v0, s[2:3]
|
2019-05-22 05:49:14 +08:00
|
|
|
; VARIANT3-NEXT: s_endpgm
|
2016-01-23 05:30:43 +08:00
|
|
|
entry:
|
2016-02-11 14:02:01 +08:00
|
|
|
%tmp = call i32 @llvm.amdgcn.workitem.id.x()
|
2016-01-23 05:30:43 +08:00
|
|
|
%tmp1 = getelementptr i32, i32 addrspace(1)* %out, i32 %tmp
|
|
|
|
|
store i32 %tmp, i32 addrspace(1)* %tmp1
|
|
|
|
|
call void @llvm.amdgcn.s.barrier()
|
2017-05-18 05:38:21 +08:00
|
|
|
%tmp3 = sub i32 %size, 1
|
2016-01-23 05:30:43 +08:00
|
|
|
%tmp4 = sub i32 %tmp3, %tmp
|
|
|
|
|
%tmp5 = getelementptr i32, i32 addrspace(1)* %out, i32 %tmp4
|
|
|
|
|
%tmp6 = load i32, i32 addrspace(1)* %tmp5
|
|
|
|
|
store i32 %tmp6, i32 addrspace(1)* %tmp1
|
|
|
|
|
ret void
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
declare void @llvm.amdgcn.s.barrier() #1
|
2016-02-11 14:02:01 +08:00
|
|
|
declare i32 @llvm.amdgcn.workitem.id.x() #2
|
2016-01-23 05:30:43 +08:00
|
|
|
|
|
|
|
|
attributes #0 = { nounwind }
|
|
|
|
|
attributes #1 = { convergent nounwind }
|
|
|
|
|
attributes #2 = { nounwind readnone }
|