2017-03-25 03:52:05 +08:00
|
|
|
; RUN: opt -mtriple=amdgcn-- -S -amdgpu-unify-divergent-exit-nodes -verify -structurizecfg -verify -si-annotate-control-flow %s | FileCheck -check-prefix=IR %s
|
|
|
|
; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
|
|
|
|
|
|
|
|
; Add an extra verifier runs. There were some cases where invalid IR
|
|
|
|
; was produced but happened to be fixed by the later passes.
|
|
|
|
|
|
|
|
; Make sure divergent control flow with multiple exits from a region
|
|
|
|
; is properly handled. UnifyFunctionExitNodes should be run before
|
|
|
|
; StructurizeCFG.
|
|
|
|
|
|
|
|
; IR-LABEL: @multi_divergent_region_exit_ret_ret(
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: %1 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %0)
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %2 = extractvalue { i1, i64 } %1, 0
|
|
|
|
; IR: %3 = extractvalue { i1, i64 } %1, 1
|
|
|
|
; IR: br i1 %2, label %LeafBlock1, label %Flow
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: Flow:
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %4 = phi i1 [ true, %LeafBlock1 ], [ false, %entry ]
|
|
|
|
; IR: %5 = phi i1 [ %10, %LeafBlock1 ], [ false, %entry ]
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: %6 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %3)
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %7 = extractvalue { i1, i64 } %6, 0
|
|
|
|
; IR: %8 = extractvalue { i1, i64 } %6, 1
|
|
|
|
; IR: br i1 %7, label %LeafBlock, label %Flow1
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: LeafBlock:
|
|
|
|
; IR: br label %Flow1
|
|
|
|
|
|
|
|
; IR: LeafBlock1:
|
|
|
|
; IR: br label %Flow{{$}}
|
|
|
|
|
|
|
|
; IR: Flow2:
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %11 = phi i1 [ false, %exit1 ], [ %15, %Flow1 ]
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: call void @llvm.amdgcn.end.cf.i64(i64 %19)
|
|
|
|
; IR: %12 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %11)
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %13 = extractvalue { i1, i64 } %12, 0
|
|
|
|
; IR: %14 = extractvalue { i1, i64 } %12, 1
|
|
|
|
; IR: br i1 %13, label %exit0, label %UnifiedReturnBlock
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: exit0:
|
|
|
|
; IR: store volatile i32 9, i32 addrspace(1)* undef
|
|
|
|
; IR: br label %UnifiedReturnBlock
|
|
|
|
|
|
|
|
; IR: Flow1:
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %15 = phi i1 [ %SwitchLeaf, %LeafBlock ], [ %4, %Flow ]
|
|
|
|
; IR: %16 = phi i1 [ %9, %LeafBlock ], [ %5, %Flow ]
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: call void @llvm.amdgcn.end.cf.i64(i64 %8)
|
|
|
|
; IR: %17 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %16)
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %18 = extractvalue { i1, i64 } %17, 0
|
|
|
|
; IR: %19 = extractvalue { i1, i64 } %17, 1
|
|
|
|
; IR: br i1 %18, label %exit1, label %Flow2
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: exit1:
|
|
|
|
; IR: store volatile i32 17, i32 addrspace(3)* undef
|
|
|
|
; IR: br label %Flow2
|
|
|
|
|
|
|
|
; IR: UnifiedReturnBlock:
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: call void @llvm.amdgcn.end.cf.i64(i64 %14)
|
2017-03-25 03:52:05 +08:00
|
|
|
; IR: ret void
|
|
|
|
|
|
|
|
|
|
|
|
; GCN-LABEL: {{^}}multi_divergent_region_exit_ret_ret:
|
|
|
|
|
Bias physical register immediate assignments
The machine scheduler currently biases register copies to/from
physical registers to be closer to their point of use / def to
minimize their live ranges. This change extends this to also physical
register assignments from immediate values.
This causes a reduction in reduction in overall register pressure and
minor reduction in spills and indirectly fixes an out-of-registers
assertion (PR39391).
Most test changes are from minor instruction reorderings and register
name selection changes and direct consequences of that.
Reviewers: MatzeB, qcolombet, myatsina, pcc
Subscribers: nemanjai, jvesely, nhaehnle, eraman, hiraditya,
javed.absar, arphaman, jfb, jsji, llvm-commits
Differential Revision: https://reviews.llvm.org/D54218
llvm-svn: 346894
2018-11-15 05:11:53 +08:00
|
|
|
; GCN-DAG: s_mov_b64 [[EXIT1:s\[[0-9]+:[0-9]+\]]], 0
|
|
|
|
; GCN-DAG: v_cmp_lt_i32_e32 vcc, 1,
|
|
|
|
; GCN-DAG: s_mov_b64 [[EXIT0:s\[[0-9]+:[0-9]+\]]], 0
|
|
|
|
; GCN-DAG: s_and_saveexec_b64
|
|
|
|
; GCN-DAG: s_xor_b64
|
AMDGPU: Rewrite SILowerI1Copies to always stay on SALU
Summary:
Instead of writing boolean values temporarily into 32-bit VGPRs
if they are involved in PHIs or are observed from outside a loop,
we use bitwise masking operations to combine lane masks in a way
that is consistent with wave control flow.
Move SIFixSGPRCopies to before this pass, since that pass
incorrectly attempts to move SGPR phis to VGPRs.
This should recover most of the code quality that was lost with
the bug fix in "AMDGPU: Remove PHI loop condition optimization".
There are still some relevant cases where code quality could be
improved, in particular:
- We often introduce redundant masks with EXEC. Ideally, we'd
have a generic computeKnownBits-like analysis to determine
whether masks are already masked by EXEC, so we can avoid this
masking both here and when lowering uniform control flow.
- The criterion we use to determine whether a def is observed
from outside a loop is conservative: it doesn't check whether
(loop) branch conditions are uniform.
Change-Id: Ibabdb373a7510e426b90deef00f5e16c5d56e64b
Reviewers: arsenm, rampitec, tpr
Subscribers: kzhuravl, jvesely, wdng, mgorny, yaxunl, dstuttard, t-tye, eraman, llvm-commits
Differential Revision: https://reviews.llvm.org/D53496
llvm-svn: 345719
2018-10-31 21:27:08 +08:00
|
|
|
|
|
|
|
; GCN: ; %LeafBlock1
|
|
|
|
; GCN-NEXT: s_mov_b64 [[EXIT0]], exec
|
|
|
|
; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 2,
|
|
|
|
; GCN-NEXT: s_and_b64 [[EXIT1]], vcc, exec
|
|
|
|
|
|
|
|
; GCN: ; %Flow
|
|
|
|
; GCN-NEXT: s_or_saveexec_b64
|
|
|
|
; GCN-NEXT: s_xor_b64
|
2017-03-25 03:52:05 +08:00
|
|
|
|
2017-04-25 04:25:01 +08:00
|
|
|
; FIXME: Why is this compare essentially repeated?
|
AMDGPU: Rewrite SILowerI1Copies to always stay on SALU
Summary:
Instead of writing boolean values temporarily into 32-bit VGPRs
if they are involved in PHIs or are observed from outside a loop,
we use bitwise masking operations to combine lane masks in a way
that is consistent with wave control flow.
Move SIFixSGPRCopies to before this pass, since that pass
incorrectly attempts to move SGPR phis to VGPRs.
This should recover most of the code quality that was lost with
the bug fix in "AMDGPU: Remove PHI loop condition optimization".
There are still some relevant cases where code quality could be
improved, in particular:
- We often introduce redundant masks with EXEC. Ideally, we'd
have a generic computeKnownBits-like analysis to determine
whether masks are already masked by EXEC, so we can avoid this
masking both here and when lowering uniform control flow.
- The criterion we use to determine whether a def is observed
from outside a loop is conservative: it doesn't check whether
(loop) branch conditions are uniform.
Change-Id: Ibabdb373a7510e426b90deef00f5e16c5d56e64b
Reviewers: arsenm, rampitec, tpr
Subscribers: kzhuravl, jvesely, wdng, mgorny, yaxunl, dstuttard, t-tye, eraman, llvm-commits
Differential Revision: https://reviews.llvm.org/D53496
llvm-svn: 345719
2018-10-31 21:27:08 +08:00
|
|
|
; GCN: ; %LeafBlock
|
|
|
|
; GCN-DAG: v_cmp_eq_u32_e32 vcc, 1,
|
|
|
|
; GCN-DAG: v_cmp_ne_u32_e64 [[TMP1:s\[[0-9]+:[0-9]+\]]], 1,
|
|
|
|
; GCN-DAG: s_andn2_b64 [[EXIT0]], [[EXIT0]], exec
|
|
|
|
; GCN-DAG: s_andn2_b64 [[EXIT1]], [[EXIT1]], exec
|
|
|
|
; GCN-DAG: s_and_b64 [[TMP0:s\[[0-9]+:[0-9]+\]]], vcc, exec
|
|
|
|
; GCN-DAG: s_and_b64 [[TMP1]], [[TMP1]], exec
|
|
|
|
; GCN-DAG: s_or_b64 [[EXIT0]], [[EXIT0]], [[TMP0]]
|
|
|
|
; GCN-DAG: s_or_b64 [[EXIT1]], [[EXIT1]], [[TMP1]]
|
2017-03-25 03:52:05 +08:00
|
|
|
|
2018-06-30 01:31:42 +08:00
|
|
|
; GCN: ; %Flow4
|
AMDGPU: Rewrite SILowerI1Copies to always stay on SALU
Summary:
Instead of writing boolean values temporarily into 32-bit VGPRs
if they are involved in PHIs or are observed from outside a loop,
we use bitwise masking operations to combine lane masks in a way
that is consistent with wave control flow.
Move SIFixSGPRCopies to before this pass, since that pass
incorrectly attempts to move SGPR phis to VGPRs.
This should recover most of the code quality that was lost with
the bug fix in "AMDGPU: Remove PHI loop condition optimization".
There are still some relevant cases where code quality could be
improved, in particular:
- We often introduce redundant masks with EXEC. Ideally, we'd
have a generic computeKnownBits-like analysis to determine
whether masks are already masked by EXEC, so we can avoid this
masking both here and when lowering uniform control flow.
- The criterion we use to determine whether a def is observed
from outside a loop is conservative: it doesn't check whether
(loop) branch conditions are uniform.
Change-Id: Ibabdb373a7510e426b90deef00f5e16c5d56e64b
Reviewers: arsenm, rampitec, tpr
Subscribers: kzhuravl, jvesely, wdng, mgorny, yaxunl, dstuttard, t-tye, eraman, llvm-commits
Differential Revision: https://reviews.llvm.org/D53496
llvm-svn: 345719
2018-10-31 21:27:08 +08:00
|
|
|
; GCN-NEXT: s_or_b64 exec, exec,
|
|
|
|
; GCN-NEXT: s_and_saveexec_b64 {{s\[[0-9]+:[0-9]+\]}}, [[EXIT1]]
|
|
|
|
; GCN-NEXT: s_xor_b64
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; GCN: ; %exit1
|
Bias physical register immediate assignments
The machine scheduler currently biases register copies to/from
physical registers to be closer to their point of use / def to
minimize their live ranges. This change extends this to also physical
register assignments from immediate values.
This causes a reduction in reduction in overall register pressure and
minor reduction in spills and indirectly fixes an out-of-registers
assertion (PR39391).
Most test changes are from minor instruction reorderings and register
name selection changes and direct consequences of that.
Reviewers: MatzeB, qcolombet, myatsina, pcc
Subscribers: nemanjai, jvesely, nhaehnle, eraman, hiraditya,
javed.absar, arphaman, jfb, jsji, llvm-commits
Differential Revision: https://reviews.llvm.org/D54218
llvm-svn: 346894
2018-11-15 05:11:53 +08:00
|
|
|
; GCN-DAG: ds_write_b32
|
|
|
|
; GCN-DAG: s_andn2_b64 [[EXIT0]], [[EXIT0]], exec
|
2017-03-25 03:52:05 +08:00
|
|
|
|
AMDGPU: Rewrite SILowerI1Copies to always stay on SALU
Summary:
Instead of writing boolean values temporarily into 32-bit VGPRs
if they are involved in PHIs or are observed from outside a loop,
we use bitwise masking operations to combine lane masks in a way
that is consistent with wave control flow.
Move SIFixSGPRCopies to before this pass, since that pass
incorrectly attempts to move SGPR phis to VGPRs.
This should recover most of the code quality that was lost with
the bug fix in "AMDGPU: Remove PHI loop condition optimization".
There are still some relevant cases where code quality could be
improved, in particular:
- We often introduce redundant masks with EXEC. Ideally, we'd
have a generic computeKnownBits-like analysis to determine
whether masks are already masked by EXEC, so we can avoid this
masking both here and when lowering uniform control flow.
- The criterion we use to determine whether a def is observed
from outside a loop is conservative: it doesn't check whether
(loop) branch conditions are uniform.
Change-Id: Ibabdb373a7510e426b90deef00f5e16c5d56e64b
Reviewers: arsenm, rampitec, tpr
Subscribers: kzhuravl, jvesely, wdng, mgorny, yaxunl, dstuttard, t-tye, eraman, llvm-commits
Differential Revision: https://reviews.llvm.org/D53496
llvm-svn: 345719
2018-10-31 21:27:08 +08:00
|
|
|
; GCN: ; %Flow5
|
|
|
|
; GCN-NEXT: s_or_b64 exec, exec,
|
|
|
|
; GCN-NEXT; s_and_saveexec_b64 {{s\[[0-9]+:[0-9]+\]}}, [[EXIT0]]
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; GCN: ; %exit0
|
AMDGPU: Rewrite SILowerI1Copies to always stay on SALU
Summary:
Instead of writing boolean values temporarily into 32-bit VGPRs
if they are involved in PHIs or are observed from outside a loop,
we use bitwise masking operations to combine lane masks in a way
that is consistent with wave control flow.
Move SIFixSGPRCopies to before this pass, since that pass
incorrectly attempts to move SGPR phis to VGPRs.
This should recover most of the code quality that was lost with
the bug fix in "AMDGPU: Remove PHI loop condition optimization".
There are still some relevant cases where code quality could be
improved, in particular:
- We often introduce redundant masks with EXEC. Ideally, we'd
have a generic computeKnownBits-like analysis to determine
whether masks are already masked by EXEC, so we can avoid this
masking both here and when lowering uniform control flow.
- The criterion we use to determine whether a def is observed
from outside a loop is conservative: it doesn't check whether
(loop) branch conditions are uniform.
Change-Id: Ibabdb373a7510e426b90deef00f5e16c5d56e64b
Reviewers: arsenm, rampitec, tpr
Subscribers: kzhuravl, jvesely, wdng, mgorny, yaxunl, dstuttard, t-tye, eraman, llvm-commits
Differential Revision: https://reviews.llvm.org/D53496
llvm-svn: 345719
2018-10-31 21:27:08 +08:00
|
|
|
; GCN: buffer_store_dword
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; GCN: ; %UnifiedReturnBlock
|
|
|
|
; GCN-NEXT: s_endpgm
|
|
|
|
define amdgpu_kernel void @multi_divergent_region_exit_ret_ret(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2) #0 {
|
|
|
|
entry:
|
|
|
|
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1
|
|
|
|
%tmp1 = add i32 0, %tmp
|
|
|
|
%tmp2 = zext i32 %tmp1 to i64
|
|
|
|
%tmp3 = add i64 0, %tmp2
|
|
|
|
%tmp4 = shl i64 %tmp3, 32
|
|
|
|
%tmp5 = ashr exact i64 %tmp4, 32
|
|
|
|
%tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %tmp5
|
|
|
|
%tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4
|
|
|
|
%tmp8 = sext i32 %tmp7 to i64
|
|
|
|
%tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp8
|
|
|
|
%tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4
|
|
|
|
%tmp13 = zext i32 %tmp10 to i64
|
|
|
|
%tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg2, i64 %tmp13
|
|
|
|
%tmp16 = load i32, i32 addrspace(1)* %tmp14, align 16
|
|
|
|
%Pivot = icmp slt i32 %tmp16, 2
|
|
|
|
br i1 %Pivot, label %LeafBlock, label %LeafBlock1
|
|
|
|
|
|
|
|
LeafBlock: ; preds = %entry
|
|
|
|
%SwitchLeaf = icmp eq i32 %tmp16, 1
|
|
|
|
br i1 %SwitchLeaf, label %exit0, label %exit1
|
|
|
|
|
|
|
|
LeafBlock1: ; preds = %entry
|
|
|
|
%SwitchLeaf2 = icmp eq i32 %tmp16, 2
|
|
|
|
br i1 %SwitchLeaf2, label %exit0, label %exit1
|
|
|
|
|
|
|
|
exit0: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store volatile i32 9, i32 addrspace(1)* undef
|
|
|
|
ret void
|
|
|
|
|
|
|
|
exit1: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store volatile i32 17, i32 addrspace(3)* undef
|
|
|
|
ret void
|
|
|
|
}
|
|
|
|
|
|
|
|
; IR-LABEL: @multi_divergent_region_exit_unreachable_unreachable(
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: %1 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %0)
|
2017-03-25 03:52:05 +08:00
|
|
|
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: %6 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %3)
|
2017-03-25 03:52:05 +08:00
|
|
|
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %11 = phi i1 [ false, %exit1 ], [ %15, %Flow1 ]
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: call void @llvm.amdgcn.end.cf.i64(i64 %19)
|
|
|
|
; IR: %12 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %11)
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: br i1 %13, label %exit0, label %UnifiedUnreachableBlock
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
|
|
|
|
; IR: UnifiedUnreachableBlock:
|
|
|
|
; IR-NEXT: unreachable
|
|
|
|
|
|
|
|
|
|
|
|
; FIXME: Probably should insert an s_endpgm anyway.
|
|
|
|
; GCN-LABEL: {{^}}multi_divergent_region_exit_unreachable_unreachable:
|
|
|
|
; GCN: ; %UnifiedUnreachableBlock
|
|
|
|
; GCN-NEXT: .Lfunc_end
|
|
|
|
define amdgpu_kernel void @multi_divergent_region_exit_unreachable_unreachable(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2) #0 {
|
|
|
|
entry:
|
|
|
|
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1
|
|
|
|
%tmp1 = add i32 0, %tmp
|
|
|
|
%tmp2 = zext i32 %tmp1 to i64
|
|
|
|
%tmp3 = add i64 0, %tmp2
|
|
|
|
%tmp4 = shl i64 %tmp3, 32
|
|
|
|
%tmp5 = ashr exact i64 %tmp4, 32
|
|
|
|
%tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %tmp5
|
|
|
|
%tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4
|
|
|
|
%tmp8 = sext i32 %tmp7 to i64
|
|
|
|
%tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp8
|
|
|
|
%tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4
|
|
|
|
%tmp13 = zext i32 %tmp10 to i64
|
|
|
|
%tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg2, i64 %tmp13
|
|
|
|
%tmp16 = load i32, i32 addrspace(1)* %tmp14, align 16
|
|
|
|
%Pivot = icmp slt i32 %tmp16, 2
|
|
|
|
br i1 %Pivot, label %LeafBlock, label %LeafBlock1
|
|
|
|
|
|
|
|
LeafBlock: ; preds = %entry
|
|
|
|
%SwitchLeaf = icmp eq i32 %tmp16, 1
|
|
|
|
br i1 %SwitchLeaf, label %exit0, label %exit1
|
|
|
|
|
|
|
|
LeafBlock1: ; preds = %entry
|
|
|
|
%SwitchLeaf2 = icmp eq i32 %tmp16, 2
|
|
|
|
br i1 %SwitchLeaf2, label %exit0, label %exit1
|
|
|
|
|
|
|
|
exit0: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store volatile i32 9, i32 addrspace(1)* undef
|
|
|
|
unreachable
|
|
|
|
|
|
|
|
exit1: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store volatile i32 17, i32 addrspace(3)* undef
|
|
|
|
unreachable
|
|
|
|
}
|
|
|
|
|
|
|
|
; IR-LABEL: @multi_exit_region_divergent_ret_uniform_ret(
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %divergent.cond0 = icmp slt i32 %tmp16, 2
|
2017-03-25 03:52:05 +08:00
|
|
|
; IR: llvm.amdgcn.if
|
|
|
|
; IR: br i1
|
|
|
|
|
|
|
|
; IR: {{^}}Flow:
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %4 = phi i1 [ true, %LeafBlock1 ], [ false, %entry ]
|
|
|
|
; IR: %5 = phi i1 [ %10, %LeafBlock1 ], [ false, %entry ]
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: %6 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %3)
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: br i1 %7, label %LeafBlock, label %Flow1
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: {{^}}LeafBlock:
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %divergent.cond1 = icmp eq i32 %tmp16, 1
|
|
|
|
; IR: %9 = xor i1 %divergent.cond1, true
|
2017-03-25 03:52:05 +08:00
|
|
|
; IR: br label %Flow1
|
|
|
|
|
|
|
|
; IR: LeafBlock1:
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %uniform.cond0 = icmp eq i32 %arg3, 2
|
|
|
|
; IR: %10 = xor i1 %uniform.cond0, true
|
2017-03-25 03:52:05 +08:00
|
|
|
; IR: br label %Flow
|
|
|
|
|
|
|
|
; IR: Flow2:
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %11 = phi i1 [ false, %exit1 ], [ %15, %Flow1 ]
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: call void @llvm.amdgcn.end.cf.i64(i64 %19)
|
|
|
|
; IR: %12 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %11)
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: br i1 %13, label %exit0, label %UnifiedReturnBlock
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: exit0:
|
|
|
|
; IR: store volatile i32 9, i32 addrspace(1)* undef
|
|
|
|
; IR: br label %UnifiedReturnBlock
|
|
|
|
|
|
|
|
; IR: {{^}}Flow1:
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %15 = phi i1 [ %divergent.cond1, %LeafBlock ], [ %4, %Flow ]
|
|
|
|
; IR: %16 = phi i1 [ %9, %LeafBlock ], [ %5, %Flow ]
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: call void @llvm.amdgcn.end.cf.i64(i64 %8)
|
|
|
|
; IR: %17 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %16)
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %18 = extractvalue { i1, i64 } %17, 0
|
|
|
|
; IR: %19 = extractvalue { i1, i64 } %17, 1
|
|
|
|
; IR: br i1 %18, label %exit1, label %Flow2
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: exit1:
|
|
|
|
; IR: store volatile i32 17, i32 addrspace(3)* undef
|
|
|
|
; IR: br label %Flow2
|
|
|
|
|
|
|
|
; IR: UnifiedReturnBlock:
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: call void @llvm.amdgcn.end.cf.i64(i64 %14)
|
2017-03-25 03:52:05 +08:00
|
|
|
; IR: ret void
|
|
|
|
define amdgpu_kernel void @multi_exit_region_divergent_ret_uniform_ret(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2, i32 %arg3) #0 {
|
|
|
|
entry:
|
|
|
|
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1
|
|
|
|
%tmp1 = add i32 0, %tmp
|
|
|
|
%tmp2 = zext i32 %tmp1 to i64
|
|
|
|
%tmp3 = add i64 0, %tmp2
|
|
|
|
%tmp4 = shl i64 %tmp3, 32
|
|
|
|
%tmp5 = ashr exact i64 %tmp4, 32
|
|
|
|
%tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %tmp5
|
|
|
|
%tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4
|
|
|
|
%tmp8 = sext i32 %tmp7 to i64
|
|
|
|
%tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp8
|
|
|
|
%tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4
|
|
|
|
%tmp13 = zext i32 %tmp10 to i64
|
|
|
|
%tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg2, i64 %tmp13
|
|
|
|
%tmp16 = load i32, i32 addrspace(1)* %tmp14, align 16
|
|
|
|
%divergent.cond0 = icmp slt i32 %tmp16, 2
|
|
|
|
br i1 %divergent.cond0, label %LeafBlock, label %LeafBlock1
|
|
|
|
|
|
|
|
LeafBlock: ; preds = %entry
|
|
|
|
%divergent.cond1 = icmp eq i32 %tmp16, 1
|
|
|
|
br i1 %divergent.cond1, label %exit0, label %exit1
|
|
|
|
|
|
|
|
LeafBlock1: ; preds = %entry
|
|
|
|
%uniform.cond0 = icmp eq i32 %arg3, 2
|
|
|
|
br i1 %uniform.cond0, label %exit0, label %exit1
|
|
|
|
|
|
|
|
exit0: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store volatile i32 9, i32 addrspace(1)* undef
|
|
|
|
ret void
|
|
|
|
|
|
|
|
exit1: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store volatile i32 17, i32 addrspace(3)* undef
|
|
|
|
ret void
|
|
|
|
}
|
|
|
|
|
|
|
|
; IR-LABEL: @multi_exit_region_uniform_ret_divergent_ret(
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: %1 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %0)
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: br i1 %2, label %LeafBlock1, label %Flow
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: Flow:
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %4 = phi i1 [ true, %LeafBlock1 ], [ false, %entry ]
|
|
|
|
; IR: %5 = phi i1 [ %10, %LeafBlock1 ], [ false, %entry ]
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: %6 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %3)
|
2017-03-25 03:52:05 +08:00
|
|
|
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %11 = phi i1 [ false, %exit1 ], [ %15, %Flow1 ]
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: call void @llvm.amdgcn.end.cf.i64(i64 %19)
|
|
|
|
; IR: %12 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %11)
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
define amdgpu_kernel void @multi_exit_region_uniform_ret_divergent_ret(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2, i32 %arg3) #0 {
|
|
|
|
entry:
|
|
|
|
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1
|
|
|
|
%tmp1 = add i32 0, %tmp
|
|
|
|
%tmp2 = zext i32 %tmp1 to i64
|
|
|
|
%tmp3 = add i64 0, %tmp2
|
|
|
|
%tmp4 = shl i64 %tmp3, 32
|
|
|
|
%tmp5 = ashr exact i64 %tmp4, 32
|
|
|
|
%tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %tmp5
|
|
|
|
%tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4
|
|
|
|
%tmp8 = sext i32 %tmp7 to i64
|
|
|
|
%tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp8
|
|
|
|
%tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4
|
|
|
|
%tmp13 = zext i32 %tmp10 to i64
|
|
|
|
%tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg2, i64 %tmp13
|
|
|
|
%tmp16 = load i32, i32 addrspace(1)* %tmp14, align 16
|
|
|
|
%Pivot = icmp slt i32 %tmp16, 2
|
|
|
|
br i1 %Pivot, label %LeafBlock, label %LeafBlock1
|
|
|
|
|
|
|
|
LeafBlock: ; preds = %entry
|
|
|
|
%SwitchLeaf = icmp eq i32 %arg3, 1
|
|
|
|
br i1 %SwitchLeaf, label %exit0, label %exit1
|
|
|
|
|
|
|
|
LeafBlock1: ; preds = %entry
|
|
|
|
%SwitchLeaf2 = icmp eq i32 %tmp16, 2
|
|
|
|
br i1 %SwitchLeaf2, label %exit0, label %exit1
|
|
|
|
|
|
|
|
exit0: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store volatile i32 9, i32 addrspace(1)* undef
|
|
|
|
ret void
|
|
|
|
|
|
|
|
exit1: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store volatile i32 17, i32 addrspace(3)* undef
|
|
|
|
ret void
|
|
|
|
}
|
|
|
|
|
|
|
|
; IR-LABEL: @multi_divergent_region_exit_ret_ret_return_value(
|
|
|
|
; IR: Flow2:
|
2018-10-17 23:37:41 +08:00
|
|
|
; IR: %11 = phi i1 [ false, %exit1 ], [ %15, %Flow1 ]
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: call void @llvm.amdgcn.end.cf.i64(i64 %19)
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: UnifiedReturnBlock:
|
2018-10-17 23:37:41 +08:00
|
|
|
; IR: %UnifiedRetVal = phi float [ 2.000000e+00, %Flow2 ], [ 1.000000e+00, %exit0 ]
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: call void @llvm.amdgcn.end.cf.i64(i64 %14)
|
2017-03-25 03:52:05 +08:00
|
|
|
; IR: ret float %UnifiedRetVal
|
|
|
|
define amdgpu_ps float @multi_divergent_region_exit_ret_ret_return_value(i32 %vgpr) #0 {
|
|
|
|
entry:
|
|
|
|
%Pivot = icmp slt i32 %vgpr, 2
|
|
|
|
br i1 %Pivot, label %LeafBlock, label %LeafBlock1
|
|
|
|
|
|
|
|
LeafBlock: ; preds = %entry
|
|
|
|
%SwitchLeaf = icmp eq i32 %vgpr, 1
|
|
|
|
br i1 %SwitchLeaf, label %exit0, label %exit1
|
|
|
|
|
|
|
|
LeafBlock1: ; preds = %entry
|
|
|
|
%SwitchLeaf2 = icmp eq i32 %vgpr, 2
|
|
|
|
br i1 %SwitchLeaf2, label %exit0, label %exit1
|
|
|
|
|
|
|
|
exit0: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store i32 9, i32 addrspace(1)* undef
|
|
|
|
ret float 1.0
|
|
|
|
|
|
|
|
exit1: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store i32 17, i32 addrspace(3)* undef
|
|
|
|
ret float 2.0
|
|
|
|
}
|
|
|
|
|
|
|
|
; IR-LABEL: @uniform_branch_to_multi_divergent_region_exit_ret_ret_return_value(
|
|
|
|
|
|
|
|
; GCN-LABEL: {{^}}uniform_branch_to_multi_divergent_region_exit_ret_ret_return_value:
|
|
|
|
; GCN: s_cmp_gt_i32 s0, 1
|
|
|
|
; GCN: s_cbranch_scc0 [[FLOW:BB[0-9]+_[0-9]+]]
|
|
|
|
|
|
|
|
; GCN: v_cmp_ne_u32_e32 vcc, 7, v0
|
|
|
|
|
|
|
|
; GCN: {{^}}[[FLOW]]:
|
|
|
|
; GCN: s_cbranch_vccnz [[FLOW1:BB[0-9]+]]
|
|
|
|
|
|
|
|
; GCN: s_or_b64 exec, exec
|
2018-10-17 23:37:41 +08:00
|
|
|
; GCN: v_mov_b32_e32 v0, 2.0
|
[AMDGPU] Don't force WQM for DS op
Summary:
Previously, all DS ops forced WQM in a pixel shader. That was a hack to
allow for graphics frontends using ds_swizzle to implement explicit
derivatives, on SI/CI at least where DPP is not available. But it forced
WQM for _any_ DS op.
With this commit, DS ops no longer force WQM. Both graphics frontends
(Mesa and LLPC) need to change to issue an explicit llvm.amdgcn.wqm
intrinsic call when calculating explicit derivatives.
The required Mesa change is: "amd/common: use llvm.amdgcn.wqm for
explicit derivatives".
Subscribers: qcolombet, arsenm, kzhuravl, wdng, nhaehnle, yaxunl, dstuttard, t-tye, llvm-commits
Differential Revision: https://reviews.llvm.org/D46051
Change-Id: I9b745b626fa91bbd66456e6cf41ee07eeea42f81
llvm-svn: 331633
2018-05-07 21:21:26 +08:00
|
|
|
; GCN-NOT: s_and_b64 exec, exec
|
2017-03-25 03:52:05 +08:00
|
|
|
; GCN: v_mov_b32_e32 v0, 1.0
|
|
|
|
|
|
|
|
; GCN: {{^BB[0-9]+_[0-9]+}}: ; %UnifiedReturnBlock
|
|
|
|
; GCN-NEXT: s_or_b64 exec, exec
|
2017-06-02 22:19:25 +08:00
|
|
|
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
2017-03-25 03:52:05 +08:00
|
|
|
; GCN-NEXT: ; return
|
|
|
|
|
|
|
|
define amdgpu_ps float @uniform_branch_to_multi_divergent_region_exit_ret_ret_return_value(i32 inreg %sgpr, i32 %vgpr) #0 {
|
|
|
|
entry:
|
|
|
|
%uniform.cond = icmp slt i32 %sgpr, 2
|
|
|
|
br i1 %uniform.cond, label %LeafBlock, label %LeafBlock1
|
|
|
|
|
|
|
|
LeafBlock: ; preds = %entry
|
|
|
|
%divergent.cond0 = icmp eq i32 %vgpr, 3
|
|
|
|
br i1 %divergent.cond0, label %exit0, label %exit1
|
|
|
|
|
|
|
|
LeafBlock1: ; preds = %entry
|
|
|
|
%divergent.cond1 = icmp eq i32 %vgpr, 7
|
|
|
|
br i1 %divergent.cond1, label %exit0, label %exit1
|
|
|
|
|
|
|
|
exit0: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store i32 9, i32 addrspace(1)* undef
|
|
|
|
ret float 1.0
|
|
|
|
|
|
|
|
exit1: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store i32 17, i32 addrspace(3)* undef
|
|
|
|
ret float 2.0
|
|
|
|
}
|
|
|
|
|
|
|
|
; IR-LABEL: @multi_divergent_region_exit_ret_unreachable(
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: %1 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %0)
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: Flow:
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %4 = phi i1 [ true, %LeafBlock1 ], [ false, %entry ]
|
|
|
|
; IR: %5 = phi i1 [ %10, %LeafBlock1 ], [ false, %entry ]
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: %6 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %3)
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: Flow2:
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %11 = phi i1 [ false, %exit1 ], [ %15, %Flow1 ]
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: call void @llvm.amdgcn.end.cf.i64(i64 %19)
|
|
|
|
; IR: %12 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %11)
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: br i1 %13, label %exit0, label %UnifiedReturnBlock
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: exit0:
|
|
|
|
; IR-NEXT: store volatile i32 17, i32 addrspace(3)* undef
|
|
|
|
; IR-NEXT: br label %UnifiedReturnBlock
|
|
|
|
|
|
|
|
; IR: Flow1:
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %15 = phi i1 [ %SwitchLeaf, %LeafBlock ], [ %4, %Flow ]
|
|
|
|
; IR: %16 = phi i1 [ %9, %LeafBlock ], [ %5, %Flow ]
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR: call void @llvm.amdgcn.end.cf.i64(i64 %8)
|
|
|
|
; IR: %17 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %16)
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %18 = extractvalue { i1, i64 } %17, 0
|
|
|
|
; IR: %19 = extractvalue { i1, i64 } %17, 1
|
|
|
|
; IR: br i1 %18, label %exit1, label %Flow2
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: exit1:
|
|
|
|
; IR-NEXT: store volatile i32 9, i32 addrspace(1)* undef
|
|
|
|
; IR-NEXT: call void @llvm.amdgcn.unreachable()
|
|
|
|
; IR-NEXT: br label %Flow2
|
|
|
|
|
|
|
|
; IR: UnifiedReturnBlock:
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 %14)
|
2017-03-25 03:52:05 +08:00
|
|
|
; IR-NEXT: ret void
|
|
|
|
define amdgpu_kernel void @multi_divergent_region_exit_ret_unreachable(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2) #0 {
|
|
|
|
entry:
|
|
|
|
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1
|
|
|
|
%tmp1 = add i32 0, %tmp
|
|
|
|
%tmp2 = zext i32 %tmp1 to i64
|
|
|
|
%tmp3 = add i64 0, %tmp2
|
|
|
|
%tmp4 = shl i64 %tmp3, 32
|
|
|
|
%tmp5 = ashr exact i64 %tmp4, 32
|
|
|
|
%tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %tmp5
|
|
|
|
%tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4
|
|
|
|
%tmp8 = sext i32 %tmp7 to i64
|
|
|
|
%tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp8
|
|
|
|
%tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4
|
|
|
|
%tmp13 = zext i32 %tmp10 to i64
|
|
|
|
%tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg2, i64 %tmp13
|
|
|
|
%tmp16 = load i32, i32 addrspace(1)* %tmp14, align 16
|
|
|
|
%Pivot = icmp slt i32 %tmp16, 2
|
|
|
|
br i1 %Pivot, label %LeafBlock, label %LeafBlock1
|
|
|
|
|
|
|
|
LeafBlock: ; preds = %entry
|
|
|
|
%SwitchLeaf = icmp eq i32 %tmp16, 1
|
|
|
|
br i1 %SwitchLeaf, label %exit0, label %exit1
|
|
|
|
|
|
|
|
LeafBlock1: ; preds = %entry
|
|
|
|
%SwitchLeaf2 = icmp eq i32 %tmp16, 2
|
|
|
|
br i1 %SwitchLeaf2, label %exit0, label %exit1
|
|
|
|
|
|
|
|
exit0: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store volatile i32 17, i32 addrspace(3)* undef
|
|
|
|
ret void
|
|
|
|
|
|
|
|
exit1: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store volatile i32 9, i32 addrspace(1)* undef
|
|
|
|
unreachable
|
|
|
|
}
|
|
|
|
|
|
|
|
; The non-uniformity of the branch to the exiting blocks requires
|
|
|
|
; looking at transitive predecessors.
|
|
|
|
|
|
|
|
; IR-LABEL: @indirect_multi_divergent_region_exit_ret_unreachable(
|
|
|
|
|
|
|
|
; IR: exit0: ; preds = %Flow2
|
|
|
|
; IR-NEXT: store volatile i32 17, i32 addrspace(3)* undef
|
|
|
|
; IR-NEXT: br label %UnifiedReturnBlock
|
|
|
|
|
|
|
|
|
|
|
|
; IR: indirect.exit1:
|
|
|
|
; IR: %load = load volatile i32, i32 addrspace(1)* undef
|
|
|
|
; IR: store volatile i32 %load, i32 addrspace(1)* undef
|
|
|
|
; IR: store volatile i32 9, i32 addrspace(1)* undef
|
|
|
|
; IR: call void @llvm.amdgcn.unreachable()
|
|
|
|
; IR-NEXT: br label %Flow2
|
|
|
|
|
|
|
|
; IR: UnifiedReturnBlock: ; preds = %exit0, %Flow2
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 %14)
|
2017-03-25 03:52:05 +08:00
|
|
|
; IR-NEXT: ret void
|
|
|
|
define amdgpu_kernel void @indirect_multi_divergent_region_exit_ret_unreachable(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2) #0 {
|
|
|
|
entry:
|
|
|
|
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1
|
|
|
|
%tmp1 = add i32 0, %tmp
|
|
|
|
%tmp2 = zext i32 %tmp1 to i64
|
|
|
|
%tmp3 = add i64 0, %tmp2
|
|
|
|
%tmp4 = shl i64 %tmp3, 32
|
|
|
|
%tmp5 = ashr exact i64 %tmp4, 32
|
|
|
|
%tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %tmp5
|
|
|
|
%tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4
|
|
|
|
%tmp8 = sext i32 %tmp7 to i64
|
|
|
|
%tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp8
|
|
|
|
%tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4
|
|
|
|
%tmp13 = zext i32 %tmp10 to i64
|
|
|
|
%tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg2, i64 %tmp13
|
|
|
|
%tmp16 = load i32, i32 addrspace(1)* %tmp14, align 16
|
|
|
|
%Pivot = icmp slt i32 %tmp16, 2
|
|
|
|
br i1 %Pivot, label %LeafBlock, label %LeafBlock1
|
|
|
|
|
|
|
|
LeafBlock: ; preds = %entry
|
|
|
|
%SwitchLeaf = icmp eq i32 %tmp16, 1
|
|
|
|
br i1 %SwitchLeaf, label %exit0, label %indirect.exit1
|
|
|
|
|
|
|
|
LeafBlock1: ; preds = %entry
|
|
|
|
%SwitchLeaf2 = icmp eq i32 %tmp16, 2
|
|
|
|
br i1 %SwitchLeaf2, label %exit0, label %indirect.exit1
|
|
|
|
|
|
|
|
exit0: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store volatile i32 17, i32 addrspace(3)* undef
|
|
|
|
ret void
|
|
|
|
|
|
|
|
indirect.exit1:
|
|
|
|
%load = load volatile i32, i32 addrspace(1)* undef
|
|
|
|
store volatile i32 %load, i32 addrspace(1)* undef
|
|
|
|
br label %exit1
|
|
|
|
|
|
|
|
exit1: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store volatile i32 9, i32 addrspace(1)* undef
|
|
|
|
unreachable
|
|
|
|
}
|
|
|
|
|
|
|
|
; IR-LABEL: @multi_divergent_region_exit_ret_switch(
|
|
|
|
define amdgpu_kernel void @multi_divergent_region_exit_ret_switch(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2) #0 {
|
|
|
|
entry:
|
|
|
|
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1
|
|
|
|
%tmp1 = add i32 0, %tmp
|
|
|
|
%tmp2 = zext i32 %tmp1 to i64
|
|
|
|
%tmp3 = add i64 0, %tmp2
|
|
|
|
%tmp4 = shl i64 %tmp3, 32
|
|
|
|
%tmp5 = ashr exact i64 %tmp4, 32
|
|
|
|
%tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %tmp5
|
|
|
|
%tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4
|
|
|
|
%tmp8 = sext i32 %tmp7 to i64
|
|
|
|
%tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp8
|
|
|
|
%tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4
|
|
|
|
%tmp13 = zext i32 %tmp10 to i64
|
|
|
|
%tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg2, i64 %tmp13
|
|
|
|
%tmp16 = load i32, i32 addrspace(1)* %tmp14, align 16
|
|
|
|
switch i32 %tmp16, label %exit1
|
|
|
|
[ i32 1, label %LeafBlock
|
|
|
|
i32 2, label %LeafBlock1
|
|
|
|
i32 3, label %exit0 ]
|
|
|
|
|
|
|
|
LeafBlock: ; preds = %entry
|
|
|
|
%SwitchLeaf = icmp eq i32 %tmp16, 1
|
|
|
|
br i1 %SwitchLeaf, label %exit0, label %exit1
|
|
|
|
|
|
|
|
LeafBlock1: ; preds = %entry
|
|
|
|
%SwitchLeaf2 = icmp eq i32 %tmp16, 2
|
|
|
|
br i1 %SwitchLeaf2, label %exit0, label %exit1
|
|
|
|
|
|
|
|
exit0: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store volatile i32 17, i32 addrspace(3)* undef
|
|
|
|
ret void
|
|
|
|
|
|
|
|
exit1: ; preds = %LeafBlock, %LeafBlock1
|
|
|
|
store volatile i32 9, i32 addrspace(1)* undef
|
|
|
|
unreachable
|
|
|
|
}
|
|
|
|
|
|
|
|
; IR-LABEL: @divergent_multi_ret_nest_in_uniform_triangle(
|
|
|
|
define amdgpu_kernel void @divergent_multi_ret_nest_in_uniform_triangle(i32 %arg0) #0 {
|
|
|
|
entry:
|
|
|
|
%uniform.cond0 = icmp eq i32 %arg0, 4
|
|
|
|
br i1 %uniform.cond0, label %divergent.multi.exit.region, label %uniform.ret
|
|
|
|
|
|
|
|
divergent.multi.exit.region:
|
|
|
|
%id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
|
|
|
|
%divergent.cond0 = icmp eq i32 %id.x, 0
|
|
|
|
br i1 %divergent.cond0, label %divergent.ret0, label %divergent.ret1
|
|
|
|
|
|
|
|
divergent.ret0:
|
|
|
|
store volatile i32 11, i32 addrspace(3)* undef
|
|
|
|
ret void
|
|
|
|
|
|
|
|
divergent.ret1:
|
|
|
|
store volatile i32 42, i32 addrspace(3)* undef
|
|
|
|
ret void
|
|
|
|
|
|
|
|
uniform.ret:
|
|
|
|
store volatile i32 9, i32 addrspace(1)* undef
|
|
|
|
ret void
|
|
|
|
}
|
|
|
|
|
|
|
|
; IR-LABEL: @divergent_complex_multi_ret_nest_in_uniform_triangle(
|
|
|
|
define amdgpu_kernel void @divergent_complex_multi_ret_nest_in_uniform_triangle(i32 %arg0) #0 {
|
|
|
|
entry:
|
|
|
|
%uniform.cond0 = icmp eq i32 %arg0, 4
|
|
|
|
br i1 %uniform.cond0, label %divergent.multi.exit.region, label %uniform.ret
|
|
|
|
|
|
|
|
divergent.multi.exit.region:
|
|
|
|
%id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
|
|
|
|
%divergent.cond0 = icmp eq i32 %id.x, 0
|
|
|
|
br i1 %divergent.cond0, label %divergent.if, label %divergent.ret1
|
|
|
|
|
|
|
|
divergent.if:
|
|
|
|
%vgpr0 = load volatile float, float addrspace(1)* undef
|
|
|
|
%divergent.cond1 = fcmp ogt float %vgpr0, 1.0
|
|
|
|
br i1 %divergent.cond1, label %divergent.then, label %divergent.endif
|
|
|
|
|
|
|
|
divergent.then:
|
|
|
|
%vgpr1 = load volatile float, float addrspace(1)* undef
|
|
|
|
%divergent.cond2 = fcmp olt float %vgpr1, 4.0
|
|
|
|
store volatile i32 33, i32 addrspace(1)* undef
|
|
|
|
br i1 %divergent.cond2, label %divergent.ret0, label %divergent.endif
|
|
|
|
|
|
|
|
divergent.endif:
|
|
|
|
store volatile i32 38, i32 addrspace(1)* undef
|
|
|
|
br label %divergent.ret0
|
|
|
|
|
|
|
|
divergent.ret0:
|
|
|
|
store volatile i32 11, i32 addrspace(3)* undef
|
|
|
|
ret void
|
|
|
|
|
|
|
|
divergent.ret1:
|
|
|
|
store volatile i32 42, i32 addrspace(3)* undef
|
|
|
|
ret void
|
|
|
|
|
|
|
|
uniform.ret:
|
|
|
|
store volatile i32 9, i32 addrspace(1)* undef
|
|
|
|
ret void
|
|
|
|
}
|
|
|
|
|
|
|
|
; IR-LABEL: @uniform_complex_multi_ret_nest_in_divergent_triangle(
|
|
|
|
; IR: Flow1: ; preds = %uniform.ret1, %uniform.multi.exit.region
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %8 = phi i1 [ false, %uniform.ret1 ], [ true, %uniform.multi.exit.region ]
|
|
|
|
; IR: br i1 %8, label %uniform.if, label %Flow2
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: Flow: ; preds = %uniform.then, %uniform.if
|
2017-04-25 04:25:01 +08:00
|
|
|
; IR: %11 = phi i1 [ %10, %uniform.then ], [ %9, %uniform.if ]
|
|
|
|
; IR: br i1 %11, label %uniform.endif, label %uniform.ret0
|
2017-03-25 03:52:05 +08:00
|
|
|
|
|
|
|
; IR: UnifiedReturnBlock: ; preds = %Flow3, %Flow2
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 %6)
|
2017-03-25 03:52:05 +08:00
|
|
|
; IR-NEXT: ret void
|
|
|
|
define amdgpu_kernel void @uniform_complex_multi_ret_nest_in_divergent_triangle(i32 %arg0) #0 {
|
|
|
|
entry:
|
|
|
|
%id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
|
|
|
|
%divergent.cond0 = icmp eq i32 %id.x, 0
|
|
|
|
br i1 %divergent.cond0, label %uniform.multi.exit.region, label %divergent.ret
|
|
|
|
|
|
|
|
uniform.multi.exit.region:
|
|
|
|
%uniform.cond0 = icmp eq i32 %arg0, 4
|
|
|
|
br i1 %uniform.cond0, label %uniform.if, label %uniform.ret1
|
|
|
|
|
|
|
|
uniform.if:
|
2018-02-14 02:00:25 +08:00
|
|
|
%sgpr0 = load volatile i32, i32 addrspace(4)* undef
|
2017-03-25 03:52:05 +08:00
|
|
|
%uniform.cond1 = icmp slt i32 %sgpr0, 1
|
|
|
|
br i1 %uniform.cond1, label %uniform.then, label %uniform.endif
|
|
|
|
|
|
|
|
uniform.then:
|
2018-02-14 02:00:25 +08:00
|
|
|
%sgpr1 = load volatile i32, i32 addrspace(4)* undef
|
2017-03-25 03:52:05 +08:00
|
|
|
%uniform.cond2 = icmp sge i32 %sgpr1, 4
|
|
|
|
store volatile i32 33, i32 addrspace(1)* undef
|
|
|
|
br i1 %uniform.cond2, label %uniform.ret0, label %uniform.endif
|
|
|
|
|
|
|
|
uniform.endif:
|
|
|
|
store volatile i32 38, i32 addrspace(1)* undef
|
|
|
|
br label %uniform.ret0
|
|
|
|
|
|
|
|
uniform.ret0:
|
|
|
|
store volatile i32 11, i32 addrspace(3)* undef
|
|
|
|
ret void
|
|
|
|
|
|
|
|
uniform.ret1:
|
|
|
|
store volatile i32 42, i32 addrspace(3)* undef
|
|
|
|
ret void
|
|
|
|
|
|
|
|
divergent.ret:
|
|
|
|
store volatile i32 9, i32 addrspace(1)* undef
|
|
|
|
ret void
|
|
|
|
}
|
|
|
|
|
|
|
|
; IR-LABEL: @multi_divergent_unreachable_exit(
|
|
|
|
; IR: UnifiedUnreachableBlock:
|
|
|
|
; IR-NEXT: call void @llvm.amdgcn.unreachable()
|
|
|
|
; IR-NEXT: br label %UnifiedReturnBlock
|
|
|
|
|
|
|
|
; IR: UnifiedReturnBlock:
|
2019-06-14 07:47:36 +08:00
|
|
|
; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64
|
2017-03-25 03:52:05 +08:00
|
|
|
; IR-NEXT: ret void
|
|
|
|
define amdgpu_kernel void @multi_divergent_unreachable_exit() #0 {
|
|
|
|
bb:
|
|
|
|
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
|
|
|
|
switch i32 %tmp, label %bb3 [
|
|
|
|
i32 2, label %bb1
|
|
|
|
i32 0, label %bb2
|
|
|
|
]
|
|
|
|
|
|
|
|
bb1: ; preds = %bb
|
|
|
|
unreachable
|
|
|
|
|
|
|
|
bb2: ; preds = %bb
|
|
|
|
unreachable
|
|
|
|
|
|
|
|
bb3: ; preds = %bb
|
|
|
|
switch i32 undef, label %bb5 [
|
|
|
|
i32 2, label %bb4
|
|
|
|
]
|
|
|
|
|
|
|
|
bb4: ; preds = %bb3
|
|
|
|
ret void
|
|
|
|
|
|
|
|
bb5: ; preds = %bb3
|
|
|
|
unreachable
|
|
|
|
}
|
|
|
|
|
[AMDGPU] Fix AMDGPUUnifyDivergentExitNodes
Summary:
For the case where "done" bits on existing exports are removed
by unifyReturnBlockSet(), unify all return blocks - even the
uniformly reached ones. We do not want to end up with a non-unified,
uniformly reached block containing a normal export with the "done"
bit cleared.
That case is believed to be rare - possible with infinite loops
in pixel shaders.
This is a fix for D71192.
Subscribers: arsenm, kzhuravl, jvesely, wdng, nhaehnle, yaxunl, dstuttard, tpr, t-tye, hiraditya, kerbowa, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D76364
2020-03-18 21:24:09 +08:00
|
|
|
; Test that there is an extra export inserted after the normal export,
|
|
|
|
; if the normal export is inside a uniformly reached block and there is
|
|
|
|
; an infinite loop in the pixel shader.
|
|
|
|
|
|
|
|
; IR-LABEL: @uniformly_reached_export
|
|
|
|
; IR-NEXT: .entry:
|
|
|
|
; IR: br i1 [[CND:%.*]], label %[[EXP:.*]], label %[[FLOW:.*]]
|
|
|
|
|
|
|
|
; IR: [[FLOW]]:
|
|
|
|
; IR-NEXT: phi
|
2020-03-19 20:33:51 +08:00
|
|
|
; IR-NEXT: br i1 [[CND2:%.*]], label %[[LOOP:.*]], label %UnifiedReturnBlock
|
[AMDGPU] Fix AMDGPUUnifyDivergentExitNodes
Summary:
For the case where "done" bits on existing exports are removed
by unifyReturnBlockSet(), unify all return blocks - even the
uniformly reached ones. We do not want to end up with a non-unified,
uniformly reached block containing a normal export with the "done"
bit cleared.
That case is believed to be rare - possible with infinite loops
in pixel shaders.
This is a fix for D71192.
Subscribers: arsenm, kzhuravl, jvesely, wdng, nhaehnle, yaxunl, dstuttard, tpr, t-tye, hiraditya, kerbowa, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D76364
2020-03-18 21:24:09 +08:00
|
|
|
|
2020-03-19 20:33:51 +08:00
|
|
|
; IR: [[LOOP]]:
|
|
|
|
; IR-NEXT: br i1 false, label %[[FLOW1:.*]], label %[[LOOP]]
|
[AMDGPU] Fix AMDGPUUnifyDivergentExitNodes
Summary:
For the case where "done" bits on existing exports are removed
by unifyReturnBlockSet(), unify all return blocks - even the
uniformly reached ones. We do not want to end up with a non-unified,
uniformly reached block containing a normal export with the "done"
bit cleared.
That case is believed to be rare - possible with infinite loops
in pixel shaders.
This is a fix for D71192.
Subscribers: arsenm, kzhuravl, jvesely, wdng, nhaehnle, yaxunl, dstuttard, tpr, t-tye, hiraditya, kerbowa, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D76364
2020-03-18 21:24:09 +08:00
|
|
|
|
|
|
|
; IR: [[EXP]]:
|
|
|
|
; IR-NEXT: call void @llvm.amdgcn.exp.compr.v2f16(i32 immarg 0, i32 immarg 15, <2 x half> <half 0xH3C00, half 0xH0000>, <2 x half> <half 0xH0000, half 0xH3C00>, i1 immarg false, i1 immarg true)
|
|
|
|
; IR-NEXT: br label %[[FLOW]]
|
|
|
|
|
2020-03-19 20:33:51 +08:00
|
|
|
; IR: [[FLOW1]]:
|
|
|
|
; IR-NEXT: br label %UnifiedReturnBlock
|
|
|
|
|
[AMDGPU] Fix AMDGPUUnifyDivergentExitNodes
Summary:
For the case where "done" bits on existing exports are removed
by unifyReturnBlockSet(), unify all return blocks - even the
uniformly reached ones. We do not want to end up with a non-unified,
uniformly reached block containing a normal export with the "done"
bit cleared.
That case is believed to be rare - possible with infinite loops
in pixel shaders.
This is a fix for D71192.
Subscribers: arsenm, kzhuravl, jvesely, wdng, nhaehnle, yaxunl, dstuttard, tpr, t-tye, hiraditya, kerbowa, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D76364
2020-03-18 21:24:09 +08:00
|
|
|
; IR: UnifiedReturnBlock:
|
|
|
|
; IR-NEXT: call void @llvm.amdgcn.exp.f32(i32 9, i32 0, float undef, float undef, float undef, float undef, i1 true, i1 true)
|
|
|
|
; IR-NEXT: ret void
|
|
|
|
|
|
|
|
define amdgpu_ps void @uniformly_reached_export(float inreg %tmp25) {
|
|
|
|
.entry:
|
|
|
|
%tmp26 = fcmp olt float %tmp25, 0.000000e+00
|
2020-03-19 20:33:51 +08:00
|
|
|
br i1 %tmp26, label %loop, label %bb27
|
[AMDGPU] Fix AMDGPUUnifyDivergentExitNodes
Summary:
For the case where "done" bits on existing exports are removed
by unifyReturnBlockSet(), unify all return blocks - even the
uniformly reached ones. We do not want to end up with a non-unified,
uniformly reached block containing a normal export with the "done"
bit cleared.
That case is believed to be rare - possible with infinite loops
in pixel shaders.
This is a fix for D71192.
Subscribers: arsenm, kzhuravl, jvesely, wdng, nhaehnle, yaxunl, dstuttard, tpr, t-tye, hiraditya, kerbowa, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D76364
2020-03-18 21:24:09 +08:00
|
|
|
|
2020-03-19 20:33:51 +08:00
|
|
|
loop: ; preds = %loop, %.entry
|
|
|
|
br label %loop
|
[AMDGPU] Fix AMDGPUUnifyDivergentExitNodes
Summary:
For the case where "done" bits on existing exports are removed
by unifyReturnBlockSet(), unify all return blocks - even the
uniformly reached ones. We do not want to end up with a non-unified,
uniformly reached block containing a normal export with the "done"
bit cleared.
That case is believed to be rare - possible with infinite loops
in pixel shaders.
This is a fix for D71192.
Subscribers: arsenm, kzhuravl, jvesely, wdng, nhaehnle, yaxunl, dstuttard, tpr, t-tye, hiraditya, kerbowa, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D76364
2020-03-18 21:24:09 +08:00
|
|
|
|
|
|
|
bb27: ; preds = %.entry
|
|
|
|
call void @llvm.amdgcn.exp.compr.v2f16(i32 immarg 0, i32 immarg 15, <2 x half> <half 0xH3C00, half 0xH0000>, <2 x half> <half 0xH0000, half 0xH3C00>, i1 immarg true, i1 immarg true)
|
|
|
|
ret void
|
|
|
|
}
|
|
|
|
|
|
|
|
declare void @llvm.amdgcn.exp.compr.v2f16(i32 immarg, i32 immarg, <2 x half>, <2 x half>, i1 immarg, i1 immarg) #0
|
2017-03-25 03:52:05 +08:00
|
|
|
declare i32 @llvm.amdgcn.workitem.id.x() #1
|
|
|
|
|
|
|
|
attributes #0 = { nounwind }
|
|
|
|
attributes #1 = { nounwind readnone }
|