2019-03-06 19:47:43 +08:00
|
|
|
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
|
|
|
; RUN: llc < %s -mtriple=x86_64-unknown-linux | FileCheck %s --check-prefix=CHECK --check-prefix=CMOV
|
|
|
|
|
; RUN: llc < %s -mtriple=i686-unknown-linux | FileCheck %s --check-prefix=CHECK --check-prefix=NOCMOV
|
[X86] Combine (cmov (and/or (setcc) (setcc))) into (cmov (cmov)).
Fold and/or of setcc's to double CMOV:
(CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2)
(CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2)
When we can't use the CMOV instruction, it might increase branch
mispredicts. When we can, or when there is no mispredict, this
improves throughput and reduces register pressure.
These can't be catched by generic combines, because the pattern can
appear when legalizing some instructions (such as fcmp une).
rdar://19767934
http://reviews.llvm.org/D7634
llvm-svn: 231045
2015-03-03 09:09:14 +08:00
|
|
|
|
|
|
|
|
target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
|
|
|
|
|
|
|
|
|
|
; Test 2xCMOV patterns exposed after legalization.
|
|
|
|
|
; One way to do that is with (select (fcmp une/oeq)), which gets
|
|
|
|
|
; legalized to setp/setne.
|
|
|
|
|
|
2019-03-22 23:33:51 +08:00
|
|
|
define i32 @test_select_fcmp_oeq_i32(float %a, float %b, i32 %c, i32 %d) nounwind {
|
2019-03-06 19:47:43 +08:00
|
|
|
; CMOV-LABEL: test_select_fcmp_oeq_i32:
|
|
|
|
|
; CMOV: # %bb.0: # %entry
|
|
|
|
|
; CMOV-NEXT: movl %edi, %eax
|
|
|
|
|
; CMOV-NEXT: ucomiss %xmm1, %xmm0
|
|
|
|
|
; CMOV-NEXT: cmovnel %esi, %eax
|
|
|
|
|
; CMOV-NEXT: cmovpl %esi, %eax
|
|
|
|
|
; CMOV-NEXT: retq
|
|
|
|
|
;
|
|
|
|
|
; NOCMOV-LABEL: test_select_fcmp_oeq_i32:
|
|
|
|
|
; NOCMOV: # %bb.0: # %entry
|
|
|
|
|
; NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: fucompp
|
|
|
|
|
; NOCMOV-NEXT: fnstsw %ax
|
|
|
|
|
; NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
|
|
|
|
|
; NOCMOV-NEXT: sahf
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %eax
|
|
|
|
|
; NOCMOV-NEXT: jne .LBB0_3
|
|
|
|
|
; NOCMOV-NEXT: # %bb.1: # %entry
|
|
|
|
|
; NOCMOV-NEXT: jp .LBB0_3
|
|
|
|
|
; NOCMOV-NEXT: # %bb.2: # %entry
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %eax
|
|
|
|
|
; NOCMOV-NEXT: .LBB0_3: # %entry
|
|
|
|
|
; NOCMOV-NEXT: movl (%eax), %eax
|
|
|
|
|
; NOCMOV-NEXT: retl
|
[X86] Combine (cmov (and/or (setcc) (setcc))) into (cmov (cmov)).
Fold and/or of setcc's to double CMOV:
(CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2)
(CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2)
When we can't use the CMOV instruction, it might increase branch
mispredicts. When we can, or when there is no mispredict, this
improves throughput and reduces register pressure.
These can't be catched by generic combines, because the pattern can
appear when legalizing some instructions (such as fcmp une).
rdar://19767934
http://reviews.llvm.org/D7634
llvm-svn: 231045
2015-03-03 09:09:14 +08:00
|
|
|
entry:
|
|
|
|
|
%cmp = fcmp oeq float %a, %b
|
|
|
|
|
%r = select i1 %cmp, i32 %c, i32 %d
|
|
|
|
|
ret i32 %r
|
|
|
|
|
}
|
|
|
|
|
|
2019-03-22 23:33:51 +08:00
|
|
|
define i64 @test_select_fcmp_oeq_i64(float %a, float %b, i64 %c, i64 %d) nounwind {
|
2019-03-06 19:47:43 +08:00
|
|
|
; CMOV-LABEL: test_select_fcmp_oeq_i64:
|
|
|
|
|
; CMOV: # %bb.0: # %entry
|
|
|
|
|
; CMOV-NEXT: movq %rdi, %rax
|
|
|
|
|
; CMOV-NEXT: ucomiss %xmm1, %xmm0
|
|
|
|
|
; CMOV-NEXT: cmovneq %rsi, %rax
|
|
|
|
|
; CMOV-NEXT: cmovpq %rsi, %rax
|
|
|
|
|
; CMOV-NEXT: retq
|
|
|
|
|
;
|
|
|
|
|
; NOCMOV-LABEL: test_select_fcmp_oeq_i64:
|
|
|
|
|
; NOCMOV: # %bb.0: # %entry
|
|
|
|
|
; NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: fucompp
|
|
|
|
|
; NOCMOV-NEXT: fnstsw %ax
|
|
|
|
|
; NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
|
|
|
|
|
; NOCMOV-NEXT: sahf
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx
|
|
|
|
|
; NOCMOV-NEXT: jne .LBB1_3
|
|
|
|
|
; NOCMOV-NEXT: # %bb.1: # %entry
|
|
|
|
|
; NOCMOV-NEXT: jp .LBB1_3
|
|
|
|
|
; NOCMOV-NEXT: # %bb.2: # %entry
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx
|
|
|
|
|
; NOCMOV-NEXT: .LBB1_3: # %entry
|
|
|
|
|
; NOCMOV-NEXT: movl (%ecx), %eax
|
|
|
|
|
; NOCMOV-NEXT: movl 4(%ecx), %edx
|
|
|
|
|
; NOCMOV-NEXT: retl
|
[X86] Combine (cmov (and/or (setcc) (setcc))) into (cmov (cmov)).
Fold and/or of setcc's to double CMOV:
(CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2)
(CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2)
When we can't use the CMOV instruction, it might increase branch
mispredicts. When we can, or when there is no mispredict, this
improves throughput and reduces register pressure.
These can't be catched by generic combines, because the pattern can
appear when legalizing some instructions (such as fcmp une).
rdar://19767934
http://reviews.llvm.org/D7634
llvm-svn: 231045
2015-03-03 09:09:14 +08:00
|
|
|
entry:
|
|
|
|
|
%cmp = fcmp oeq float %a, %b
|
|
|
|
|
%r = select i1 %cmp, i64 %c, i64 %d
|
|
|
|
|
ret i64 %r
|
|
|
|
|
}
|
|
|
|
|
|
2019-03-22 23:33:51 +08:00
|
|
|
define i64 @test_select_fcmp_une_i64(float %a, float %b, i64 %c, i64 %d) nounwind {
|
2019-03-06 19:47:43 +08:00
|
|
|
; CMOV-LABEL: test_select_fcmp_une_i64:
|
|
|
|
|
; CMOV: # %bb.0: # %entry
|
|
|
|
|
; CMOV-NEXT: movq %rsi, %rax
|
|
|
|
|
; CMOV-NEXT: ucomiss %xmm1, %xmm0
|
|
|
|
|
; CMOV-NEXT: cmovneq %rdi, %rax
|
|
|
|
|
; CMOV-NEXT: cmovpq %rdi, %rax
|
|
|
|
|
; CMOV-NEXT: retq
|
|
|
|
|
;
|
|
|
|
|
; NOCMOV-LABEL: test_select_fcmp_une_i64:
|
|
|
|
|
; NOCMOV: # %bb.0: # %entry
|
|
|
|
|
; NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: fucompp
|
|
|
|
|
; NOCMOV-NEXT: fnstsw %ax
|
|
|
|
|
; NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
|
|
|
|
|
; NOCMOV-NEXT: sahf
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx
|
|
|
|
|
; NOCMOV-NEXT: jne .LBB2_3
|
|
|
|
|
; NOCMOV-NEXT: # %bb.1: # %entry
|
|
|
|
|
; NOCMOV-NEXT: jp .LBB2_3
|
|
|
|
|
; NOCMOV-NEXT: # %bb.2: # %entry
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx
|
|
|
|
|
; NOCMOV-NEXT: .LBB2_3: # %entry
|
|
|
|
|
; NOCMOV-NEXT: movl (%ecx), %eax
|
|
|
|
|
; NOCMOV-NEXT: movl 4(%ecx), %edx
|
|
|
|
|
; NOCMOV-NEXT: retl
|
[X86] Combine (cmov (and/or (setcc) (setcc))) into (cmov (cmov)).
Fold and/or of setcc's to double CMOV:
(CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2)
(CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2)
When we can't use the CMOV instruction, it might increase branch
mispredicts. When we can, or when there is no mispredict, this
improves throughput and reduces register pressure.
These can't be catched by generic combines, because the pattern can
appear when legalizing some instructions (such as fcmp une).
rdar://19767934
http://reviews.llvm.org/D7634
llvm-svn: 231045
2015-03-03 09:09:14 +08:00
|
|
|
entry:
|
|
|
|
|
%cmp = fcmp une float %a, %b
|
|
|
|
|
%r = select i1 %cmp, i64 %c, i64 %d
|
|
|
|
|
ret i64 %r
|
|
|
|
|
}
|
|
|
|
|
|
2019-03-22 23:33:51 +08:00
|
|
|
define double @test_select_fcmp_oeq_f64(float %a, float %b, double %c, double %d) nounwind {
|
2019-03-06 19:47:43 +08:00
|
|
|
; CMOV-LABEL: test_select_fcmp_oeq_f64:
|
|
|
|
|
; CMOV: # %bb.0: # %entry
|
|
|
|
|
; CMOV-NEXT: ucomiss %xmm1, %xmm0
|
|
|
|
|
; CMOV-NEXT: jne .LBB3_3
|
|
|
|
|
; CMOV-NEXT: # %bb.1: # %entry
|
|
|
|
|
; CMOV-NEXT: jp .LBB3_3
|
|
|
|
|
; CMOV-NEXT: # %bb.2: # %entry
|
|
|
|
|
; CMOV-NEXT: movaps %xmm2, %xmm3
|
|
|
|
|
; CMOV-NEXT: .LBB3_3: # %entry
|
|
|
|
|
; CMOV-NEXT: movaps %xmm3, %xmm0
|
|
|
|
|
; CMOV-NEXT: retq
|
|
|
|
|
;
|
|
|
|
|
; NOCMOV-LABEL: test_select_fcmp_oeq_f64:
|
|
|
|
|
; NOCMOV: # %bb.0: # %entry
|
|
|
|
|
; NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: fucompp
|
|
|
|
|
; NOCMOV-NEXT: fnstsw %ax
|
|
|
|
|
; NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
|
|
|
|
|
; NOCMOV-NEXT: sahf
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %eax
|
|
|
|
|
; NOCMOV-NEXT: jne .LBB3_3
|
|
|
|
|
; NOCMOV-NEXT: # %bb.1: # %entry
|
|
|
|
|
; NOCMOV-NEXT: jp .LBB3_3
|
|
|
|
|
; NOCMOV-NEXT: # %bb.2: # %entry
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %eax
|
|
|
|
|
; NOCMOV-NEXT: .LBB3_3: # %entry
|
|
|
|
|
; NOCMOV-NEXT: fldl (%eax)
|
|
|
|
|
; NOCMOV-NEXT: retl
|
[X86] Combine (cmov (and/or (setcc) (setcc))) into (cmov (cmov)).
Fold and/or of setcc's to double CMOV:
(CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2)
(CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2)
When we can't use the CMOV instruction, it might increase branch
mispredicts. When we can, or when there is no mispredict, this
improves throughput and reduces register pressure.
These can't be catched by generic combines, because the pattern can
appear when legalizing some instructions (such as fcmp une).
rdar://19767934
http://reviews.llvm.org/D7634
llvm-svn: 231045
2015-03-03 09:09:14 +08:00
|
|
|
entry:
|
|
|
|
|
%cmp = fcmp oeq float %a, %b
|
|
|
|
|
%r = select i1 %cmp, double %c, double %d
|
|
|
|
|
ret double %r
|
|
|
|
|
}
|
|
|
|
|
|
2019-03-22 23:33:51 +08:00
|
|
|
define <4 x i32> @test_select_fcmp_oeq_v4i32(float %a, float %b, <4 x i32> %c, <4 x i32> %d) nounwind {
|
2019-03-06 19:47:43 +08:00
|
|
|
; CMOV-LABEL: test_select_fcmp_oeq_v4i32:
|
|
|
|
|
; CMOV: # %bb.0: # %entry
|
|
|
|
|
; CMOV-NEXT: ucomiss %xmm1, %xmm0
|
|
|
|
|
; CMOV-NEXT: jne .LBB4_3
|
|
|
|
|
; CMOV-NEXT: # %bb.1: # %entry
|
|
|
|
|
; CMOV-NEXT: jp .LBB4_3
|
|
|
|
|
; CMOV-NEXT: # %bb.2: # %entry
|
|
|
|
|
; CMOV-NEXT: movaps %xmm2, %xmm3
|
|
|
|
|
; CMOV-NEXT: .LBB4_3: # %entry
|
|
|
|
|
; CMOV-NEXT: movaps %xmm3, %xmm0
|
|
|
|
|
; CMOV-NEXT: retq
|
|
|
|
|
;
|
|
|
|
|
; NOCMOV-LABEL: test_select_fcmp_oeq_v4i32:
|
|
|
|
|
; NOCMOV: # %bb.0: # %entry
|
|
|
|
|
; NOCMOV-NEXT: pushl %edi
|
|
|
|
|
; NOCMOV-NEXT: pushl %esi
|
|
|
|
|
; NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: fucompp
|
|
|
|
|
; NOCMOV-NEXT: fnstsw %ax
|
|
|
|
|
; NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
|
|
|
|
|
; NOCMOV-NEXT: sahf
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %eax
|
|
|
|
|
; NOCMOV-NEXT: jne .LBB4_3
|
|
|
|
|
; NOCMOV-NEXT: # %bb.1: # %entry
|
|
|
|
|
; NOCMOV-NEXT: jp .LBB4_3
|
|
|
|
|
; NOCMOV-NEXT: # %bb.2: # %entry
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %eax
|
|
|
|
|
; NOCMOV-NEXT: .LBB4_3: # %entry
|
|
|
|
|
; NOCMOV-NEXT: movl (%eax), %ecx
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %edx
|
|
|
|
|
; NOCMOV-NEXT: jne .LBB4_6
|
|
|
|
|
; NOCMOV-NEXT: # %bb.4: # %entry
|
|
|
|
|
; NOCMOV-NEXT: jp .LBB4_6
|
|
|
|
|
; NOCMOV-NEXT: # %bb.5: # %entry
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %edx
|
|
|
|
|
; NOCMOV-NEXT: .LBB4_6: # %entry
|
|
|
|
|
; NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
|
|
|
|
|
; NOCMOV-NEXT: movl (%edx), %edx
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %esi
|
|
|
|
|
; NOCMOV-NEXT: jne .LBB4_9
|
|
|
|
|
; NOCMOV-NEXT: # %bb.7: # %entry
|
|
|
|
|
; NOCMOV-NEXT: jp .LBB4_9
|
|
|
|
|
; NOCMOV-NEXT: # %bb.8: # %entry
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %esi
|
|
|
|
|
; NOCMOV-NEXT: .LBB4_9: # %entry
|
|
|
|
|
; NOCMOV-NEXT: movl (%esi), %esi
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %edi
|
|
|
|
|
; NOCMOV-NEXT: jne .LBB4_12
|
|
|
|
|
; NOCMOV-NEXT: # %bb.10: # %entry
|
|
|
|
|
; NOCMOV-NEXT: jp .LBB4_12
|
|
|
|
|
; NOCMOV-NEXT: # %bb.11: # %entry
|
|
|
|
|
; NOCMOV-NEXT: leal {{[0-9]+}}(%esp), %edi
|
|
|
|
|
; NOCMOV-NEXT: .LBB4_12: # %entry
|
|
|
|
|
; NOCMOV-NEXT: movl (%edi), %edi
|
|
|
|
|
; NOCMOV-NEXT: movl %edi, 12(%eax)
|
|
|
|
|
; NOCMOV-NEXT: movl %esi, 8(%eax)
|
|
|
|
|
; NOCMOV-NEXT: movl %edx, 4(%eax)
|
|
|
|
|
; NOCMOV-NEXT: movl %ecx, (%eax)
|
|
|
|
|
; NOCMOV-NEXT: popl %esi
|
|
|
|
|
; NOCMOV-NEXT: popl %edi
|
|
|
|
|
; NOCMOV-NEXT: retl $4
|
[X86] Combine (cmov (and/or (setcc) (setcc))) into (cmov (cmov)).
Fold and/or of setcc's to double CMOV:
(CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2)
(CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2)
When we can't use the CMOV instruction, it might increase branch
mispredicts. When we can, or when there is no mispredict, this
improves throughput and reduces register pressure.
These can't be catched by generic combines, because the pattern can
appear when legalizing some instructions (such as fcmp une).
rdar://19767934
http://reviews.llvm.org/D7634
llvm-svn: 231045
2015-03-03 09:09:14 +08:00
|
|
|
entry:
|
|
|
|
|
%cmp = fcmp oeq float %a, %b
|
|
|
|
|
%r = select i1 %cmp, <4 x i32> %c, <4 x i32> %d
|
|
|
|
|
ret <4 x i32> %r
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
; Also make sure we catch the original code-sequence of interest:
|
|
|
|
|
|
2019-03-22 23:33:51 +08:00
|
|
|
define float @test_zext_fcmp_une(float %a, float %b) nounwind {
|
2019-03-06 19:47:43 +08:00
|
|
|
; CMOV-LABEL: test_zext_fcmp_une:
|
|
|
|
|
; CMOV: # %bb.0: # %entry
|
|
|
|
|
; CMOV-NEXT: ucomiss %xmm1, %xmm0
|
|
|
|
|
; CMOV-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
|
|
|
|
|
; CMOV-NEXT: jne .LBB5_3
|
|
|
|
|
; CMOV-NEXT: # %bb.1: # %entry
|
|
|
|
|
; CMOV-NEXT: jp .LBB5_3
|
|
|
|
|
; CMOV-NEXT: # %bb.2: # %entry
|
|
|
|
|
; CMOV-NEXT: xorps %xmm0, %xmm0
|
|
|
|
|
; CMOV-NEXT: .LBB5_3: # %entry
|
|
|
|
|
; CMOV-NEXT: retq
|
|
|
|
|
;
|
|
|
|
|
; NOCMOV-LABEL: test_zext_fcmp_une:
|
|
|
|
|
; NOCMOV: # %bb.0: # %entry
|
|
|
|
|
; NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: fucompp
|
|
|
|
|
; NOCMOV-NEXT: fnstsw %ax
|
|
|
|
|
; NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
|
|
|
|
|
; NOCMOV-NEXT: sahf
|
|
|
|
|
; NOCMOV-NEXT: fld1
|
|
|
|
|
; NOCMOV-NEXT: fldz
|
|
|
|
|
; NOCMOV-NEXT: jne .LBB5_1
|
|
|
|
|
; NOCMOV-NEXT: # %bb.2: # %entry
|
|
|
|
|
; NOCMOV-NEXT: jp .LBB5_5
|
|
|
|
|
; NOCMOV-NEXT: # %bb.3: # %entry
|
|
|
|
|
; NOCMOV-NEXT: fstp %st(1)
|
|
|
|
|
; NOCMOV-NEXT: jmp .LBB5_4
|
|
|
|
|
; NOCMOV-NEXT: .LBB5_1:
|
|
|
|
|
; NOCMOV-NEXT: fstp %st(0)
|
|
|
|
|
; NOCMOV-NEXT: .LBB5_4: # %entry
|
|
|
|
|
; NOCMOV-NEXT: fldz
|
|
|
|
|
; NOCMOV-NEXT: .LBB5_5: # %entry
|
|
|
|
|
; NOCMOV-NEXT: fstp %st(0)
|
|
|
|
|
; NOCMOV-NEXT: retl
|
[X86] Combine (cmov (and/or (setcc) (setcc))) into (cmov (cmov)).
Fold and/or of setcc's to double CMOV:
(CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2)
(CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2)
When we can't use the CMOV instruction, it might increase branch
mispredicts. When we can, or when there is no mispredict, this
improves throughput and reduces register pressure.
These can't be catched by generic combines, because the pattern can
appear when legalizing some instructions (such as fcmp une).
rdar://19767934
http://reviews.llvm.org/D7634
llvm-svn: 231045
2015-03-03 09:09:14 +08:00
|
|
|
entry:
|
|
|
|
|
%cmp = fcmp une float %a, %b
|
|
|
|
|
%conv1 = zext i1 %cmp to i32
|
|
|
|
|
%conv2 = sitofp i32 %conv1 to float
|
|
|
|
|
ret float %conv2
|
|
|
|
|
}
|
|
|
|
|
|
2019-03-22 23:33:51 +08:00
|
|
|
define float @test_zext_fcmp_oeq(float %a, float %b) nounwind {
|
2019-03-06 19:47:43 +08:00
|
|
|
; CMOV-LABEL: test_zext_fcmp_oeq:
|
|
|
|
|
; CMOV: # %bb.0: # %entry
|
|
|
|
|
; CMOV-NEXT: ucomiss %xmm1, %xmm0
|
|
|
|
|
; CMOV-NEXT: xorps %xmm0, %xmm0
|
|
|
|
|
; CMOV-NEXT: jne .LBB6_3
|
|
|
|
|
; CMOV-NEXT: # %bb.1: # %entry
|
|
|
|
|
; CMOV-NEXT: jp .LBB6_3
|
|
|
|
|
; CMOV-NEXT: # %bb.2: # %entry
|
|
|
|
|
; CMOV-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
|
|
|
|
|
; CMOV-NEXT: .LBB6_3: # %entry
|
|
|
|
|
; CMOV-NEXT: retq
|
|
|
|
|
;
|
|
|
|
|
; NOCMOV-LABEL: test_zext_fcmp_oeq:
|
|
|
|
|
; NOCMOV: # %bb.0: # %entry
|
|
|
|
|
; NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: fucompp
|
|
|
|
|
; NOCMOV-NEXT: fnstsw %ax
|
|
|
|
|
; NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
|
|
|
|
|
; NOCMOV-NEXT: sahf
|
|
|
|
|
; NOCMOV-NEXT: fldz
|
|
|
|
|
; NOCMOV-NEXT: fld1
|
|
|
|
|
; NOCMOV-NEXT: jne .LBB6_1
|
|
|
|
|
; NOCMOV-NEXT: # %bb.2: # %entry
|
|
|
|
|
; NOCMOV-NEXT: jp .LBB6_5
|
|
|
|
|
; NOCMOV-NEXT: # %bb.3: # %entry
|
|
|
|
|
; NOCMOV-NEXT: fstp %st(1)
|
|
|
|
|
; NOCMOV-NEXT: jmp .LBB6_4
|
|
|
|
|
; NOCMOV-NEXT: .LBB6_1:
|
|
|
|
|
; NOCMOV-NEXT: fstp %st(0)
|
|
|
|
|
; NOCMOV-NEXT: .LBB6_4: # %entry
|
|
|
|
|
; NOCMOV-NEXT: fldz
|
|
|
|
|
; NOCMOV-NEXT: .LBB6_5: # %entry
|
|
|
|
|
; NOCMOV-NEXT: fstp %st(0)
|
|
|
|
|
; NOCMOV-NEXT: retl
|
[X86] Combine (cmov (and/or (setcc) (setcc))) into (cmov (cmov)).
Fold and/or of setcc's to double CMOV:
(CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2)
(CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2)
When we can't use the CMOV instruction, it might increase branch
mispredicts. When we can, or when there is no mispredict, this
improves throughput and reduces register pressure.
These can't be catched by generic combines, because the pattern can
appear when legalizing some instructions (such as fcmp une).
rdar://19767934
http://reviews.llvm.org/D7634
llvm-svn: 231045
2015-03-03 09:09:14 +08:00
|
|
|
entry:
|
|
|
|
|
%cmp = fcmp oeq float %a, %b
|
|
|
|
|
%conv1 = zext i1 %cmp to i32
|
|
|
|
|
%conv2 = sitofp i32 %conv1 to float
|
|
|
|
|
ret float %conv2
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
attributes #0 = { nounwind }
|
2016-01-26 06:08:25 +08:00
|
|
|
|
|
|
|
|
@g8 = global i8 0
|
|
|
|
|
|
|
|
|
|
; The following test failed because llvm had a bug where a structure like:
|
|
|
|
|
;
|
2017-12-07 18:40:31 +08:00
|
|
|
; %12 = CMOV_GR8 %7, %11 ... (lt)
|
|
|
|
|
; %13 = CMOV_GR8 %12, %11 ... (gt)
|
2016-01-26 06:08:25 +08:00
|
|
|
;
|
|
|
|
|
; was lowered to:
|
|
|
|
|
;
|
|
|
|
|
; The first two cmovs got expanded to:
|
2017-12-05 01:18:51 +08:00
|
|
|
; %bb.0:
|
[X86] Merge the different Jcc instructions for each condition code into single instructions that store the condition code as an operand.
Summary:
This avoids needing an isel pattern for each condition code. And it removes translation switches for converting between Jcc instructions and condition codes.
Now the printer, encoder and disassembler take care of converting the immediate. We use InstAliases to handle the assembly matching. But we print using the asm string in the instruction definition. The instruction itself is marked IsCodeGenOnly=1 to hide it from the assembly parser.
Reviewers: spatel, lebedev.ri, courbet, gchatelet, RKSimon
Reviewed By: RKSimon
Subscribers: MatzeB, qcolombet, eraman, hiraditya, arphaman, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D60228
llvm-svn: 357802
2019-04-06 03:28:09 +08:00
|
|
|
; JCC_1 %bb.9, 12
|
2017-12-05 01:18:51 +08:00
|
|
|
; %bb.7:
|
[X86] Merge the different Jcc instructions for each condition code into single instructions that store the condition code as an operand.
Summary:
This avoids needing an isel pattern for each condition code. And it removes translation switches for converting between Jcc instructions and condition codes.
Now the printer, encoder and disassembler take care of converting the immediate. We use InstAliases to handle the assembly matching. But we print using the asm string in the instruction definition. The instruction itself is marked IsCodeGenOnly=1 to hide it from the assembly parser.
Reviewers: spatel, lebedev.ri, courbet, gchatelet, RKSimon
Reviewed By: RKSimon
Subscribers: MatzeB, qcolombet, eraman, hiraditya, arphaman, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D60228
llvm-svn: 357802
2019-04-06 03:28:09 +08:00
|
|
|
; JCC_1 %bb.9, 15
|
2017-12-05 01:18:51 +08:00
|
|
|
; %bb.8:
|
|
|
|
|
; %bb.9:
|
|
|
|
|
; %12 = phi(%7, %bb.8, %11, %bb.0, %12, %bb.7)
|
2017-11-30 20:12:19 +08:00
|
|
|
; %13 = COPY %12
|
|
|
|
|
; Which was invalid as %12 is not the same value as %13
|
2016-01-26 06:08:25 +08:00
|
|
|
|
2019-03-22 23:33:51 +08:00
|
|
|
define void @no_cascade_opt(i32 %v0, i32 %v1, i32 %v2, i32 %v3) nounwind {
|
2019-03-06 19:47:43 +08:00
|
|
|
; CMOV-LABEL: no_cascade_opt:
|
|
|
|
|
; CMOV: # %bb.0: # %entry
|
|
|
|
|
; CMOV-NEXT: cmpl %edx, %esi
|
[X86] Promote i8 CMOV's (PR40965)
Summary:
@mclow.lists brought up this issue up in IRC, it came up during
implementation of libc++ `std::midpoint()` implementation (D59099)
https://godbolt.org/z/oLrHBP
Currently LLVM X86 backend only promotes i8 CMOV if it came from 2x`trunc`.
This differential proposes to always promote i8 CMOV.
There are several concerns here:
* Is this actually more performant, or is it just the ASM that looks cuter?
* Does this result in partial register stalls?
* What about branch predictor?
# Indeed, performance should be the main point here.
Let's look at a simple microbenchmark: {F8412076}
```
#include "benchmark/benchmark.h"
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <iterator>
#include <limits>
#include <random>
#include <type_traits>
#include <utility>
#include <vector>
// Future preliminary libc++ code, from Marshall Clow.
namespace std {
template <class _Tp>
__inline _Tp midpoint(_Tp __a, _Tp __b) noexcept {
using _Up = typename std::make_unsigned<typename remove_cv<_Tp>::type>::type;
int __sign = 1;
_Up __m = __a;
_Up __M = __b;
if (__a > __b) {
__sign = -1;
__m = __b;
__M = __a;
}
return __a + __sign * _Tp(_Up(__M - __m) >> 1);
}
} // namespace std
template <typename T>
std::vector<T> getVectorOfRandomNumbers(size_t count) {
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<T> dis(std::numeric_limits<T>::min(),
std::numeric_limits<T>::max());
std::vector<T> v;
v.reserve(count);
std::generate_n(std::back_inserter(v), count,
[&dis, &gen]() { return dis(gen); });
assert(v.size() == count);
return v;
}
struct RandRand {
template <typename T>
static std::pair<std::vector<T>, std::vector<T>> Gen(size_t count) {
return std::make_pair(getVectorOfRandomNumbers<T>(count),
getVectorOfRandomNumbers<T>(count));
}
};
struct ZeroRand {
template <typename T>
static std::pair<std::vector<T>, std::vector<T>> Gen(size_t count) {
return std::make_pair(std::vector<T>(count, T(0)),
getVectorOfRandomNumbers<T>(count));
}
};
template <class T, class Gen>
void BM_StdMidpoint(benchmark::State& state) {
const size_t Length = state.range(0);
const std::pair<std::vector<T>, std::vector<T>> Data =
Gen::template Gen<T>(Length);
const std::vector<T>& a = Data.first;
const std::vector<T>& b = Data.second;
assert(a.size() == Length && b.size() == a.size());
benchmark::ClobberMemory();
benchmark::DoNotOptimize(a);
benchmark::DoNotOptimize(a.data());
benchmark::DoNotOptimize(b);
benchmark::DoNotOptimize(b.data());
for (auto _ : state) {
for (size_t i = 0; i < Length; i++) {
const auto calculated = std::midpoint(a[i], b[i]);
benchmark::DoNotOptimize(calculated);
}
}
state.SetComplexityN(Length);
state.counters["midpoints"] =
benchmark::Counter(Length, benchmark::Counter::kIsIterationInvariant);
state.counters["midpoints/sec"] =
benchmark::Counter(Length, benchmark::Counter::kIsIterationInvariantRate);
const size_t BytesRead = 2 * sizeof(T) * Length;
state.counters["bytes_read/iteration"] =
benchmark::Counter(BytesRead, benchmark::Counter::kDefaults,
benchmark::Counter::OneK::kIs1024);
state.counters["bytes_read/sec"] = benchmark::Counter(
BytesRead, benchmark::Counter::kIsIterationInvariantRate,
benchmark::Counter::OneK::kIs1024);
}
template <typename T>
static void CustomArguments(benchmark::internal::Benchmark* b) {
const size_t L2SizeBytes = 2 * 1024 * 1024;
// What is the largest range we can check to always fit within given L2 cache?
const size_t MaxLen = L2SizeBytes / /*total bufs*/ 2 /
/*maximal elt size*/ sizeof(T) / /*safety margin*/ 2;
b->RangeMultiplier(2)->Range(1, MaxLen)->Complexity(benchmark::oN);
}
// Both of the values are random.
// The comparison is unpredictable.
BENCHMARK_TEMPLATE(BM_StdMidpoint, int32_t, RandRand)
->Apply(CustomArguments<int32_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint32_t, RandRand)
->Apply(CustomArguments<uint32_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, int64_t, RandRand)
->Apply(CustomArguments<int64_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint64_t, RandRand)
->Apply(CustomArguments<uint64_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, int16_t, RandRand)
->Apply(CustomArguments<int16_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint16_t, RandRand)
->Apply(CustomArguments<uint16_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, int8_t, RandRand)
->Apply(CustomArguments<int8_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint8_t, RandRand)
->Apply(CustomArguments<uint8_t>);
// One value is always zero, and another is bigger or equal than zero.
// The comparison is predictable.
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint32_t, ZeroRand)
->Apply(CustomArguments<uint32_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint64_t, ZeroRand)
->Apply(CustomArguments<uint64_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint16_t, ZeroRand)
->Apply(CustomArguments<uint16_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint8_t, ZeroRand)
->Apply(CustomArguments<uint8_t>);
```
```
$ ~/src/googlebenchmark/tools/compare.py --no-utest benchmarks ./llvm-cmov-bench-OLD ./llvm-cmov-bench-NEW
RUNNING: ./llvm-cmov-bench-OLD --benchmark_out=/tmp/tmp5a5qjm
2019-03-06 21:53:31
Running ./llvm-cmov-bench-OLD
Run on (8 X 4000 MHz CPU s)
CPU Caches:
L1 Data 16K (x8)
L1 Instruction 64K (x4)
L2 Unified 2048K (x4)
L3 Unified 8192K (x1)
Load Average: 1.78, 1.81, 1.36
----------------------------------------------------------------------------------------------------
Benchmark Time CPU Iterations UserCounters<...>
----------------------------------------------------------------------------------------------------
<...>
BM_StdMidpoint<int32_t, RandRand>/131072 300398 ns 300404 ns 2330 bytes_read/iteration=1024k bytes_read/sec=3.25083G/s midpoints=305.398M midpoints/sec=436.319M/s
BM_StdMidpoint<int32_t, RandRand>_BigO 2.29 N 2.29 N
BM_StdMidpoint<int32_t, RandRand>_RMS 2 % 2 %
<...>
BM_StdMidpoint<uint32_t, RandRand>/131072 300433 ns 300433 ns 2330 bytes_read/iteration=1024k bytes_read/sec=3.25052G/s midpoints=305.398M midpoints/sec=436.278M/s
BM_StdMidpoint<uint32_t, RandRand>_BigO 2.29 N 2.29 N
BM_StdMidpoint<uint32_t, RandRand>_RMS 2 % 2 %
<...>
BM_StdMidpoint<int64_t, RandRand>/65536 169857 ns 169858 ns 4121 bytes_read/iteration=1024k bytes_read/sec=5.74929G/s midpoints=270.074M midpoints/sec=385.828M/s
BM_StdMidpoint<int64_t, RandRand>_BigO 2.59 N 2.59 N
BM_StdMidpoint<int64_t, RandRand>_RMS 3 % 3 %
<...>
BM_StdMidpoint<uint64_t, RandRand>/65536 169770 ns 169771 ns 4125 bytes_read/iteration=1024k bytes_read/sec=5.75223G/s midpoints=270.336M midpoints/sec=386.026M/s
BM_StdMidpoint<uint64_t, RandRand>_BigO 2.59 N 2.59 N
BM_StdMidpoint<uint64_t, RandRand>_RMS 3 % 3 %
<...>
BM_StdMidpoint<int16_t, RandRand>/262144 591169 ns 591179 ns 1182 bytes_read/iteration=1024k bytes_read/sec=1.65189G/s midpoints=309.854M midpoints/sec=443.426M/s
BM_StdMidpoint<int16_t, RandRand>_BigO 2.25 N 2.25 N
BM_StdMidpoint<int16_t, RandRand>_RMS 1 % 1 %
<...>
BM_StdMidpoint<uint16_t, RandRand>/262144 591264 ns 591274 ns 1184 bytes_read/iteration=1024k bytes_read/sec=1.65162G/s midpoints=310.378M midpoints/sec=443.354M/s
BM_StdMidpoint<uint16_t, RandRand>_BigO 2.25 N 2.25 N
BM_StdMidpoint<uint16_t, RandRand>_RMS 1 % 1 %
<...>
BM_StdMidpoint<int8_t, RandRand>/524288 2983669 ns 2983689 ns 235 bytes_read/iteration=1024k bytes_read/sec=335.156M/s midpoints=123.208M midpoints/sec=175.718M/s
BM_StdMidpoint<int8_t, RandRand>_BigO 5.69 N 5.69 N
BM_StdMidpoint<int8_t, RandRand>_RMS 0 % 0 %
<...>
BM_StdMidpoint<uint8_t, RandRand>/524288 2668398 ns 2668419 ns 262 bytes_read/iteration=1024k bytes_read/sec=374.754M/s midpoints=137.363M midpoints/sec=196.479M/s
BM_StdMidpoint<uint8_t, RandRand>_BigO 5.09 N 5.09 N
BM_StdMidpoint<uint8_t, RandRand>_RMS 0 % 0 %
<...>
BM_StdMidpoint<uint32_t, ZeroRand>/131072 300887 ns 300887 ns 2331 bytes_read/iteration=1024k bytes_read/sec=3.24561G/s midpoints=305.529M midpoints/sec=435.619M/s
BM_StdMidpoint<uint32_t, ZeroRand>_BigO 2.29 N 2.29 N
BM_StdMidpoint<uint32_t, ZeroRand>_RMS 2 % 2 %
<...>
BM_StdMidpoint<uint64_t, ZeroRand>/65536 169634 ns 169634 ns 4102 bytes_read/iteration=1024k bytes_read/sec=5.75688G/s midpoints=268.829M midpoints/sec=386.338M/s
BM_StdMidpoint<uint64_t, ZeroRand>_BigO 2.59 N 2.59 N
BM_StdMidpoint<uint64_t, ZeroRand>_RMS 3 % 3 %
<...>
BM_StdMidpoint<uint16_t, ZeroRand>/262144 592252 ns 592255 ns 1182 bytes_read/iteration=1024k bytes_read/sec=1.64889G/s midpoints=309.854M midpoints/sec=442.62M/s
BM_StdMidpoint<uint16_t, ZeroRand>_BigO 2.26 N 2.26 N
BM_StdMidpoint<uint16_t, ZeroRand>_RMS 1 % 1 %
<...>
BM_StdMidpoint<uint8_t, ZeroRand>/524288 987295 ns 987309 ns 711 bytes_read/iteration=1024k bytes_read/sec=1012.85M/s midpoints=372.769M midpoints/sec=531.028M/s
BM_StdMidpoint<uint8_t, ZeroRand>_BigO 1.88 N 1.88 N
BM_StdMidpoint<uint8_t, ZeroRand>_RMS 1 % 1 %
RUNNING: ./llvm-cmov-bench-NEW --benchmark_out=/tmp/tmpPvwpfW
2019-03-06 21:56:58
Running ./llvm-cmov-bench-NEW
Run on (8 X 4000 MHz CPU s)
CPU Caches:
L1 Data 16K (x8)
L1 Instruction 64K (x4)
L2 Unified 2048K (x4)
L3 Unified 8192K (x1)
Load Average: 1.17, 1.46, 1.30
----------------------------------------------------------------------------------------------------
Benchmark Time CPU Iterations UserCounters<...>
----------------------------------------------------------------------------------------------------
<...>
BM_StdMidpoint<int32_t, RandRand>/131072 300878 ns 300880 ns 2324 bytes_read/iteration=1024k bytes_read/sec=3.24569G/s midpoints=304.611M midpoints/sec=435.629M/s
BM_StdMidpoint<int32_t, RandRand>_BigO 2.29 N 2.29 N
BM_StdMidpoint<int32_t, RandRand>_RMS 2 % 2 %
<...>
BM_StdMidpoint<uint32_t, RandRand>/131072 300231 ns 300226 ns 2330 bytes_read/iteration=1024k bytes_read/sec=3.25276G/s midpoints=305.398M midpoints/sec=436.578M/s
BM_StdMidpoint<uint32_t, RandRand>_BigO 2.29 N 2.29 N
BM_StdMidpoint<uint32_t, RandRand>_RMS 2 % 2 %
<...>
BM_StdMidpoint<int64_t, RandRand>/65536 170819 ns 170777 ns 4115 bytes_read/iteration=1024k bytes_read/sec=5.71835G/s midpoints=269.681M midpoints/sec=383.752M/s
BM_StdMidpoint<int64_t, RandRand>_BigO 2.60 N 2.60 N
BM_StdMidpoint<int64_t, RandRand>_RMS 3 % 3 %
<...>
BM_StdMidpoint<uint64_t, RandRand>/65536 171705 ns 171708 ns 4106 bytes_read/iteration=1024k bytes_read/sec=5.68733G/s midpoints=269.091M midpoints/sec=381.671M/s
BM_StdMidpoint<uint64_t, RandRand>_BigO 2.62 N 2.62 N
BM_StdMidpoint<uint64_t, RandRand>_RMS 3 % 3 %
<...>
BM_StdMidpoint<int16_t, RandRand>/262144 592510 ns 592516 ns 1182 bytes_read/iteration=1024k bytes_read/sec=1.64816G/s midpoints=309.854M midpoints/sec=442.425M/s
BM_StdMidpoint<int16_t, RandRand>_BigO 2.26 N 2.26 N
BM_StdMidpoint<int16_t, RandRand>_RMS 1 % 1 %
<...>
BM_StdMidpoint<uint16_t, RandRand>/262144 614823 ns 614823 ns 1180 bytes_read/iteration=1024k bytes_read/sec=1.58836G/s midpoints=309.33M midpoints/sec=426.373M/s
BM_StdMidpoint<uint16_t, RandRand>_BigO 2.33 N 2.33 N
BM_StdMidpoint<uint16_t, RandRand>_RMS 4 % 4 %
<...>
BM_StdMidpoint<int8_t, RandRand>/524288 1073181 ns 1073201 ns 650 bytes_read/iteration=1024k bytes_read/sec=931.791M/s midpoints=340.787M midpoints/sec=488.527M/s
BM_StdMidpoint<int8_t, RandRand>_BigO 2.05 N 2.05 N
BM_StdMidpoint<int8_t, RandRand>_RMS 1 % 1 %
BM_StdMidpoint<uint8_t, RandRand>/524288 1071010 ns 1071020 ns 653 bytes_read/iteration=1024k bytes_read/sec=933.689M/s midpoints=342.36M midpoints/sec=489.522M/s
BM_StdMidpoint<uint8_t, RandRand>_BigO 2.05 N 2.05 N
BM_StdMidpoint<uint8_t, RandRand>_RMS 1 % 1 %
<...>
BM_StdMidpoint<uint32_t, ZeroRand>/131072 300413 ns 300416 ns 2330 bytes_read/iteration=1024k bytes_read/sec=3.2507G/s midpoints=305.398M midpoints/sec=436.302M/s
BM_StdMidpoint<uint32_t, ZeroRand>_BigO 2.29 N 2.29 N
BM_StdMidpoint<uint32_t, ZeroRand>_RMS 2 % 2 %
<...>
BM_StdMidpoint<uint64_t, ZeroRand>/65536 169667 ns 169669 ns 4123 bytes_read/iteration=1024k bytes_read/sec=5.75568G/s midpoints=270.205M midpoints/sec=386.257M/s
BM_StdMidpoint<uint64_t, ZeroRand>_BigO 2.59 N 2.59 N
BM_StdMidpoint<uint64_t, ZeroRand>_RMS 3 % 3 %
<...>
BM_StdMidpoint<uint16_t, ZeroRand>/262144 591396 ns 591404 ns 1184 bytes_read/iteration=1024k bytes_read/sec=1.65126G/s midpoints=310.378M midpoints/sec=443.257M/s
BM_StdMidpoint<uint16_t, ZeroRand>_BigO 2.26 N 2.26 N
BM_StdMidpoint<uint16_t, ZeroRand>_RMS 1 % 1 %
<...>
BM_StdMidpoint<uint8_t, ZeroRand>/524288 1069421 ns 1069413 ns 655 bytes_read/iteration=1024k bytes_read/sec=935.092M/s midpoints=343.409M midpoints/sec=490.258M/s
BM_StdMidpoint<uint8_t, ZeroRand>_BigO 2.04 N 2.04 N
BM_StdMidpoint<uint8_t, ZeroRand>_RMS 0 % 0 %
Comparing ./llvm-cmov-bench-OLD to ./llvm-cmov-bench-NEW
Benchmark Time CPU Time Old Time New CPU Old CPU New
----------------------------------------------------------------------------------------------------------------------------------------
<...>
BM_StdMidpoint<int32_t, RandRand>/131072 +0.0016 +0.0016 300398 300878 300404 300880
<...>
BM_StdMidpoint<uint32_t, RandRand>/131072 -0.0007 -0.0007 300433 300231 300433 300226
<...>
BM_StdMidpoint<int64_t, RandRand>/65536 +0.0057 +0.0054 169857 170819 169858 170777
<...>
BM_StdMidpoint<uint64_t, RandRand>/65536 +0.0114 +0.0114 169770 171705 169771 171708
<...>
BM_StdMidpoint<int16_t, RandRand>/262144 +0.0023 +0.0023 591169 592510 591179 592516
<...>
BM_StdMidpoint<uint16_t, RandRand>/262144 +0.0398 +0.0398 591264 614823 591274 614823
<...>
BM_StdMidpoint<int8_t, RandRand>/524288 -0.6403 -0.6403 2983669 1073181 2983689 1073201
<...>
BM_StdMidpoint<uint8_t, RandRand>/524288 -0.5986 -0.5986 2668398 1071010 2668419 1071020
<...>
BM_StdMidpoint<uint32_t, ZeroRand>/131072 -0.0016 -0.0016 300887 300413 300887 300416
<...>
BM_StdMidpoint<uint64_t, ZeroRand>/65536 +0.0002 +0.0002 169634 169667 169634 169669
<...>
BM_StdMidpoint<uint16_t, ZeroRand>/262144 -0.0014 -0.0014 592252 591396 592255 591404
<...>
BM_StdMidpoint<uint8_t, ZeroRand>/524288 +0.0832 +0.0832 987295 1069421 987309 1069413
```
What can we tell from the benchmark?
* `BM_StdMidpoint<[u]int8_t, RandRand>` indeed has the worst performance.
* All `BM_StdMidpoint<uint{8,16,32}_t, ZeroRand>` are all performant, even the 8-bit case.
That is because there we are computing mid point between zero and some random number,
thus if the branch predictor is in use, it is in optimal situation.
* Promoting 8-bit CMOV did improve performance of `BM_StdMidpoint<[u]int8_t, RandRand>`, by -59%..-64%.
# What about branch predictor?
* `BM_StdMidpoint<uint8_t, ZeroRand>` was faster than `BM_StdMidpoint<uint{16,32,64}_t, ZeroRand>`,
which may mean that well-predicted branch is better than `cmov`.
* Promoting 8-bit CMOV degraded performance of `BM_StdMidpoint<uint8_t, ZeroRand>`,
`cmov` is up to +10% worse than well-predicted branch.
* However, i do not believe this is a concern. If the branch is well predicted, then the PGO
will also say that it is well predicted, and LLVM will happily expand cmov back into branch:
https://godbolt.org/z/P5ufig
# What about partial register stalls?
I'm not really able to answer that.
What i can say is that if the branch is unpredictable (if it is predictable, then use PGO and you'll have branch)
in ~50% of cases you will have to pay branch misprediction penalty.
```
$ grep -i MispredictPenalty X86Sched*.td
X86SchedBroadwell.td: let MispredictPenalty = 16;
X86SchedHaswell.td: let MispredictPenalty = 16;
X86SchedSandyBridge.td: let MispredictPenalty = 16;
X86SchedSkylakeClient.td: let MispredictPenalty = 14;
X86SchedSkylakeServer.td: let MispredictPenalty = 14;
X86ScheduleBdVer2.td: let MispredictPenalty = 20; // Minimum branch misdirection penalty.
X86ScheduleBtVer2.td: let MispredictPenalty = 14; // Minimum branch misdirection penalty
X86ScheduleSLM.td: let MispredictPenalty = 10;
X86ScheduleZnver1.td: let MispredictPenalty = 17;
```
.. which it can be as small as 10 cycles and as large as 20 cycles.
Partial register stalls do not seem to be an issue for AMD CPU's.
For intel CPU's, they should be around ~5 cycles?
Is that actually an issue here? I'm not sure.
In short, i'd say this is an improvement, at least on this microbenchmark.
Fixes [[ https://bugs.llvm.org/show_bug.cgi?id=40965 | PR40965 ]].
Reviewers: craig.topper, RKSimon, spatel, andreadb, nikic
Reviewed By: craig.topper, andreadb
Subscribers: jfb, jdoerfert, llvm-commits, mclow.lists
Tags: #llvm, #libc
Differential Revision: https://reviews.llvm.org/D59035
llvm-svn: 356300
2019-03-16 05:17:53 +08:00
|
|
|
; CMOV-NEXT: movl $20, %eax
|
|
|
|
|
; CMOV-NEXT: cmovll %eax, %ecx
|
|
|
|
|
; CMOV-NEXT: cmovlel %ecx, %eax
|
2019-03-06 19:47:43 +08:00
|
|
|
; CMOV-NEXT: testl %edi, %edi
|
[X86] Promote i8 CMOV's (PR40965)
Summary:
@mclow.lists brought up this issue up in IRC, it came up during
implementation of libc++ `std::midpoint()` implementation (D59099)
https://godbolt.org/z/oLrHBP
Currently LLVM X86 backend only promotes i8 CMOV if it came from 2x`trunc`.
This differential proposes to always promote i8 CMOV.
There are several concerns here:
* Is this actually more performant, or is it just the ASM that looks cuter?
* Does this result in partial register stalls?
* What about branch predictor?
# Indeed, performance should be the main point here.
Let's look at a simple microbenchmark: {F8412076}
```
#include "benchmark/benchmark.h"
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <iterator>
#include <limits>
#include <random>
#include <type_traits>
#include <utility>
#include <vector>
// Future preliminary libc++ code, from Marshall Clow.
namespace std {
template <class _Tp>
__inline _Tp midpoint(_Tp __a, _Tp __b) noexcept {
using _Up = typename std::make_unsigned<typename remove_cv<_Tp>::type>::type;
int __sign = 1;
_Up __m = __a;
_Up __M = __b;
if (__a > __b) {
__sign = -1;
__m = __b;
__M = __a;
}
return __a + __sign * _Tp(_Up(__M - __m) >> 1);
}
} // namespace std
template <typename T>
std::vector<T> getVectorOfRandomNumbers(size_t count) {
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<T> dis(std::numeric_limits<T>::min(),
std::numeric_limits<T>::max());
std::vector<T> v;
v.reserve(count);
std::generate_n(std::back_inserter(v), count,
[&dis, &gen]() { return dis(gen); });
assert(v.size() == count);
return v;
}
struct RandRand {
template <typename T>
static std::pair<std::vector<T>, std::vector<T>> Gen(size_t count) {
return std::make_pair(getVectorOfRandomNumbers<T>(count),
getVectorOfRandomNumbers<T>(count));
}
};
struct ZeroRand {
template <typename T>
static std::pair<std::vector<T>, std::vector<T>> Gen(size_t count) {
return std::make_pair(std::vector<T>(count, T(0)),
getVectorOfRandomNumbers<T>(count));
}
};
template <class T, class Gen>
void BM_StdMidpoint(benchmark::State& state) {
const size_t Length = state.range(0);
const std::pair<std::vector<T>, std::vector<T>> Data =
Gen::template Gen<T>(Length);
const std::vector<T>& a = Data.first;
const std::vector<T>& b = Data.second;
assert(a.size() == Length && b.size() == a.size());
benchmark::ClobberMemory();
benchmark::DoNotOptimize(a);
benchmark::DoNotOptimize(a.data());
benchmark::DoNotOptimize(b);
benchmark::DoNotOptimize(b.data());
for (auto _ : state) {
for (size_t i = 0; i < Length; i++) {
const auto calculated = std::midpoint(a[i], b[i]);
benchmark::DoNotOptimize(calculated);
}
}
state.SetComplexityN(Length);
state.counters["midpoints"] =
benchmark::Counter(Length, benchmark::Counter::kIsIterationInvariant);
state.counters["midpoints/sec"] =
benchmark::Counter(Length, benchmark::Counter::kIsIterationInvariantRate);
const size_t BytesRead = 2 * sizeof(T) * Length;
state.counters["bytes_read/iteration"] =
benchmark::Counter(BytesRead, benchmark::Counter::kDefaults,
benchmark::Counter::OneK::kIs1024);
state.counters["bytes_read/sec"] = benchmark::Counter(
BytesRead, benchmark::Counter::kIsIterationInvariantRate,
benchmark::Counter::OneK::kIs1024);
}
template <typename T>
static void CustomArguments(benchmark::internal::Benchmark* b) {
const size_t L2SizeBytes = 2 * 1024 * 1024;
// What is the largest range we can check to always fit within given L2 cache?
const size_t MaxLen = L2SizeBytes / /*total bufs*/ 2 /
/*maximal elt size*/ sizeof(T) / /*safety margin*/ 2;
b->RangeMultiplier(2)->Range(1, MaxLen)->Complexity(benchmark::oN);
}
// Both of the values are random.
// The comparison is unpredictable.
BENCHMARK_TEMPLATE(BM_StdMidpoint, int32_t, RandRand)
->Apply(CustomArguments<int32_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint32_t, RandRand)
->Apply(CustomArguments<uint32_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, int64_t, RandRand)
->Apply(CustomArguments<int64_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint64_t, RandRand)
->Apply(CustomArguments<uint64_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, int16_t, RandRand)
->Apply(CustomArguments<int16_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint16_t, RandRand)
->Apply(CustomArguments<uint16_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, int8_t, RandRand)
->Apply(CustomArguments<int8_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint8_t, RandRand)
->Apply(CustomArguments<uint8_t>);
// One value is always zero, and another is bigger or equal than zero.
// The comparison is predictable.
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint32_t, ZeroRand)
->Apply(CustomArguments<uint32_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint64_t, ZeroRand)
->Apply(CustomArguments<uint64_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint16_t, ZeroRand)
->Apply(CustomArguments<uint16_t>);
BENCHMARK_TEMPLATE(BM_StdMidpoint, uint8_t, ZeroRand)
->Apply(CustomArguments<uint8_t>);
```
```
$ ~/src/googlebenchmark/tools/compare.py --no-utest benchmarks ./llvm-cmov-bench-OLD ./llvm-cmov-bench-NEW
RUNNING: ./llvm-cmov-bench-OLD --benchmark_out=/tmp/tmp5a5qjm
2019-03-06 21:53:31
Running ./llvm-cmov-bench-OLD
Run on (8 X 4000 MHz CPU s)
CPU Caches:
L1 Data 16K (x8)
L1 Instruction 64K (x4)
L2 Unified 2048K (x4)
L3 Unified 8192K (x1)
Load Average: 1.78, 1.81, 1.36
----------------------------------------------------------------------------------------------------
Benchmark Time CPU Iterations UserCounters<...>
----------------------------------------------------------------------------------------------------
<...>
BM_StdMidpoint<int32_t, RandRand>/131072 300398 ns 300404 ns 2330 bytes_read/iteration=1024k bytes_read/sec=3.25083G/s midpoints=305.398M midpoints/sec=436.319M/s
BM_StdMidpoint<int32_t, RandRand>_BigO 2.29 N 2.29 N
BM_StdMidpoint<int32_t, RandRand>_RMS 2 % 2 %
<...>
BM_StdMidpoint<uint32_t, RandRand>/131072 300433 ns 300433 ns 2330 bytes_read/iteration=1024k bytes_read/sec=3.25052G/s midpoints=305.398M midpoints/sec=436.278M/s
BM_StdMidpoint<uint32_t, RandRand>_BigO 2.29 N 2.29 N
BM_StdMidpoint<uint32_t, RandRand>_RMS 2 % 2 %
<...>
BM_StdMidpoint<int64_t, RandRand>/65536 169857 ns 169858 ns 4121 bytes_read/iteration=1024k bytes_read/sec=5.74929G/s midpoints=270.074M midpoints/sec=385.828M/s
BM_StdMidpoint<int64_t, RandRand>_BigO 2.59 N 2.59 N
BM_StdMidpoint<int64_t, RandRand>_RMS 3 % 3 %
<...>
BM_StdMidpoint<uint64_t, RandRand>/65536 169770 ns 169771 ns 4125 bytes_read/iteration=1024k bytes_read/sec=5.75223G/s midpoints=270.336M midpoints/sec=386.026M/s
BM_StdMidpoint<uint64_t, RandRand>_BigO 2.59 N 2.59 N
BM_StdMidpoint<uint64_t, RandRand>_RMS 3 % 3 %
<...>
BM_StdMidpoint<int16_t, RandRand>/262144 591169 ns 591179 ns 1182 bytes_read/iteration=1024k bytes_read/sec=1.65189G/s midpoints=309.854M midpoints/sec=443.426M/s
BM_StdMidpoint<int16_t, RandRand>_BigO 2.25 N 2.25 N
BM_StdMidpoint<int16_t, RandRand>_RMS 1 % 1 %
<...>
BM_StdMidpoint<uint16_t, RandRand>/262144 591264 ns 591274 ns 1184 bytes_read/iteration=1024k bytes_read/sec=1.65162G/s midpoints=310.378M midpoints/sec=443.354M/s
BM_StdMidpoint<uint16_t, RandRand>_BigO 2.25 N 2.25 N
BM_StdMidpoint<uint16_t, RandRand>_RMS 1 % 1 %
<...>
BM_StdMidpoint<int8_t, RandRand>/524288 2983669 ns 2983689 ns 235 bytes_read/iteration=1024k bytes_read/sec=335.156M/s midpoints=123.208M midpoints/sec=175.718M/s
BM_StdMidpoint<int8_t, RandRand>_BigO 5.69 N 5.69 N
BM_StdMidpoint<int8_t, RandRand>_RMS 0 % 0 %
<...>
BM_StdMidpoint<uint8_t, RandRand>/524288 2668398 ns 2668419 ns 262 bytes_read/iteration=1024k bytes_read/sec=374.754M/s midpoints=137.363M midpoints/sec=196.479M/s
BM_StdMidpoint<uint8_t, RandRand>_BigO 5.09 N 5.09 N
BM_StdMidpoint<uint8_t, RandRand>_RMS 0 % 0 %
<...>
BM_StdMidpoint<uint32_t, ZeroRand>/131072 300887 ns 300887 ns 2331 bytes_read/iteration=1024k bytes_read/sec=3.24561G/s midpoints=305.529M midpoints/sec=435.619M/s
BM_StdMidpoint<uint32_t, ZeroRand>_BigO 2.29 N 2.29 N
BM_StdMidpoint<uint32_t, ZeroRand>_RMS 2 % 2 %
<...>
BM_StdMidpoint<uint64_t, ZeroRand>/65536 169634 ns 169634 ns 4102 bytes_read/iteration=1024k bytes_read/sec=5.75688G/s midpoints=268.829M midpoints/sec=386.338M/s
BM_StdMidpoint<uint64_t, ZeroRand>_BigO 2.59 N 2.59 N
BM_StdMidpoint<uint64_t, ZeroRand>_RMS 3 % 3 %
<...>
BM_StdMidpoint<uint16_t, ZeroRand>/262144 592252 ns 592255 ns 1182 bytes_read/iteration=1024k bytes_read/sec=1.64889G/s midpoints=309.854M midpoints/sec=442.62M/s
BM_StdMidpoint<uint16_t, ZeroRand>_BigO 2.26 N 2.26 N
BM_StdMidpoint<uint16_t, ZeroRand>_RMS 1 % 1 %
<...>
BM_StdMidpoint<uint8_t, ZeroRand>/524288 987295 ns 987309 ns 711 bytes_read/iteration=1024k bytes_read/sec=1012.85M/s midpoints=372.769M midpoints/sec=531.028M/s
BM_StdMidpoint<uint8_t, ZeroRand>_BigO 1.88 N 1.88 N
BM_StdMidpoint<uint8_t, ZeroRand>_RMS 1 % 1 %
RUNNING: ./llvm-cmov-bench-NEW --benchmark_out=/tmp/tmpPvwpfW
2019-03-06 21:56:58
Running ./llvm-cmov-bench-NEW
Run on (8 X 4000 MHz CPU s)
CPU Caches:
L1 Data 16K (x8)
L1 Instruction 64K (x4)
L2 Unified 2048K (x4)
L3 Unified 8192K (x1)
Load Average: 1.17, 1.46, 1.30
----------------------------------------------------------------------------------------------------
Benchmark Time CPU Iterations UserCounters<...>
----------------------------------------------------------------------------------------------------
<...>
BM_StdMidpoint<int32_t, RandRand>/131072 300878 ns 300880 ns 2324 bytes_read/iteration=1024k bytes_read/sec=3.24569G/s midpoints=304.611M midpoints/sec=435.629M/s
BM_StdMidpoint<int32_t, RandRand>_BigO 2.29 N 2.29 N
BM_StdMidpoint<int32_t, RandRand>_RMS 2 % 2 %
<...>
BM_StdMidpoint<uint32_t, RandRand>/131072 300231 ns 300226 ns 2330 bytes_read/iteration=1024k bytes_read/sec=3.25276G/s midpoints=305.398M midpoints/sec=436.578M/s
BM_StdMidpoint<uint32_t, RandRand>_BigO 2.29 N 2.29 N
BM_StdMidpoint<uint32_t, RandRand>_RMS 2 % 2 %
<...>
BM_StdMidpoint<int64_t, RandRand>/65536 170819 ns 170777 ns 4115 bytes_read/iteration=1024k bytes_read/sec=5.71835G/s midpoints=269.681M midpoints/sec=383.752M/s
BM_StdMidpoint<int64_t, RandRand>_BigO 2.60 N 2.60 N
BM_StdMidpoint<int64_t, RandRand>_RMS 3 % 3 %
<...>
BM_StdMidpoint<uint64_t, RandRand>/65536 171705 ns 171708 ns 4106 bytes_read/iteration=1024k bytes_read/sec=5.68733G/s midpoints=269.091M midpoints/sec=381.671M/s
BM_StdMidpoint<uint64_t, RandRand>_BigO 2.62 N 2.62 N
BM_StdMidpoint<uint64_t, RandRand>_RMS 3 % 3 %
<...>
BM_StdMidpoint<int16_t, RandRand>/262144 592510 ns 592516 ns 1182 bytes_read/iteration=1024k bytes_read/sec=1.64816G/s midpoints=309.854M midpoints/sec=442.425M/s
BM_StdMidpoint<int16_t, RandRand>_BigO 2.26 N 2.26 N
BM_StdMidpoint<int16_t, RandRand>_RMS 1 % 1 %
<...>
BM_StdMidpoint<uint16_t, RandRand>/262144 614823 ns 614823 ns 1180 bytes_read/iteration=1024k bytes_read/sec=1.58836G/s midpoints=309.33M midpoints/sec=426.373M/s
BM_StdMidpoint<uint16_t, RandRand>_BigO 2.33 N 2.33 N
BM_StdMidpoint<uint16_t, RandRand>_RMS 4 % 4 %
<...>
BM_StdMidpoint<int8_t, RandRand>/524288 1073181 ns 1073201 ns 650 bytes_read/iteration=1024k bytes_read/sec=931.791M/s midpoints=340.787M midpoints/sec=488.527M/s
BM_StdMidpoint<int8_t, RandRand>_BigO 2.05 N 2.05 N
BM_StdMidpoint<int8_t, RandRand>_RMS 1 % 1 %
BM_StdMidpoint<uint8_t, RandRand>/524288 1071010 ns 1071020 ns 653 bytes_read/iteration=1024k bytes_read/sec=933.689M/s midpoints=342.36M midpoints/sec=489.522M/s
BM_StdMidpoint<uint8_t, RandRand>_BigO 2.05 N 2.05 N
BM_StdMidpoint<uint8_t, RandRand>_RMS 1 % 1 %
<...>
BM_StdMidpoint<uint32_t, ZeroRand>/131072 300413 ns 300416 ns 2330 bytes_read/iteration=1024k bytes_read/sec=3.2507G/s midpoints=305.398M midpoints/sec=436.302M/s
BM_StdMidpoint<uint32_t, ZeroRand>_BigO 2.29 N 2.29 N
BM_StdMidpoint<uint32_t, ZeroRand>_RMS 2 % 2 %
<...>
BM_StdMidpoint<uint64_t, ZeroRand>/65536 169667 ns 169669 ns 4123 bytes_read/iteration=1024k bytes_read/sec=5.75568G/s midpoints=270.205M midpoints/sec=386.257M/s
BM_StdMidpoint<uint64_t, ZeroRand>_BigO 2.59 N 2.59 N
BM_StdMidpoint<uint64_t, ZeroRand>_RMS 3 % 3 %
<...>
BM_StdMidpoint<uint16_t, ZeroRand>/262144 591396 ns 591404 ns 1184 bytes_read/iteration=1024k bytes_read/sec=1.65126G/s midpoints=310.378M midpoints/sec=443.257M/s
BM_StdMidpoint<uint16_t, ZeroRand>_BigO 2.26 N 2.26 N
BM_StdMidpoint<uint16_t, ZeroRand>_RMS 1 % 1 %
<...>
BM_StdMidpoint<uint8_t, ZeroRand>/524288 1069421 ns 1069413 ns 655 bytes_read/iteration=1024k bytes_read/sec=935.092M/s midpoints=343.409M midpoints/sec=490.258M/s
BM_StdMidpoint<uint8_t, ZeroRand>_BigO 2.04 N 2.04 N
BM_StdMidpoint<uint8_t, ZeroRand>_RMS 0 % 0 %
Comparing ./llvm-cmov-bench-OLD to ./llvm-cmov-bench-NEW
Benchmark Time CPU Time Old Time New CPU Old CPU New
----------------------------------------------------------------------------------------------------------------------------------------
<...>
BM_StdMidpoint<int32_t, RandRand>/131072 +0.0016 +0.0016 300398 300878 300404 300880
<...>
BM_StdMidpoint<uint32_t, RandRand>/131072 -0.0007 -0.0007 300433 300231 300433 300226
<...>
BM_StdMidpoint<int64_t, RandRand>/65536 +0.0057 +0.0054 169857 170819 169858 170777
<...>
BM_StdMidpoint<uint64_t, RandRand>/65536 +0.0114 +0.0114 169770 171705 169771 171708
<...>
BM_StdMidpoint<int16_t, RandRand>/262144 +0.0023 +0.0023 591169 592510 591179 592516
<...>
BM_StdMidpoint<uint16_t, RandRand>/262144 +0.0398 +0.0398 591264 614823 591274 614823
<...>
BM_StdMidpoint<int8_t, RandRand>/524288 -0.6403 -0.6403 2983669 1073181 2983689 1073201
<...>
BM_StdMidpoint<uint8_t, RandRand>/524288 -0.5986 -0.5986 2668398 1071010 2668419 1071020
<...>
BM_StdMidpoint<uint32_t, ZeroRand>/131072 -0.0016 -0.0016 300887 300413 300887 300416
<...>
BM_StdMidpoint<uint64_t, ZeroRand>/65536 +0.0002 +0.0002 169634 169667 169634 169669
<...>
BM_StdMidpoint<uint16_t, ZeroRand>/262144 -0.0014 -0.0014 592252 591396 592255 591404
<...>
BM_StdMidpoint<uint8_t, ZeroRand>/524288 +0.0832 +0.0832 987295 1069421 987309 1069413
```
What can we tell from the benchmark?
* `BM_StdMidpoint<[u]int8_t, RandRand>` indeed has the worst performance.
* All `BM_StdMidpoint<uint{8,16,32}_t, ZeroRand>` are all performant, even the 8-bit case.
That is because there we are computing mid point between zero and some random number,
thus if the branch predictor is in use, it is in optimal situation.
* Promoting 8-bit CMOV did improve performance of `BM_StdMidpoint<[u]int8_t, RandRand>`, by -59%..-64%.
# What about branch predictor?
* `BM_StdMidpoint<uint8_t, ZeroRand>` was faster than `BM_StdMidpoint<uint{16,32,64}_t, ZeroRand>`,
which may mean that well-predicted branch is better than `cmov`.
* Promoting 8-bit CMOV degraded performance of `BM_StdMidpoint<uint8_t, ZeroRand>`,
`cmov` is up to +10% worse than well-predicted branch.
* However, i do not believe this is a concern. If the branch is well predicted, then the PGO
will also say that it is well predicted, and LLVM will happily expand cmov back into branch:
https://godbolt.org/z/P5ufig
# What about partial register stalls?
I'm not really able to answer that.
What i can say is that if the branch is unpredictable (if it is predictable, then use PGO and you'll have branch)
in ~50% of cases you will have to pay branch misprediction penalty.
```
$ grep -i MispredictPenalty X86Sched*.td
X86SchedBroadwell.td: let MispredictPenalty = 16;
X86SchedHaswell.td: let MispredictPenalty = 16;
X86SchedSandyBridge.td: let MispredictPenalty = 16;
X86SchedSkylakeClient.td: let MispredictPenalty = 14;
X86SchedSkylakeServer.td: let MispredictPenalty = 14;
X86ScheduleBdVer2.td: let MispredictPenalty = 20; // Minimum branch misdirection penalty.
X86ScheduleBtVer2.td: let MispredictPenalty = 14; // Minimum branch misdirection penalty
X86ScheduleSLM.td: let MispredictPenalty = 10;
X86ScheduleZnver1.td: let MispredictPenalty = 17;
```
.. which it can be as small as 10 cycles and as large as 20 cycles.
Partial register stalls do not seem to be an issue for AMD CPU's.
For intel CPU's, they should be around ~5 cycles?
Is that actually an issue here? I'm not sure.
In short, i'd say this is an improvement, at least on this microbenchmark.
Fixes [[ https://bugs.llvm.org/show_bug.cgi?id=40965 | PR40965 ]].
Reviewers: craig.topper, RKSimon, spatel, andreadb, nikic
Reviewed By: craig.topper, andreadb
Subscribers: jfb, jdoerfert, llvm-commits, mclow.lists
Tags: #llvm, #libc
Differential Revision: https://reviews.llvm.org/D59035
llvm-svn: 356300
2019-03-16 05:17:53 +08:00
|
|
|
; CMOV-NEXT: cmovnel %ecx, %eax
|
2019-03-06 19:47:43 +08:00
|
|
|
; CMOV-NEXT: movb %al, {{.*}}(%rip)
|
|
|
|
|
; CMOV-NEXT: retq
|
|
|
|
|
;
|
|
|
|
|
; NOCMOV-LABEL: no_cascade_opt:
|
|
|
|
|
; NOCMOV: # %bb.0: # %entry
|
|
|
|
|
; NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
|
|
|
|
|
; NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %eax
|
|
|
|
|
; NOCMOV-NEXT: movb $20, %al
|
|
|
|
|
; NOCMOV-NEXT: movb $20, %cl
|
|
|
|
|
; NOCMOV-NEXT: jge .LBB7_1
|
|
|
|
|
; NOCMOV-NEXT: # %bb.2: # %entry
|
|
|
|
|
; NOCMOV-NEXT: jle .LBB7_3
|
|
|
|
|
; NOCMOV-NEXT: .LBB7_4: # %entry
|
|
|
|
|
; NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: jne .LBB7_5
|
|
|
|
|
; NOCMOV-NEXT: .LBB7_6: # %entry
|
|
|
|
|
; NOCMOV-NEXT: movb %al, g8
|
|
|
|
|
; NOCMOV-NEXT: retl
|
|
|
|
|
; NOCMOV-NEXT: .LBB7_1: # %entry
|
|
|
|
|
; NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %cl
|
|
|
|
|
; NOCMOV-NEXT: jg .LBB7_4
|
|
|
|
|
; NOCMOV-NEXT: .LBB7_3: # %entry
|
|
|
|
|
; NOCMOV-NEXT: movl %ecx, %eax
|
|
|
|
|
; NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
|
|
|
|
|
; NOCMOV-NEXT: je .LBB7_6
|
|
|
|
|
; NOCMOV-NEXT: .LBB7_5: # %entry
|
|
|
|
|
; NOCMOV-NEXT: movl %ecx, %eax
|
|
|
|
|
; NOCMOV-NEXT: movb %al, g8
|
|
|
|
|
; NOCMOV-NEXT: retl
|
2016-01-26 06:08:25 +08:00
|
|
|
entry:
|
|
|
|
|
%c0 = icmp eq i32 %v0, 0
|
|
|
|
|
%c1 = icmp slt i32 %v1, %v2
|
|
|
|
|
%c2 = icmp sgt i32 %v1, %v2
|
|
|
|
|
%trunc = trunc i32 %v3 to i8
|
|
|
|
|
%sel0 = select i1 %c1, i8 20, i8 %trunc
|
|
|
|
|
%sel1 = select i1 %c2, i8 20, i8 %sel0
|
|
|
|
|
%sel2 = select i1 %c0, i8 %sel1, i8 %sel0
|
|
|
|
|
store volatile i8 %sel2, i8* @g8
|
|
|
|
|
ret void
|
|
|
|
|
}
|