llvm-project/llvm/test/CodeGen/X86/avx512-cvt.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl  | FileCheck %s --check-prefix=ALL --check-prefix=NOVL --check-prefix=NODQ --check-prefix=NOVLDQ --check-prefix=KNL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx  | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=DQ --check-prefix=VL --check-prefix=VLDQ --check-prefix=VLBW --check-prefix=SKX
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl  | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=NODQ --check-prefix=VL --check-prefix=VLNODQ --check-prefix=VLNOBW --check-prefix=AVX512VL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512dq  | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=NOVL --check-prefix=DQ --check-prefix=AVX512DQ
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512bw  | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=NOVL --check-prefix=NODQ --check-prefix=NOVLDQ --check-prefix=AVX512BW
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl,avx512dq  | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=DQ --check-prefix=VL --check-prefix=VLDQ --check-prefix=VLNOBW --check-prefix=AVX512VLDQ
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl,avx512bw  | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=NODQ --check-prefix=VL --check-prefix=VLNODQ --check-prefix=VLBW --check-prefix=AVX512VLBW


define <16 x float> @sitof32(<16 x i32> %a) nounwind {
; ALL-LABEL: sitof32:
; ALL:       # BB#0:
; ALL-NEXT:    vcvtdq2ps %zmm0, %zmm0
; ALL-NEXT:    retq
  %b = sitofp <16 x i32> %a to <16 x float>
  ret <16 x float> %b
}

define <8 x double> @sltof864(<8 x i64> %a) {
; NODQ-LABEL: sltof864:
; NODQ:       # BB#0:
; NODQ-NEXT:    vextracti32x4 $3, %zmm0, %xmm1
; NODQ-NEXT:    vpextrq $1, %xmm1, %rax
; NODQ-NEXT:    vcvtsi2sdq %rax, %xmm2, %xmm2
; NODQ-NEXT:    vmovq %xmm1, %rax
; NODQ-NEXT:    vcvtsi2sdq %rax, %xmm3, %xmm1
; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm2
; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
; NODQ-NEXT:    vcvtsi2sdq %rax, %xmm3, %xmm3
; NODQ-NEXT:    vmovq %xmm2, %rax
; NODQ-NEXT:    vcvtsi2sdq %rax, %xmm4, %xmm2
; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
; NODQ-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; NODQ-NEXT:    vextracti32x4 $1, %zmm0, %xmm2
; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
; NODQ-NEXT:    vcvtsi2sdq %rax, %xmm4, %xmm3
; NODQ-NEXT:    vmovq %xmm2, %rax
; NODQ-NEXT:    vcvtsi2sdq %rax, %xmm4, %xmm2
; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
; NODQ-NEXT:    vcvtsi2sdq %rax, %xmm4, %xmm3
; NODQ-NEXT:    vmovq %xmm0, %rax
; NODQ-NEXT:    vcvtsi2sdq %rax, %xmm4, %xmm0
; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm3[0]
; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
; NODQ-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
; NODQ-NEXT:    retq
;
; DQ-LABEL: sltof864:
; DQ:       # BB#0:
; DQ-NEXT:    vcvtqq2pd %zmm0, %zmm0
; DQ-NEXT:    retq
  %b = sitofp <8 x i64> %a to <8 x double>
  ret <8 x double> %b
}

define <4 x double> @sltof464(<4 x i64> %a) {
; NODQ-LABEL: sltof464:
; NODQ:       # BB#0:
; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
; NODQ-NEXT:    vpextrq $1, %xmm1, %rax
; NODQ-NEXT:    vcvtsi2sdq %rax, %xmm2, %xmm2
; NODQ-NEXT:    vmovq %xmm1, %rax
; NODQ-NEXT:    vcvtsi2sdq %rax, %xmm3, %xmm1
; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
; NODQ-NEXT:    vcvtsi2sdq %rax, %xmm3, %xmm2
; NODQ-NEXT:    vmovq %xmm0, %rax
; NODQ-NEXT:    vcvtsi2sdq %rax, %xmm3, %xmm0
; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; NODQ-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; NODQ-NEXT:    retq
;
; VLDQ-LABEL: sltof464:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vcvtqq2pd %ymm0, %ymm0
; VLDQ-NEXT:    retq
;
; AVX512DQ-LABEL: sltof464:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
; AVX512DQ-NEXT:    vcvtqq2pd %zmm0, %zmm0
; AVX512DQ-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
; AVX512DQ-NEXT:    retq
  %b = sitofp <4 x i64> %a to <4 x double>
  ret <4 x double> %b
}

define <2 x float> @sltof2f32(<2 x i64> %a) {
; NODQ-LABEL: sltof2f32:
; NODQ:       # BB#0:
; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
; NODQ-NEXT:    vcvtsi2ssq %rax, %xmm1, %xmm1
; NODQ-NEXT:    vmovq %xmm0, %rax
; NODQ-NEXT:    vcvtsi2ssq %rax, %xmm2, %xmm0
; NODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
; NODQ-NEXT:    vcvtsi2ssq %rax, %xmm2, %xmm1
; NODQ-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,0]
; NODQ-NEXT:    retq
;
; VLDQ-LABEL: sltof2f32:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vcvtqq2ps %xmm0, %xmm0
; VLDQ-NEXT:    retq
;
; AVX512DQ-LABEL: sltof2f32:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<def>
; AVX512DQ-NEXT:    vcvtqq2ps %zmm0, %ymm0
; AVX512DQ-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
; AVX512DQ-NEXT:    vzeroupper
; AVX512DQ-NEXT:    retq
  %b = sitofp <2 x i64> %a to <2 x float>
  ret <2 x float>%b
}

define <4 x float> @sltof4f32_mem(<4 x i64>* %a) {
; KNL-LABEL: sltof4f32_mem:
; KNL:       # BB#0:
; KNL-NEXT:    vmovdqu (%rdi), %ymm0
; KNL-NEXT:    vpextrq $1, %xmm0, %rax
; KNL-NEXT:    vcvtsi2ssq %rax, %xmm1, %xmm1
; KNL-NEXT:    vmovq %xmm0, %rax
; KNL-NEXT:    vcvtsi2ssq %rax, %xmm2, %xmm2
; KNL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
; KNL-NEXT:    vmovq %xmm0, %rax
; KNL-NEXT:    vcvtsi2ssq %rax, %xmm3, %xmm2
; KNL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
; KNL-NEXT:    vpextrq $1, %xmm0, %rax
; KNL-NEXT:    vcvtsi2ssq %rax, %xmm3, %xmm0
; KNL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
; KNL-NEXT:    retq
;
; VLDQ-LABEL: sltof4f32_mem:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vcvtqq2psy (%rdi), %xmm0
; VLDQ-NEXT:    retq
;
; VLNODQ-LABEL: sltof4f32_mem:
; VLNODQ:       # BB#0:
; VLNODQ-NEXT:    vmovdqu (%rdi), %ymm0
; VLNODQ-NEXT:    vpextrq $1, %xmm0, %rax
; VLNODQ-NEXT:    vcvtsi2ssq %rax, %xmm1, %xmm1
; VLNODQ-NEXT:    vmovq %xmm0, %rax
; VLNODQ-NEXT:    vcvtsi2ssq %rax, %xmm2, %xmm2
; VLNODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
; VLNODQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
; VLNODQ-NEXT:    vmovq %xmm0, %rax
; VLNODQ-NEXT:    vcvtsi2ssq %rax, %xmm3, %xmm2
; VLNODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
; VLNODQ-NEXT:    vpextrq $1, %xmm0, %rax
; VLNODQ-NEXT:    vcvtsi2ssq %rax, %xmm3, %xmm0
; VLNODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
; VLNODQ-NEXT:    vzeroupper
; VLNODQ-NEXT:    retq
;
; AVX512DQ-LABEL: sltof4f32_mem:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    vmovups (%rdi), %ymm0
; AVX512DQ-NEXT:    vcvtqq2ps %zmm0, %ymm0
; AVX512DQ-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
; AVX512DQ-NEXT:    vzeroupper
; AVX512DQ-NEXT:    retq
;
; AVX512BW-LABEL: sltof4f32_mem:
; AVX512BW:       # BB#0:
; AVX512BW-NEXT:    vmovdqu (%rdi), %ymm0
; AVX512BW-NEXT:    vpextrq $1, %xmm0, %rax
; AVX512BW-NEXT:    vcvtsi2ssq %rax, %xmm1, %xmm1
; AVX512BW-NEXT:    vmovq %xmm0, %rax
; AVX512BW-NEXT:    vcvtsi2ssq %rax, %xmm2, %xmm2
; AVX512BW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm0
; AVX512BW-NEXT:    vmovq %xmm0, %rax
; AVX512BW-NEXT:    vcvtsi2ssq %rax, %xmm3, %xmm2
; AVX512BW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
; AVX512BW-NEXT:    vpextrq $1, %xmm0, %rax
; AVX512BW-NEXT:    vcvtsi2ssq %rax, %xmm3, %xmm0
; AVX512BW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
; AVX512BW-NEXT:    vzeroupper
; AVX512BW-NEXT:    retq
  %a1 = load <4 x i64>, <4 x i64>* %a, align 8
  %b = sitofp <4 x i64> %a1 to <4 x float>
  ret <4 x float>%b
}

define <4 x i64> @f64tosl(<4 x double> %a) {
; NODQ-LABEL: f64tosl:
; NODQ:       # BB#0:
; NODQ-NEXT:    vextractf128 $1, %ymm0, %xmm1
; NODQ-NEXT:    vcvttsd2si %xmm1, %rax
; NODQ-NEXT:    vmovq %rax, %xmm2
; NODQ-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
; NODQ-NEXT:    vcvttsd2si %xmm1, %rax
; NODQ-NEXT:    vmovq %rax, %xmm1
; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; NODQ-NEXT:    vcvttsd2si %xmm0, %rax
; NODQ-NEXT:    vmovq %rax, %xmm2
; NODQ-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
; NODQ-NEXT:    vcvttsd2si %xmm0, %rax
; NODQ-NEXT:    vmovq %rax, %xmm0
; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
; NODQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
; NODQ-NEXT:    retq
;
; VLDQ-LABEL: f64tosl:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vcvttpd2qq %ymm0, %ymm0
; VLDQ-NEXT:    retq
;
; AVX512DQ-LABEL: f64tosl:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
; AVX512DQ-NEXT:    vcvttpd2qq %zmm0, %zmm0
; AVX512DQ-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
; AVX512DQ-NEXT:    retq
  %b = fptosi <4 x double> %a to <4 x i64>
  ret <4 x i64> %b
}

define <4 x i64> @f32tosl(<4 x float> %a) {
; NODQ-LABEL: f32tosl:
; NODQ:       # BB#0:
; NODQ-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
; NODQ-NEXT:    vcvttss2si %xmm1, %rax
; NODQ-NEXT:    vmovq %rax, %xmm1
; NODQ-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
; NODQ-NEXT:    vcvttss2si %xmm2, %rax
; NODQ-NEXT:    vmovq %rax, %xmm2
; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; NODQ-NEXT:    vcvttss2si %xmm0, %rax
; NODQ-NEXT:    vmovq %rax, %xmm2
; NODQ-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
; NODQ-NEXT:    vcvttss2si %xmm0, %rax
; NODQ-NEXT:    vmovq %rax, %xmm0
; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
; NODQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
; NODQ-NEXT:    retq
;
; VLDQ-LABEL: f32tosl:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vcvttps2qq %xmm0, %ymm0
; VLDQ-NEXT:    retq
;
; AVX512DQ-LABEL: f32tosl:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<def>
; AVX512DQ-NEXT:    vcvttps2qq %ymm0, %zmm0
; AVX512DQ-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
; AVX512DQ-NEXT:    retq
  %b = fptosi <4 x float> %a to <4 x i64>
  ret <4 x i64> %b
}

define <4 x float> @sltof432(<4 x i64> %a) {
; KNL-LABEL: sltof432:
; KNL:       # BB#0:
; KNL-NEXT:    vpextrq $1, %xmm0, %rax
; KNL-NEXT:    vcvtsi2ssq %rax, %xmm1, %xmm1
; KNL-NEXT:    vmovq %xmm0, %rax
; KNL-NEXT:    vcvtsi2ssq %rax, %xmm2, %xmm2
; KNL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
; KNL-NEXT:    vmovq %xmm0, %rax
; KNL-NEXT:    vcvtsi2ssq %rax, %xmm3, %xmm2
; KNL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
; KNL-NEXT:    vpextrq $1, %xmm0, %rax
; KNL-NEXT:    vcvtsi2ssq %rax, %xmm3, %xmm0
; KNL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
; KNL-NEXT:    retq
;
; VLDQ-LABEL: sltof432:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vcvtqq2ps %ymm0, %xmm0
; VLDQ-NEXT:    vzeroupper
; VLDQ-NEXT:    retq
;
; VLNODQ-LABEL: sltof432:
; VLNODQ:       # BB#0:
; VLNODQ-NEXT:    vpextrq $1, %xmm0, %rax
; VLNODQ-NEXT:    vcvtsi2ssq %rax, %xmm1, %xmm1
; VLNODQ-NEXT:    vmovq %xmm0, %rax
; VLNODQ-NEXT:    vcvtsi2ssq %rax, %xmm2, %xmm2
; VLNODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
; VLNODQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
; VLNODQ-NEXT:    vmovq %xmm0, %rax
; VLNODQ-NEXT:    vcvtsi2ssq %rax, %xmm3, %xmm2
; VLNODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
; VLNODQ-NEXT:    vpextrq $1, %xmm0, %rax
; VLNODQ-NEXT:    vcvtsi2ssq %rax, %xmm3, %xmm0
; VLNODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
; VLNODQ-NEXT:    vzeroupper
; VLNODQ-NEXT:    retq
;
; AVX512DQ-LABEL: sltof432:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
; AVX512DQ-NEXT:    vcvtqq2ps %zmm0, %ymm0
; AVX512DQ-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
; AVX512DQ-NEXT:    vzeroupper
; AVX512DQ-NEXT:    retq
;
; AVX512BW-LABEL: sltof432:
; AVX512BW:       # BB#0:
; AVX512BW-NEXT:    vpextrq $1, %xmm0, %rax
; AVX512BW-NEXT:    vcvtsi2ssq %rax, %xmm1, %xmm1
; AVX512BW-NEXT:    vmovq %xmm0, %rax
; AVX512BW-NEXT:    vcvtsi2ssq %rax, %xmm2, %xmm2
; AVX512BW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm0
; AVX512BW-NEXT:    vmovq %xmm0, %rax
; AVX512BW-NEXT:    vcvtsi2ssq %rax, %xmm3, %xmm2
; AVX512BW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
; AVX512BW-NEXT:    vpextrq $1, %xmm0, %rax
; AVX512BW-NEXT:    vcvtsi2ssq %rax, %xmm3, %xmm0
; AVX512BW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
; AVX512BW-NEXT:    vzeroupper
; AVX512BW-NEXT:    retq
  %b = sitofp <4 x i64> %a to <4 x float>
  ret <4 x float> %b
}

define <4 x float> @ultof432(<4 x i64> %a) {
; KNL-LABEL: ultof432:
; KNL:       # BB#0:
; KNL-NEXT:    vpextrq $1, %xmm0, %rax
; KNL-NEXT:    vcvtusi2ssq %rax, %xmm1, %xmm1
; KNL-NEXT:    vmovq %xmm0, %rax
; KNL-NEXT:    vcvtusi2ssq %rax, %xmm2, %xmm2
; KNL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
; KNL-NEXT:    vmovq %xmm0, %rax
; KNL-NEXT:    vcvtusi2ssq %rax, %xmm3, %xmm2
; KNL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
; KNL-NEXT:    vpextrq $1, %xmm0, %rax
; KNL-NEXT:    vcvtusi2ssq %rax, %xmm3, %xmm0
; KNL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
; KNL-NEXT:    retq
;
; VLDQ-LABEL: ultof432:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vcvtuqq2ps %ymm0, %xmm0
; VLDQ-NEXT:    vzeroupper
; VLDQ-NEXT:    retq
;
; VLNODQ-LABEL: ultof432:
; VLNODQ:       # BB#0:
; VLNODQ-NEXT:    vpextrq $1, %xmm0, %rax
; VLNODQ-NEXT:    vcvtusi2ssq %rax, %xmm1, %xmm1
; VLNODQ-NEXT:    vmovq %xmm0, %rax
; VLNODQ-NEXT:    vcvtusi2ssq %rax, %xmm2, %xmm2
; VLNODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
; VLNODQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
; VLNODQ-NEXT:    vmovq %xmm0, %rax
; VLNODQ-NEXT:    vcvtusi2ssq %rax, %xmm3, %xmm2
; VLNODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
; VLNODQ-NEXT:    vpextrq $1, %xmm0, %rax
; VLNODQ-NEXT:    vcvtusi2ssq %rax, %xmm3, %xmm0
; VLNODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
; VLNODQ-NEXT:    vzeroupper
; VLNODQ-NEXT:    retq
;
; AVX512DQ-LABEL: ultof432:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
; AVX512DQ-NEXT:    vcvtuqq2ps %zmm0, %ymm0
; AVX512DQ-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
; AVX512DQ-NEXT:    vzeroupper
; AVX512DQ-NEXT:    retq
;
; AVX512BW-LABEL: ultof432:
; AVX512BW:       # BB#0:
; AVX512BW-NEXT:    vpextrq $1, %xmm0, %rax
; AVX512BW-NEXT:    vcvtusi2ssq %rax, %xmm1, %xmm1
; AVX512BW-NEXT:    vmovq %xmm0, %rax
; AVX512BW-NEXT:    vcvtusi2ssq %rax, %xmm2, %xmm2
; AVX512BW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm0
; AVX512BW-NEXT:    vmovq %xmm0, %rax
; AVX512BW-NEXT:    vcvtusi2ssq %rax, %xmm3, %xmm2
; AVX512BW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
; AVX512BW-NEXT:    vpextrq $1, %xmm0, %rax
; AVX512BW-NEXT:    vcvtusi2ssq %rax, %xmm3, %xmm0
; AVX512BW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
; AVX512BW-NEXT:    vzeroupper
; AVX512BW-NEXT:    retq
  %b = uitofp <4 x i64> %a to <4 x float>
  ret <4 x float> %b
}

define <8 x double> @ultof64(<8 x i64> %a) {
; NODQ-LABEL: ultof64:
; NODQ:       # BB#0:
; NODQ-NEXT:    vextracti32x4 $3, %zmm0, %xmm1
; NODQ-NEXT:    vpextrq $1, %xmm1, %rax
; NODQ-NEXT:    vcvtusi2sdq %rax, %xmm2, %xmm2
; NODQ-NEXT:    vmovq %xmm1, %rax
; NODQ-NEXT:    vcvtusi2sdq %rax, %xmm3, %xmm1
; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm2
; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
; NODQ-NEXT:    vcvtusi2sdq %rax, %xmm3, %xmm3
; NODQ-NEXT:    vmovq %xmm2, %rax
; NODQ-NEXT:    vcvtusi2sdq %rax, %xmm4, %xmm2
; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
; NODQ-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; NODQ-NEXT:    vextracti32x4 $1, %zmm0, %xmm2
; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
; NODQ-NEXT:    vcvtusi2sdq %rax, %xmm4, %xmm3
; NODQ-NEXT:    vmovq %xmm2, %rax
; NODQ-NEXT:    vcvtusi2sdq %rax, %xmm4, %xmm2
; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
; NODQ-NEXT:    vcvtusi2sdq %rax, %xmm4, %xmm3
; NODQ-NEXT:    vmovq %xmm0, %rax
; NODQ-NEXT:    vcvtusi2sdq %rax, %xmm4, %xmm0
; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm3[0]
; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
; NODQ-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
; NODQ-NEXT:    retq
;
; DQ-LABEL: ultof64:
; DQ:       # BB#0:
; DQ-NEXT:    vcvtuqq2pd %zmm0, %zmm0
; DQ-NEXT:    retq
  %b = uitofp <8 x i64> %a to <8 x double>
  ret <8 x double> %b
}

define <16 x i32> @fptosi00(<16 x float> %a) nounwind {
; ALL-LABEL: fptosi00:
; ALL:       # BB#0:
; ALL-NEXT:    vcvttps2dq %zmm0, %zmm0
; ALL-NEXT:    retq
  %b = fptosi <16 x float> %a to <16 x i32>
  ret <16 x i32> %b
}

define <16 x i32> @fptoui00(<16 x float> %a) nounwind {
; ALL-LABEL: fptoui00:
; ALL:       # BB#0:
; ALL-NEXT:    vcvttps2udq %zmm0, %zmm0
; ALL-NEXT:    retq
  %b = fptoui <16 x float> %a to <16 x i32>
  ret <16 x i32> %b
}

define <8 x i32> @fptoui_256(<8 x float> %a) nounwind {
; NOVL-LABEL: fptoui_256:
; NOVL:       # BB#0:
; NOVL-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
; NOVL-NEXT:    vcvttps2udq %zmm0, %zmm0
; NOVL-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
; NOVL-NEXT:    retq
;
; VL-LABEL: fptoui_256:
; VL:       # BB#0:
; VL-NEXT:    vcvttps2udq %ymm0, %ymm0
; VL-NEXT:    retq
  %b = fptoui <8 x float> %a to <8 x i32>
  ret <8 x i32> %b
}

define <4 x i32> @fptoui_128(<4 x float> %a) nounwind {
; KNL-LABEL: fptoui_128:
; KNL:       # BB#0:
; KNL-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<def>
; KNL-NEXT:    vcvttps2udq %zmm0, %zmm0
; KNL-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<kill>
; KNL-NEXT:    retq
;
; VL-LABEL: fptoui_128:
; VL:       # BB#0:
; VL-NEXT:    vcvttps2udq %xmm0, %xmm0
; VL-NEXT:    retq
;
; AVX512DQ-LABEL: fptoui_128:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<def>
; AVX512DQ-NEXT:    vcvttps2udq %zmm0, %zmm0
; AVX512DQ-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<kill>
; AVX512DQ-NEXT:    vzeroupper
; AVX512DQ-NEXT:    retq
;
; AVX512BW-LABEL: fptoui_128:
; AVX512BW:       # BB#0:
; AVX512BW-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<def>
; AVX512BW-NEXT:    vcvttps2udq %zmm0, %zmm0
; AVX512BW-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<kill>
; AVX512BW-NEXT:    vzeroupper
; AVX512BW-NEXT:    retq
  %b = fptoui <4 x float> %a to <4 x i32>
  ret <4 x i32> %b
}

define <8 x i32> @fptoui01(<8 x double> %a) nounwind {
; ALL-LABEL: fptoui01:
; ALL:       # BB#0:
; ALL-NEXT:    vcvttpd2udq %zmm0, %ymm0
; ALL-NEXT:    retq
  %b = fptoui <8 x double> %a to <8 x i32>
  ret <8 x i32> %b
}

define <4 x i32> @fptoui_256d(<4 x double> %a) nounwind {
; KNL-LABEL: fptoui_256d:
; KNL:       # BB#0:
; KNL-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
; KNL-NEXT:    vcvttpd2udq %zmm0, %ymm0
; KNL-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
; KNL-NEXT:    retq
;
; VL-LABEL: fptoui_256d:
; VL:       # BB#0:
; VL-NEXT:    vcvttpd2udq %ymm0, %xmm0
; VL-NEXT:    vzeroupper
; VL-NEXT:    retq
;
; AVX512DQ-LABEL: fptoui_256d:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
; AVX512DQ-NEXT:    vcvttpd2udq %zmm0, %ymm0
; AVX512DQ-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
; AVX512DQ-NEXT:    vzeroupper
; AVX512DQ-NEXT:    retq
;
; AVX512BW-LABEL: fptoui_256d:
; AVX512BW:       # BB#0:
; AVX512BW-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
; AVX512BW-NEXT:    vcvttpd2udq %zmm0, %ymm0
; AVX512BW-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
; AVX512BW-NEXT:    vzeroupper
; AVX512BW-NEXT:    retq
  %b = fptoui <4 x double> %a to <4 x i32>
  ret <4 x i32> %b
}

define <8 x double> @sitof64(<8 x i32> %a) {
; ALL-LABEL: sitof64:
; ALL:       # BB#0:
; ALL-NEXT:    vcvtdq2pd %ymm0, %zmm0
; ALL-NEXT:    retq
  %b = sitofp <8 x i32> %a to <8 x double>
  ret <8 x double> %b
}
define <8 x double> @sitof64_mask(<8 x double> %a, <8 x i32> %b, i8 %c) nounwind {
; KNL-LABEL: sitof64_mask:
; KNL:       # BB#0:
; KNL-NEXT:    kmovw %edi, %k1
; KNL-NEXT:    vcvtdq2pd %ymm1, %zmm0 {%k1}
; KNL-NEXT:    retq
;
; VLBW-LABEL: sitof64_mask:
; VLBW:       # BB#0:
; VLBW-NEXT:    kmovd %edi, %k1
; VLBW-NEXT:    vcvtdq2pd %ymm1, %zmm0 {%k1}
; VLBW-NEXT:    retq
;
; VLNOBW-LABEL: sitof64_mask:
; VLNOBW:       # BB#0:
; VLNOBW-NEXT:    kmovw %edi, %k1
; VLNOBW-NEXT:    vcvtdq2pd %ymm1, %zmm0 {%k1}
; VLNOBW-NEXT:    retq
;
; AVX512DQ-LABEL: sitof64_mask:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    kmovw %edi, %k1
; AVX512DQ-NEXT:    vcvtdq2pd %ymm1, %zmm0 {%k1}
; AVX512DQ-NEXT:    retq
;
; AVX512BW-LABEL: sitof64_mask:
; AVX512BW:       # BB#0:
; AVX512BW-NEXT:    kmovd %edi, %k1
; AVX512BW-NEXT:    vcvtdq2pd %ymm1, %zmm0 {%k1}
; AVX512BW-NEXT:    retq
  %1 = bitcast i8 %c to <8 x i1>
  %2 = sitofp <8 x i32> %b to <8 x double>
  %3 = select <8 x i1> %1, <8 x double> %2, <8 x double> %a
  ret <8 x double> %3
}
define <8 x double> @sitof64_maskz(<8 x i32> %a, i8 %b) nounwind {
; KNL-LABEL: sitof64_maskz:
; KNL:       # BB#0:
; KNL-NEXT:    kmovw %edi, %k1
; KNL-NEXT:    vcvtdq2pd %ymm0, %zmm0 {%k1} {z}
; KNL-NEXT:    retq
;
; VLBW-LABEL: sitof64_maskz:
; VLBW:       # BB#0:
; VLBW-NEXT:    kmovd %edi, %k1
; VLBW-NEXT:    vcvtdq2pd %ymm0, %zmm0 {%k1} {z}
; VLBW-NEXT:    retq
;
; VLNOBW-LABEL: sitof64_maskz:
; VLNOBW:       # BB#0:
; VLNOBW-NEXT:    kmovw %edi, %k1
; VLNOBW-NEXT:    vcvtdq2pd %ymm0, %zmm0 {%k1} {z}
; VLNOBW-NEXT:    retq
;
; AVX512DQ-LABEL: sitof64_maskz:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    kmovw %edi, %k1
; AVX512DQ-NEXT:    vcvtdq2pd %ymm0, %zmm0 {%k1} {z}
; AVX512DQ-NEXT:    retq
;
; AVX512BW-LABEL: sitof64_maskz:
; AVX512BW:       # BB#0:
; AVX512BW-NEXT:    kmovd %edi, %k1
; AVX512BW-NEXT:    vcvtdq2pd %ymm0, %zmm0 {%k1} {z}
; AVX512BW-NEXT:    retq
  %1 = bitcast i8 %b to <8 x i1>
  %2 = sitofp <8 x i32> %a to <8 x double>
  %3 = select <8 x i1> %1, <8 x double> %2, <8 x double> zeroinitializer
  ret <8 x double> %3
}

define <8 x i32> @fptosi01(<8 x double> %a) {
; ALL-LABEL: fptosi01:
; ALL:       # BB#0:
; ALL-NEXT:    vcvttpd2dq %zmm0, %ymm0
; ALL-NEXT:    retq
  %b = fptosi <8 x double> %a to <8 x i32>
  ret <8 x i32> %b
}

define <4 x i32> @fptosi03(<4 x double> %a) {
; KNL-LABEL: fptosi03:
; KNL:       # BB#0:
; KNL-NEXT:    vcvttpd2dq %ymm0, %xmm0
; KNL-NEXT:    retq
;
; AVX512-LABEL: fptosi03:
; AVX512:       # BB#0:
; AVX512-NEXT:    vcvttpd2dq %ymm0, %xmm0
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %b = fptosi <4 x double> %a to <4 x i32>
  ret <4 x i32> %b
}

define <16 x float> @fptrunc00(<16 x double> %b) nounwind {
; ALL-LABEL: fptrunc00:
; ALL:       # BB#0:
; ALL-NEXT:    vcvtpd2ps %zmm0, %ymm0
; ALL-NEXT:    vcvtpd2ps %zmm1, %ymm1
; ALL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
; ALL-NEXT:    retq
  %a = fptrunc <16 x double> %b to <16 x float>
  ret <16 x float> %a
}

define <4 x float> @fptrunc01(<4 x double> %b) {
; KNL-LABEL: fptrunc01:
; KNL:       # BB#0:
; KNL-NEXT:    vcvtpd2ps %ymm0, %xmm0
; KNL-NEXT:    retq
;
; AVX512-LABEL: fptrunc01:
; AVX512:       # BB#0:
; AVX512-NEXT:    vcvtpd2ps %ymm0, %xmm0
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %a = fptrunc <4 x double> %b to <4 x float>
  ret <4 x float> %a
}

define <4 x float> @fptrunc02(<4 x double> %b, <4 x i1> %mask) {
; KNL-LABEL: fptrunc02:
; KNL:       # BB#0:
; KNL-NEXT:    vpslld $31, %xmm1, %xmm1
; KNL-NEXT:    vpsrad $31, %xmm1, %xmm1
; KNL-NEXT:    vcvtpd2ps %ymm0, %xmm0
; KNL-NEXT:    vpand %xmm0, %xmm1, %xmm0
; KNL-NEXT:    retq
;
; VL-LABEL: fptrunc02:
; VL:       # BB#0:
; VL-NEXT:    vpslld $31, %xmm1, %xmm1
; VL-NEXT:    vptestmd %xmm1, %xmm1, %k1
; VL-NEXT:    vcvtpd2ps %ymm0, %xmm0 {%k1} {z}
; VL-NEXT:    vzeroupper
; VL-NEXT:    retq
;
; AVX512DQ-LABEL: fptrunc02:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX512DQ-NEXT:    vpsrad $31, %xmm1, %xmm1
; AVX512DQ-NEXT:    vcvtpd2ps %ymm0, %xmm0
; AVX512DQ-NEXT:    vpand %xmm0, %xmm1, %xmm0
; AVX512DQ-NEXT:    vzeroupper
; AVX512DQ-NEXT:    retq
;
; AVX512BW-LABEL: fptrunc02:
; AVX512BW:       # BB#0:
; AVX512BW-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX512BW-NEXT:    vpsrad $31, %xmm1, %xmm1
; AVX512BW-NEXT:    vcvtpd2ps %ymm0, %xmm0
; AVX512BW-NEXT:    vpand %xmm0, %xmm1, %xmm0
; AVX512BW-NEXT:    vzeroupper
; AVX512BW-NEXT:    retq
  %a = fptrunc <4 x double> %b to <4 x float>
  %c = select <4 x i1>%mask, <4 x float>%a, <4 x float> zeroinitializer
  ret <4 x float> %c
}

define <4 x float> @fptrunc03(<2 x double> %a0, <4 x float> %a1) nounwind {
; ALL-LABEL: fptrunc03:
; ALL:       # BB#0:
; ALL-NEXT:    vcvtsd2ss %xmm0, %xmm1, %xmm0
; ALL-NEXT:    retq
  %ext = extractelement <2 x double> %a0, i32 0
  %cvt = fptrunc double %ext to float
  %res = insertelement <4 x float> %a1, float %cvt, i32 0
  ret <4 x float> %res
}

define <8 x double> @fpext00(<8 x float> %b) nounwind {
; ALL-LABEL: fpext00:
; ALL:       # BB#0:
; ALL-NEXT:    vcvtps2pd %ymm0, %zmm0
; ALL-NEXT:    retq
  %a = fpext <8 x float> %b to <8 x double>
  ret <8 x double> %a
}

define <4 x double> @fpext01(<4 x float> %b, <4 x double>%b1, <4 x double>%a1) {
; NOVL-LABEL: fpext01:
; NOVL:       # BB#0:
; NOVL-NEXT:    vcvtps2pd %xmm0, %ymm0
; NOVL-NEXT:    vcmpltpd %ymm2, %ymm1, %ymm1
; NOVL-NEXT:    vandpd %ymm0, %ymm1, %ymm0
; NOVL-NEXT:    retq
;
; VL-LABEL: fpext01:
; VL:       # BB#0:
; VL-NEXT:    vcmpltpd %ymm2, %ymm1, %k1
; VL-NEXT:    vcvtps2pd %xmm0, %ymm0 {%k1} {z}
; VL-NEXT:    retq
  %a = fpext <4 x float> %b to <4 x double>
  %mask = fcmp ogt <4 x double>%a1, %b1
  %c = select <4 x i1>%mask,  <4 x double>%a, <4 x double>zeroinitializer
  ret <4 x double> %c
}

define <2 x double> @fpext02(<2 x double> %a0, <4 x float> %a1) nounwind {
; ALL-LABEL: fpext02:
; ALL:       # BB#0:
; ALL-NEXT:    vcvtss2sd %xmm1, %xmm0, %xmm0
; ALL-NEXT:    retq
  %ext = extractelement <4 x float> %a1, i32 0
  %cvt = fpext float %ext to double
  %res = insertelement <2 x double> %a0, double %cvt, i32 0
  ret <2 x double> %res
}

define double @funcA(i64* nocapture %e) {
; ALL-LABEL: funcA:
; ALL:       # BB#0: # %entry
; ALL-NEXT:    vcvtsi2sdq (%rdi), %xmm0, %xmm0
; ALL-NEXT:    retq
entry:
  %tmp1 = load i64, i64* %e, align 8
  %conv = sitofp i64 %tmp1 to double
  ret double %conv
}

define double @funcB(i32* %e) {
; ALL-LABEL: funcB:
; ALL:       # BB#0: # %entry
; ALL-NEXT:    vcvtsi2sdl (%rdi), %xmm0, %xmm0
; ALL-NEXT:    retq
entry:
  %tmp1 = load i32, i32* %e, align 4
  %conv = sitofp i32 %tmp1 to double
  ret double %conv
}

define float @funcC(i32* %e) {
; ALL-LABEL: funcC:
; ALL:       # BB#0: # %entry
; ALL-NEXT:    vcvtsi2ssl (%rdi), %xmm0, %xmm0
; ALL-NEXT:    retq
entry:
  %tmp1 = load i32, i32* %e, align 4
  %conv = sitofp i32 %tmp1 to float
  ret float %conv
}

define float @i64tof32(i64* %e) {
; ALL-LABEL: i64tof32:
; ALL:       # BB#0: # %entry
; ALL-NEXT:    vcvtsi2ssq (%rdi), %xmm0, %xmm0
; ALL-NEXT:    retq
entry:
  %tmp1 = load i64, i64* %e, align 8
  %conv = sitofp i64 %tmp1 to float
  ret float %conv
}

define void @fpext() {
; ALL-LABEL: fpext:
; ALL:       # BB#0: # %entry
; ALL-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; ALL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
; ALL-NEXT:    vmovsd %xmm0, -{{[0-9]+}}(%rsp)
; ALL-NEXT:    retq
entry:
  %f = alloca float, align 4
  %d = alloca double, align 8
  %tmp = load float, float* %f, align 4
  %conv = fpext float %tmp to double
  store double %conv, double* %d, align 8
  ret void
}

define void @fpround_scalar() nounwind uwtable {
; ALL-LABEL: fpround_scalar:
; ALL:       # BB#0: # %entry
; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
; ALL-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0
; ALL-NEXT:    vmovss %xmm0, -{{[0-9]+}}(%rsp)
; ALL-NEXT:    retq
entry:
  %f = alloca float, align 4
  %d = alloca double, align 8
  %tmp = load double, double* %d, align 8
  %conv = fptrunc double %tmp to float
  store float %conv, float* %f, align 4
  ret void
}

define double @long_to_double(i64 %x) {
; ALL-LABEL: long_to_double:
; ALL:       # BB#0:
; ALL-NEXT:    vmovq %rdi, %xmm0
; ALL-NEXT:    retq
   %res = bitcast i64 %x to double
   ret double %res
}

define i64 @double_to_long(double %x) {
; ALL-LABEL: double_to_long:
; ALL:       # BB#0:
; ALL-NEXT:    vmovq %xmm0, %rax
; ALL-NEXT:    retq
   %res = bitcast double %x to i64
   ret i64 %res
}

define float @int_to_float(i32 %x) {
; ALL-LABEL: int_to_float:
; ALL:       # BB#0:
; ALL-NEXT:    vmovd %edi, %xmm0
; ALL-NEXT:    retq
   %res = bitcast i32 %x to float
   ret float %res
}

define i32 @float_to_int(float %x) {
; ALL-LABEL: float_to_int:
; ALL:       # BB#0:
; ALL-NEXT:    vmovd %xmm0, %eax
; ALL-NEXT:    retq
   %res = bitcast float %x to i32
   ret i32 %res
}

define <16 x double> @uitof64(<16 x i32> %a) nounwind {
; ALL-LABEL: uitof64:
; ALL:       # BB#0:
; ALL-NEXT:    vcvtudq2pd %ymm0, %zmm2
; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
; ALL-NEXT:    vcvtudq2pd %ymm0, %zmm1
; ALL-NEXT:    vmovaps %zmm2, %zmm0
; ALL-NEXT:    retq
  %b = uitofp <16 x i32> %a to <16 x double>
  ret <16 x double> %b
}
define <8 x double> @uitof64_mask(<8 x double> %a, <8 x i32> %b, i8 %c) nounwind {
; KNL-LABEL: uitof64_mask:
; KNL:       # BB#0:
; KNL-NEXT:    kmovw %edi, %k1
; KNL-NEXT:    vcvtudq2pd %ymm1, %zmm0 {%k1}
; KNL-NEXT:    retq
;
; VLBW-LABEL: uitof64_mask:
; VLBW:       # BB#0:
; VLBW-NEXT:    kmovd %edi, %k1
; VLBW-NEXT:    vcvtudq2pd %ymm1, %zmm0 {%k1}
; VLBW-NEXT:    retq
;
; VLNOBW-LABEL: uitof64_mask:
; VLNOBW:       # BB#0:
; VLNOBW-NEXT:    kmovw %edi, %k1
; VLNOBW-NEXT:    vcvtudq2pd %ymm1, %zmm0 {%k1}
; VLNOBW-NEXT:    retq
;
; AVX512DQ-LABEL: uitof64_mask:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    kmovw %edi, %k1
; AVX512DQ-NEXT:    vcvtudq2pd %ymm1, %zmm0 {%k1}
; AVX512DQ-NEXT:    retq
;
; AVX512BW-LABEL: uitof64_mask:
; AVX512BW:       # BB#0:
; AVX512BW-NEXT:    kmovd %edi, %k1
; AVX512BW-NEXT:    vcvtudq2pd %ymm1, %zmm0 {%k1}
; AVX512BW-NEXT:    retq
  %1 = bitcast i8 %c to <8 x i1>
  %2 = uitofp <8 x i32> %b to <8 x double>
  %3 = select <8 x i1> %1, <8 x double> %2, <8 x double> %a
  ret <8 x double> %3
}
define <8 x double> @uitof64_maskz(<8 x i32> %a, i8 %b) nounwind {
; KNL-LABEL: uitof64_maskz:
; KNL:       # BB#0:
; KNL-NEXT:    kmovw %edi, %k1
; KNL-NEXT:    vcvtudq2pd %ymm0, %zmm0 {%k1} {z}
; KNL-NEXT:    retq
;
; VLBW-LABEL: uitof64_maskz:
; VLBW:       # BB#0:
; VLBW-NEXT:    kmovd %edi, %k1
; VLBW-NEXT:    vcvtudq2pd %ymm0, %zmm0 {%k1} {z}
; VLBW-NEXT:    retq
;
; VLNOBW-LABEL: uitof64_maskz:
; VLNOBW:       # BB#0:
; VLNOBW-NEXT:    kmovw %edi, %k1
; VLNOBW-NEXT:    vcvtudq2pd %ymm0, %zmm0 {%k1} {z}
; VLNOBW-NEXT:    retq
;
; AVX512DQ-LABEL: uitof64_maskz:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    kmovw %edi, %k1
; AVX512DQ-NEXT:    vcvtudq2pd %ymm0, %zmm0 {%k1} {z}
; AVX512DQ-NEXT:    retq
;
; AVX512BW-LABEL: uitof64_maskz:
; AVX512BW:       # BB#0:
; AVX512BW-NEXT:    kmovd %edi, %k1
; AVX512BW-NEXT:    vcvtudq2pd %ymm0, %zmm0 {%k1} {z}
; AVX512BW-NEXT:    retq
  %1 = bitcast i8 %b to <8 x i1>
  %2 = uitofp <8 x i32> %a to <8 x double>
  %3 = select <8 x i1> %1, <8 x double> %2, <8 x double> zeroinitializer
  ret <8 x double> %3
}

define <4 x double> @uitof64_256(<4 x i32> %a) nounwind {
; NOVL-LABEL: uitof64_256:
; NOVL:       # BB#0:
; NOVL-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<def>
; NOVL-NEXT:    vcvtudq2pd %ymm0, %zmm0
; NOVL-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
; NOVL-NEXT:    retq
;
; VL-LABEL: uitof64_256:
; VL:       # BB#0:
; VL-NEXT:    vcvtudq2pd %xmm0, %ymm0
; VL-NEXT:    retq
  %b = uitofp <4 x i32> %a to <4 x double>
  ret <4 x double> %b
}

define <16 x float> @uitof32(<16 x i32> %a) nounwind {
; ALL-LABEL: uitof32:
; ALL:       # BB#0:
; ALL-NEXT:    vcvtudq2ps %zmm0, %zmm0
; ALL-NEXT:    retq
  %b = uitofp <16 x i32> %a to <16 x float>
  ret <16 x float> %b
}

define <8 x float> @uitof32_256(<8 x i32> %a) nounwind {
; NOVL-LABEL: uitof32_256:
; NOVL:       # BB#0:
; NOVL-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
; NOVL-NEXT:    vcvtudq2ps %zmm0, %zmm0
; NOVL-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
; NOVL-NEXT:    retq
;
; VL-LABEL: uitof32_256:
; VL:       # BB#0:
; VL-NEXT:    vcvtudq2ps %ymm0, %ymm0
; VL-NEXT:    retq
  %b = uitofp <8 x i32> %a to <8 x float>
  ret <8 x float> %b
}

define <4 x float> @uitof32_128(<4 x i32> %a) nounwind {
; KNL-LABEL: uitof32_128:
; KNL:       # BB#0:
; KNL-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<def>
; KNL-NEXT:    vcvtudq2ps %zmm0, %zmm0
; KNL-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<kill>
; KNL-NEXT:    retq
;
; VL-LABEL: uitof32_128:
; VL:       # BB#0:
; VL-NEXT:    vcvtudq2ps %xmm0, %xmm0
; VL-NEXT:    retq
;
; AVX512DQ-LABEL: uitof32_128:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<def>
; AVX512DQ-NEXT:    vcvtudq2ps %zmm0, %zmm0
; AVX512DQ-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<kill>
; AVX512DQ-NEXT:    vzeroupper
; AVX512DQ-NEXT:    retq
;
; AVX512BW-LABEL: uitof32_128:
; AVX512BW:       # BB#0:
; AVX512BW-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<def>
; AVX512BW-NEXT:    vcvtudq2ps %zmm0, %zmm0
; AVX512BW-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<kill>
; AVX512BW-NEXT:    vzeroupper
; AVX512BW-NEXT:    retq
  %b = uitofp <4 x i32> %a to <4 x float>
  ret <4 x float> %b
}

define i32 @fptosi02(float %a) nounwind {
; ALL-LABEL: fptosi02:
; ALL:       # BB#0:
; ALL-NEXT:    vcvttss2si %xmm0, %eax
; ALL-NEXT:    retq
  %b = fptosi float %a to i32
  ret i32 %b
}

define i32 @fptoui02(float %a) nounwind {
; ALL-LABEL: fptoui02:
; ALL:       # BB#0:
; ALL-NEXT:    vcvttss2usi %xmm0, %eax
; ALL-NEXT:    retq
  %b = fptoui float %a to i32
  ret i32 %b
}

define float @uitofp02(i32 %a) nounwind {
; ALL-LABEL: uitofp02:
; ALL:       # BB#0:
; ALL-NEXT:    vcvtusi2ssl %edi, %xmm0, %xmm0
; ALL-NEXT:    retq
  %b = uitofp i32 %a to float
  ret float %b
}

define double @uitofp03(i32 %a) nounwind {
; ALL-LABEL: uitofp03:
; ALL:       # BB#0:
; ALL-NEXT:    vcvtusi2sdl %edi, %xmm0, %xmm0
; ALL-NEXT:    retq
  %b = uitofp i32 %a to double
  ret double %b
}

define <16 x float> @sitofp_16i1_float(<16 x i32> %a) {
; NODQ-LABEL: sitofp_16i1_float:
; NODQ:       # BB#0:
; NODQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; NODQ-NEXT:    vpcmpgtd %zmm0, %zmm1, %k1
; NODQ-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
; NODQ-NEXT:    vcvtdq2ps %zmm0, %zmm0
; NODQ-NEXT:    retq
;
; DQ-LABEL: sitofp_16i1_float:
; DQ:       # BB#0:
; DQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; DQ-NEXT:    vpcmpgtd %zmm0, %zmm1, %k0
; DQ-NEXT:    vpmovm2d %k0, %zmm0
; DQ-NEXT:    vcvtdq2ps %zmm0, %zmm0
; DQ-NEXT:    retq
  %mask = icmp slt <16 x i32> %a, zeroinitializer
  %1 = sitofp <16 x i1> %mask to <16 x float>
  ret <16 x float> %1
}

define <16 x float> @sitofp_16i8_float(<16 x i8> %a) {
; ALL-LABEL: sitofp_16i8_float:
; ALL:       # BB#0:
; ALL-NEXT:    vpmovsxbd %xmm0, %zmm0
; ALL-NEXT:    vcvtdq2ps %zmm0, %zmm0
; ALL-NEXT:    retq
  %1 = sitofp <16 x i8> %a to <16 x float>
  ret <16 x float> %1
}

define <16 x float> @sitofp_16i16_float(<16 x i16> %a) {
; ALL-LABEL: sitofp_16i16_float:
; ALL:       # BB#0:
; ALL-NEXT:    vpmovsxwd %ymm0, %zmm0
; ALL-NEXT:    vcvtdq2ps %zmm0, %zmm0
; ALL-NEXT:    retq
  %1 = sitofp <16 x i16> %a to <16 x float>
  ret <16 x float> %1
}

define <8 x double> @sitofp_8i16_double(<8 x i16> %a) {
; ALL-LABEL: sitofp_8i16_double:
; ALL:       # BB#0:
; ALL-NEXT:    vpmovsxwd %xmm0, %ymm0
; ALL-NEXT:    vcvtdq2pd %ymm0, %zmm0
; ALL-NEXT:    retq
  %1 = sitofp <8 x i16> %a to <8 x double>
  ret <8 x double> %1
}

define <8 x double> @sitofp_8i8_double(<8 x i8> %a) {
; ALL-LABEL: sitofp_8i8_double:
; ALL:       # BB#0:
; ALL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; ALL-NEXT:    vpslld $24, %ymm0, %ymm0
; ALL-NEXT:    vpsrad $24, %ymm0, %ymm0
; ALL-NEXT:    vcvtdq2pd %ymm0, %zmm0
; ALL-NEXT:    retq
  %1 = sitofp <8 x i8> %a to <8 x double>
  ret <8 x double> %1
}

define <16 x double> @sitofp_16i1_double(<16 x double> %a) {
; NOVLDQ-LABEL: sitofp_16i1_double:
; NOVLDQ:       # BB#0:
; NOVLDQ-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
; NOVLDQ-NEXT:    vcmpltpd %zmm1, %zmm2, %k1
; NOVLDQ-NEXT:    vcmpltpd %zmm0, %zmm2, %k2
; NOVLDQ-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
; NOVLDQ-NEXT:    vpmovqd %zmm0, %ymm0
; NOVLDQ-NEXT:    vcvtdq2pd %ymm0, %zmm0
; NOVLDQ-NEXT:    vpternlogq $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
; NOVLDQ-NEXT:    vpmovqd %zmm1, %ymm1
; NOVLDQ-NEXT:    vcvtdq2pd %ymm1, %zmm1
; NOVLDQ-NEXT:    retq
;
; VLDQ-LABEL: sitofp_16i1_double:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
; VLDQ-NEXT:    vcmpltpd %zmm1, %zmm2, %k0
; VLDQ-NEXT:    vcmpltpd %zmm0, %zmm2, %k1
; VLDQ-NEXT:    vpmovm2d %k1, %ymm0
; VLDQ-NEXT:    vcvtdq2pd %ymm0, %zmm0
; VLDQ-NEXT:    vpmovm2d %k0, %ymm1
; VLDQ-NEXT:    vcvtdq2pd %ymm1, %zmm1
; VLDQ-NEXT:    retq
;
; VLNODQ-LABEL: sitofp_16i1_double:
; VLNODQ:       # BB#0:
; VLNODQ-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
; VLNODQ-NEXT:    vcmpltpd %zmm1, %zmm2, %k1
; VLNODQ-NEXT:    vcmpltpd %zmm0, %zmm2, %k2
; VLNODQ-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
; VLNODQ-NEXT:    vmovdqa32 %ymm1, %ymm0 {%k2} {z}
; VLNODQ-NEXT:    vcvtdq2pd %ymm0, %zmm0
; VLNODQ-NEXT:    vmovdqa32 %ymm1, %ymm1 {%k1} {z}
; VLNODQ-NEXT:    vcvtdq2pd %ymm1, %zmm1
; VLNODQ-NEXT:    retq
;
; AVX512DQ-LABEL: sitofp_16i1_double:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
; AVX512DQ-NEXT:    vcmpltpd %zmm1, %zmm2, %k0
; AVX512DQ-NEXT:    vcmpltpd %zmm0, %zmm2, %k1
; AVX512DQ-NEXT:    vpmovm2d %k1, %zmm0
; AVX512DQ-NEXT:    vcvtdq2pd %ymm0, %zmm0
; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm1
; AVX512DQ-NEXT:    vcvtdq2pd %ymm1, %zmm1
; AVX512DQ-NEXT:    retq
  %cmpres = fcmp ogt <16 x double> %a, zeroinitializer
  %1 = sitofp <16 x i1> %cmpres to <16 x double>
  ret <16 x double> %1
}

define <8 x double> @sitofp_8i1_double(<8 x double> %a) {
; NOVLDQ-LABEL: sitofp_8i1_double:
; NOVLDQ:       # BB#0:
; NOVLDQ-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
; NOVLDQ-NEXT:    vcmpltpd %zmm0, %zmm1, %k1
; NOVLDQ-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
; NOVLDQ-NEXT:    vpmovqd %zmm0, %ymm0
; NOVLDQ-NEXT:    vcvtdq2pd %ymm0, %zmm0
; NOVLDQ-NEXT:    retq
;
; VLDQ-LABEL: sitofp_8i1_double:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
; VLDQ-NEXT:    vcmpltpd %zmm0, %zmm1, %k0
; VLDQ-NEXT:    vpmovm2d %k0, %ymm0
; VLDQ-NEXT:    vcvtdq2pd %ymm0, %zmm0
; VLDQ-NEXT:    retq
;
; VLNODQ-LABEL: sitofp_8i1_double:
; VLNODQ:       # BB#0:
; VLNODQ-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
; VLNODQ-NEXT:    vcmpltpd %zmm0, %zmm1, %k1
; VLNODQ-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
; VLNODQ-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
; VLNODQ-NEXT:    vcvtdq2pd %ymm0, %zmm0
; VLNODQ-NEXT:    retq
;
; AVX512DQ-LABEL: sitofp_8i1_double:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
; AVX512DQ-NEXT:    vcmpltpd %zmm0, %zmm1, %k0
; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
; AVX512DQ-NEXT:    vcvtdq2pd %ymm0, %zmm0
; AVX512DQ-NEXT:    retq
  %cmpres = fcmp ogt <8 x double> %a, zeroinitializer
  %1 = sitofp <8 x i1> %cmpres to <8 x double>
  ret <8 x double> %1
}

define <8 x float> @sitofp_8i1_float(<8 x float> %a) {
; NOVLDQ-LABEL: sitofp_8i1_float:
; NOVLDQ:       # BB#0:
; NOVLDQ-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
; NOVLDQ-NEXT:    vxorps %xmm1, %xmm1, %xmm1
; NOVLDQ-NEXT:    vcmpltps %zmm0, %zmm1, %k1
; NOVLDQ-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
; NOVLDQ-NEXT:    vpmovqd %zmm0, %ymm0
; NOVLDQ-NEXT:    vcvtdq2ps %ymm0, %ymm0
; NOVLDQ-NEXT:    retq
;
; VLDQ-LABEL: sitofp_8i1_float:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vxorps %xmm1, %xmm1, %xmm1
; VLDQ-NEXT:    vcmpltps %ymm0, %ymm1, %k0
; VLDQ-NEXT:    vpmovm2d %k0, %ymm0
; VLDQ-NEXT:    vcvtdq2ps %ymm0, %ymm0
; VLDQ-NEXT:    retq
;
; VLNODQ-LABEL: sitofp_8i1_float:
; VLNODQ:       # BB#0:
; VLNODQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; VLNODQ-NEXT:    vcmpltps %ymm0, %ymm1, %k1
; VLNODQ-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
; VLNODQ-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
; VLNODQ-NEXT:    vcvtdq2ps %ymm0, %ymm0
; VLNODQ-NEXT:    retq
;
; AVX512DQ-LABEL: sitofp_8i1_float:
; AVX512DQ:       # BB#0:
; AVX512DQ-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
; AVX512DQ-NEXT:    vxorps %xmm1, %xmm1, %xmm1
; AVX512DQ-NEXT:    vcmpltps %zmm0, %zmm1, %k0
; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
; AVX512DQ-NEXT:    vcvtdq2ps %ymm0, %ymm0
; AVX512DQ-NEXT:    retq
  %cmpres = fcmp ogt <8 x float> %a, zeroinitializer
  %1 = sitofp <8 x i1> %cmpres to <8 x float>
  ret <8 x float> %1
}

define <4 x float> @sitofp_4i1_float(<4 x float> %a) {
; NOVL-LABEL: sitofp_4i1_float:
; NOVL:       # BB#0:
; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
; NOVL-NEXT:    vcmpltps %xmm0, %xmm1, %xmm0
; NOVL-NEXT:    vcvtdq2ps %xmm0, %xmm0
; NOVL-NEXT:    retq
;
; VLDQ-LABEL: sitofp_4i1_float:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vxorps %xmm1, %xmm1, %xmm1
; VLDQ-NEXT:    vcmpltps %xmm0, %xmm1, %k0
; VLDQ-NEXT:    vpmovm2d %k0, %xmm0
; VLDQ-NEXT:    vcvtdq2ps %xmm0, %xmm0
; VLDQ-NEXT:    retq
;
; VLNODQ-LABEL: sitofp_4i1_float:
; VLNODQ:       # BB#0:
; VLNODQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; VLNODQ-NEXT:    vcmpltps %xmm0, %xmm1, %k1
; VLNODQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
; VLNODQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
; VLNODQ-NEXT:    vcvtdq2ps %xmm0, %xmm0
; VLNODQ-NEXT:    retq
  %cmpres = fcmp ogt <4 x float> %a, zeroinitializer
  %1 = sitofp <4 x i1> %cmpres to <4 x float>
  ret <4 x float> %1
}

define <4 x double> @sitofp_4i1_double(<4 x double> %a) {
; NOVL-LABEL: sitofp_4i1_double:
; NOVL:       # BB#0:
; NOVL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
; NOVL-NEXT:    vcmpltpd %ymm0, %ymm1, %ymm0
; NOVL-NEXT:    vpmovqd %zmm0, %ymm0
; NOVL-NEXT:    vcvtdq2pd %xmm0, %ymm0
; NOVL-NEXT:    retq
;
; VLDQ-LABEL: sitofp_4i1_double:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
; VLDQ-NEXT:    vcmpltpd %ymm0, %ymm1, %k0
; VLDQ-NEXT:    vpmovm2d %k0, %xmm0
; VLDQ-NEXT:    vcvtdq2pd %xmm0, %ymm0
; VLDQ-NEXT:    retq
;
; VLNODQ-LABEL: sitofp_4i1_double:
; VLNODQ:       # BB#0:
; VLNODQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; VLNODQ-NEXT:    vcmpltpd %ymm0, %ymm1, %k1
; VLNODQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
; VLNODQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
; VLNODQ-NEXT:    vcvtdq2pd %xmm0, %ymm0
; VLNODQ-NEXT:    retq
  %cmpres = fcmp ogt <4 x double> %a, zeroinitializer
  %1 = sitofp <4 x i1> %cmpres to <4 x double>
  ret <4 x double> %1
}

define <2 x float> @sitofp_2i1_float(<2 x float> %a) {
; NOVL-LABEL: sitofp_2i1_float:
; NOVL:       # BB#0:
; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
; NOVL-NEXT:    vcmpltps %xmm0, %xmm1, %xmm0
; NOVL-NEXT:    vcvtdq2ps %xmm0, %xmm0
; NOVL-NEXT:    retq
;
; VLDQ-LABEL: sitofp_2i1_float:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vxorps %xmm1, %xmm1, %xmm1
; VLDQ-NEXT:    vcmpltps %xmm0, %xmm1, %k0
; VLDQ-NEXT:    vpmovm2d %k0, %xmm0
; VLDQ-NEXT:    vcvtdq2ps %xmm0, %xmm0
; VLDQ-NEXT:    retq
;
; VLNODQ-LABEL: sitofp_2i1_float:
; VLNODQ:       # BB#0:
; VLNODQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; VLNODQ-NEXT:    vcmpltps %xmm0, %xmm1, %k1
; VLNODQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
; VLNODQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
; VLNODQ-NEXT:    vcvtdq2ps %xmm0, %xmm0
; VLNODQ-NEXT:    retq
  %cmpres = fcmp ogt <2 x float> %a, zeroinitializer
  %1 = sitofp <2 x i1> %cmpres to <2 x float>
  ret <2 x float> %1
}

define <2 x double> @sitofp_2i1_double(<2 x double> %a) {
; NOVL-LABEL: sitofp_2i1_double:
; NOVL:       # BB#0:
; NOVL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
; NOVL-NEXT:    vcmpltpd %xmm0, %xmm1, %xmm0
; NOVL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
; NOVL-NEXT:    vcvtdq2pd %xmm0, %xmm0
; NOVL-NEXT:    retq
;
; VLDQ-LABEL: sitofp_2i1_double:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
; VLDQ-NEXT:    vcmpltpd %xmm0, %xmm1, %k0
; VLDQ-NEXT:    vpmovm2q %k0, %xmm0
; VLDQ-NEXT:    vcvtqq2pd %xmm0, %xmm0
; VLDQ-NEXT:    retq
;
; VLNODQ-LABEL: sitofp_2i1_double:
; VLNODQ:       # BB#0:
; VLNODQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; VLNODQ-NEXT:    vcmpltpd %xmm0, %xmm1, %k1
; VLNODQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
; VLNODQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
; VLNODQ-NEXT:    vpextrq $1, %xmm0, %rax
; VLNODQ-NEXT:    vcvtsi2sdq %rax, %xmm2, %xmm1
; VLNODQ-NEXT:    vmovq %xmm0, %rax
; VLNODQ-NEXT:    vcvtsi2sdq %rax, %xmm2, %xmm0
; VLNODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; VLNODQ-NEXT:    retq
  %cmpres = fcmp ogt <2 x double> %a, zeroinitializer
  %1 = sitofp <2 x i1> %cmpres to <2 x double>
  ret <2 x double> %1
}

define <16 x float> @uitofp_16i8(<16 x i8>%a) {
; ALL-LABEL: uitofp_16i8:
; ALL:       # BB#0:
; ALL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; ALL-NEXT:    vcvtdq2ps %zmm0, %zmm0
; ALL-NEXT:    retq
  %b = uitofp <16 x i8> %a to <16 x float>
  ret <16 x float>%b
}

define <16 x float> @uitofp_16i16(<16 x i16>%a) {
; ALL-LABEL: uitofp_16i16:
; ALL:       # BB#0:
; ALL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; ALL-NEXT:    vcvtdq2ps %zmm0, %zmm0
; ALL-NEXT:    retq
  %b = uitofp <16 x i16> %a to <16 x float>
  ret <16 x float>%b
}

define <16 x float> @uitofp_16i1_float(<16 x i32> %a) {
; ALL-LABEL: uitofp_16i1_float:
; ALL:       # BB#0:
; ALL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; ALL-NEXT:    vpcmpgtd %zmm0, %zmm1, %k1
; ALL-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z}
; ALL-NEXT:    vcvtudq2ps %zmm0, %zmm0
; ALL-NEXT:    retq
  %mask = icmp slt <16 x i32> %a, zeroinitializer
  %1 = uitofp <16 x i1> %mask to <16 x float>
  ret <16 x float> %1
}

define <16 x double> @uitofp_16i1_double(<16 x i32> %a) {
; NOVL-LABEL: uitofp_16i1_double:
; NOVL:       # BB#0:
; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; NOVL-NEXT:    vpcmpgtd %zmm0, %zmm1, %k1
; NOVL-NEXT:    movq {{.*}}(%rip), %rax
; NOVL-NEXT:    vpbroadcastq %rax, %zmm0 {%k1} {z}
; NOVL-NEXT:    vpmovqd %zmm0, %ymm0
; NOVL-NEXT:    vcvtudq2pd %ymm0, %zmm0
; NOVL-NEXT:    kshiftrw $8, %k1, %k1
; NOVL-NEXT:    vpbroadcastq %rax, %zmm1 {%k1} {z}
; NOVL-NEXT:    vpmovqd %zmm1, %ymm1
; NOVL-NEXT:    vcvtudq2pd %ymm1, %zmm1
; NOVL-NEXT:    retq
;
; VL-LABEL: uitofp_16i1_double:
; VL:       # BB#0:
; VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; VL-NEXT:    vpcmpgtd %zmm0, %zmm1, %k1
; VL-NEXT:    movl {{.*}}(%rip), %eax
; VL-NEXT:    vpbroadcastd %eax, %ymm0 {%k1} {z}
; VL-NEXT:    vcvtudq2pd %ymm0, %zmm0
; VL-NEXT:    kshiftrw $8, %k1, %k1
; VL-NEXT:    vpbroadcastd %eax, %ymm1 {%k1} {z}
; VL-NEXT:    vcvtudq2pd %ymm1, %zmm1
; VL-NEXT:    retq
  %mask = icmp slt <16 x i32> %a, zeroinitializer
  %1 = uitofp <16 x i1> %mask to <16 x double>
  ret <16 x double> %1
}

define <8 x float> @uitofp_8i1_float(<8 x i32> %a) {
; NOVL-LABEL: uitofp_8i1_float:
; NOVL:       # BB#0:
; NOVL-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; NOVL-NEXT:    vpcmpgtd %zmm0, %zmm1, %k1
; NOVL-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z}
; NOVL-NEXT:    vpmovqd %zmm0, %ymm0
; NOVL-NEXT:    vcvtudq2ps %zmm0, %zmm0
; NOVL-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
; NOVL-NEXT:    retq
;
; VL-LABEL: uitofp_8i1_float:
; VL:       # BB#0:
; VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; VL-NEXT:    vpcmpgtd %ymm0, %ymm1, %k1
; VL-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm0 {%k1} {z}
; VL-NEXT:    vcvtudq2ps %ymm0, %ymm0
; VL-NEXT:    retq
  %mask = icmp slt <8 x i32> %a, zeroinitializer
  %1 = uitofp <8 x i1> %mask to <8 x float>
  ret <8 x float> %1
}

define <8 x double> @uitofp_8i1_double(<8 x i32> %a) {
; NOVL-LABEL: uitofp_8i1_double:
; NOVL:       # BB#0:
; NOVL-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; NOVL-NEXT:    vpcmpgtd %zmm0, %zmm1, %k1
; NOVL-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z}
; NOVL-NEXT:    vpmovqd %zmm0, %ymm0
; NOVL-NEXT:    vcvtudq2pd %ymm0, %zmm0
; NOVL-NEXT:    retq
;
; VL-LABEL: uitofp_8i1_double:
; VL:       # BB#0:
; VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; VL-NEXT:    vpcmpgtd %ymm0, %ymm1, %k1
; VL-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm0 {%k1} {z}
; VL-NEXT:    vcvtudq2pd %ymm0, %zmm0
; VL-NEXT:    retq
  %mask = icmp slt <8 x i32> %a, zeroinitializer
  %1 = uitofp <8 x i1> %mask to <8 x double>
  ret <8 x double> %1
}

define <4 x float> @uitofp_4i1_float(<4 x i32> %a) {
; NOVL-LABEL: uitofp_4i1_float:
; NOVL:       # BB#0:
; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; NOVL-NEXT:    vpcmpgtd %xmm0, %xmm1, %xmm0
; NOVL-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1,1,1,1]
; NOVL-NEXT:    vpand %xmm1, %xmm0, %xmm0
; NOVL-NEXT:    retq
;
; VL-LABEL: uitofp_4i1_float:
; VL:       # BB#0:
; VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; VL-NEXT:    vpcmpgtd %xmm0, %xmm1, %k1
; VL-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm0 {%k1} {z}
; VL-NEXT:    vcvtudq2ps %xmm0, %xmm0
; VL-NEXT:    retq
  %mask = icmp slt <4 x i32> %a, zeroinitializer
  %1 = uitofp <4 x i1> %mask to <4 x float>
  ret <4 x float> %1
}

define <4 x double> @uitofp_4i1_double(<4 x i32> %a) {
; NOVL-LABEL: uitofp_4i1_double:
; NOVL:       # BB#0:
; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; NOVL-NEXT:    vpcmpgtd %xmm0, %xmm1, %xmm0
; NOVL-NEXT:    vpsrld $31, %xmm0, %xmm0
; NOVL-NEXT:    vcvtdq2pd %xmm0, %ymm0
; NOVL-NEXT:    retq
;
; VL-LABEL: uitofp_4i1_double:
; VL:       # BB#0:
; VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; VL-NEXT:    vpcmpgtd %xmm0, %xmm1, %k1
; VL-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm0 {%k1} {z}
; VL-NEXT:    vcvtudq2pd %xmm0, %ymm0
; VL-NEXT:    retq
  %mask = icmp slt <4 x i32> %a, zeroinitializer
  %1 = uitofp <4 x i1> %mask to <4 x double>
  ret <4 x double> %1
}

define <2 x float> @uitofp_2i1_float(<2 x i32> %a) {
; NOVL-LABEL: uitofp_2i1_float:
; NOVL:       # BB#0:
; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; NOVL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
; NOVL-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
; NOVL-NEXT:    vpextrb $8, %xmm0, %eax
; NOVL-NEXT:    andl $1, %eax
; NOVL-NEXT:    vcvtsi2ssl %eax, %xmm2, %xmm1
; NOVL-NEXT:    vpextrb $0, %xmm0, %eax
; NOVL-NEXT:    andl $1, %eax
; NOVL-NEXT:    vcvtsi2ssl %eax, %xmm2, %xmm0
; NOVL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
; NOVL-NEXT:    retq
;
; VL-LABEL: uitofp_2i1_float:
; VL:       # BB#0:
; VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
; VL-NEXT:    vpcmpltuq %xmm1, %xmm0, %k1
; VL-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm0 {%k1} {z}
; VL-NEXT:    vcvtudq2ps %xmm0, %xmm0
; VL-NEXT:    retq
  %mask = icmp ult <2 x i32> %a, zeroinitializer
  %1 = uitofp <2 x i1> %mask to <2 x float>
  ret <2 x float> %1
}

define <2 x double> @uitofp_2i1_double(<2 x i32> %a) {
; NOVL-LABEL: uitofp_2i1_double:
; NOVL:       # BB#0:
; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; NOVL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
; NOVL-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
; NOVL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
; NOVL-NEXT:    retq
;
; VLDQ-LABEL: uitofp_2i1_double:
; VLDQ:       # BB#0:
; VLDQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; VLDQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
; VLDQ-NEXT:    vpcmpltuq %xmm1, %xmm0, %k1
; VLDQ-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm0 {%k1} {z}
; VLDQ-NEXT:    vcvtuqq2pd %xmm0, %xmm0
; VLDQ-NEXT:    retq
;
; VLNODQ-LABEL: uitofp_2i1_double:
; VLNODQ:       # BB#0:
; VLNODQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; VLNODQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
; VLNODQ-NEXT:    vpcmpltuq %xmm1, %xmm0, %k1
; VLNODQ-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm0 {%k1} {z}
; VLNODQ-NEXT:    vpextrq $1, %xmm0, %rax
; VLNODQ-NEXT:    vcvtusi2sdq %rax, %xmm2, %xmm1
; VLNODQ-NEXT:    vmovq %xmm0, %rax
; VLNODQ-NEXT:    vcvtusi2sdq %rax, %xmm2, %xmm0
; VLNODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; VLNODQ-NEXT:    retq
  %mask = icmp ult <2 x i32> %a, zeroinitializer
  %1 = uitofp <2 x i1> %mask to <2 x double>
  ret <2 x double> %1
}