diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td index f9a1c6ddb6c3..12b8b0b13497 100644 --- a/llvm/lib/Target/X86/X86InstrAVX512.td +++ b/llvm/lib/Target/X86/X86InstrAVX512.td @@ -3845,138 +3845,73 @@ let Predicates = [HasVLX] in { (VMOVDQU32Z256mr addr:$dst, (v32i8 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>; } +multiclass masked_move_for_extract { + def : Pat<(Cast.VT (vselect Cast.KRCWM:$mask, + (bitconvert + (To.VT (extract_subvector + (From.VT From.RC:$src), (iPTR 0)))), + To.RC:$src0)), + (Cast.VT (!cast(InstrStr#"rrk") + Cast.RC:$src0, Cast.KRCWM:$mask, + (EXTRACT_SUBREG From.RC:$src, To.SubRegIdx)))>; + + def : Pat<(Cast.VT (vselect Cast.KRCWM:$mask, + (bitconvert + (To.VT (extract_subvector + (From.VT From.RC:$src), (iPTR 0)))), + Cast.ImmAllZerosV)), + (Cast.VT (!cast(InstrStr#"rrkz") + Cast.KRCWM:$mask, + (EXTRACT_SUBREG From.RC:$src, To.SubRegIdx)))>; +} + + let Predicates = [HasVLX] in { // A masked extract from the first 128-bits of a 256-bit vector can be // implemented with masked move. -def : Pat<(v2i64 (vselect VK2WM:$mask, - (extract_subvector (v4i64 VR256X:$src), (iPTR 0)), - VR128X:$src0)), - (v2i64 (VMOVDQA64Z128rrk VR128X:$src0, VK2WM:$mask, - (EXTRACT_SUBREG (v4i64 VR256X:$src), sub_xmm)))>; -def : Pat<(v4i32 (vselect VK4WM:$mask, - (extract_subvector (v8i32 VR256X:$src), (iPTR 0)), - VR128X:$src0)), - (v4i32 (VMOVDQA32Z128rrk VR128X:$src0, VK4WM:$mask, - (EXTRACT_SUBREG (v8i32 VR256X:$src), sub_xmm)))>; -def : Pat<(v2f64 (vselect VK2WM:$mask, - (extract_subvector (v4f64 VR256X:$src), (iPTR 0)), - VR128X:$src0)), - (v2f64 (VMOVAPDZ128rrk VR128X:$src0, VK2WM:$mask, - (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm)))>; -def : Pat<(v4f32 (vselect VK4WM:$mask, - (extract_subvector (v8f32 VR256X:$src), (iPTR 0)), - VR128X:$src0)), - (v4f32 (VMOVAPSZ128rrk VR128X:$src0, VK4WM:$mask, - (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm)))>; - -def : Pat<(v2i64 (vselect VK2WM:$mask, - (extract_subvector (v4i64 VR256X:$src), (iPTR 0)), - (bitconvert (v4i32 immAllZerosV)))), - (v2i64 (VMOVDQA64Z128rrkz VK2WM:$mask, - (EXTRACT_SUBREG (v4i64 VR256X:$src), sub_xmm)))>; -def : Pat<(v4i32 (vselect VK4WM:$mask, - (extract_subvector (v8i32 VR256X:$src), (iPTR 0)), - (bitconvert (v4i32 immAllZerosV)))), - (v4i32 (VMOVDQA32Z128rrkz VK4WM:$mask, - (EXTRACT_SUBREG (v8i32 VR256X:$src), sub_xmm)))>; -def : Pat<(v2f64 (vselect VK2WM:$mask, - (extract_subvector (v4f64 VR256X:$src), (iPTR 0)), - (bitconvert (v4i32 immAllZerosV)))), - (v2f64 (VMOVAPDZ128rrkz VK2WM:$mask, - (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm)))>; -def : Pat<(v4f32 (vselect VK4WM:$mask, - (extract_subvector (v8f32 VR256X:$src), (iPTR 0)), - (bitconvert (v4i32 immAllZerosV)))), - (v4f32 (VMOVAPSZ128rrkz VK4WM:$mask, - (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm)))>; +defm : masked_move_for_extract<"VMOVDQA64Z128", v4i64x_info, v2i64x_info, v2i64x_info>; +defm : masked_move_for_extract<"VMOVDQA64Z128", v8i32x_info, v4i32x_info, v2i64x_info>; +defm : masked_move_for_extract<"VMOVDQA64Z128", v16i16x_info, v8i16x_info, v2i64x_info>; +defm : masked_move_for_extract<"VMOVDQA64Z128", v32i8x_info, v16i8x_info, v2i64x_info>; +defm : masked_move_for_extract<"VMOVDQA32Z128", v4i64x_info, v2i64x_info, v4i32x_info>; +defm : masked_move_for_extract<"VMOVDQA32Z128", v8i32x_info, v4i32x_info, v4i32x_info>; +defm : masked_move_for_extract<"VMOVDQA32Z128", v16i16x_info, v8i16x_info, v4i32x_info>; +defm : masked_move_for_extract<"VMOVDQA32Z128", v32i8x_info, v16i8x_info, v4i32x_info>; +defm : masked_move_for_extract<"VMOVAPDZ128", v4f64x_info, v2f64x_info, v2f64x_info>; +defm : masked_move_for_extract<"VMOVAPDZ128", v8f32x_info, v4f32x_info, v2f64x_info>; +defm : masked_move_for_extract<"VMOVAPSZ128", v4f64x_info, v2f64x_info, v4f32x_info>; +defm : masked_move_for_extract<"VMOVAPSZ128", v8f32x_info, v4f32x_info, v4f32x_info>; // A masked extract from the first 128-bits of a 512-bit vector can be // implemented with masked move. -def : Pat<(v2i64 (vselect VK2WM:$mask, - (extract_subvector (v8i64 VR512:$src), (iPTR 0)), - VR128X:$src0)), - (v2i64 (VMOVDQA64Z128rrk VR128X:$src0, VK2WM:$mask, - (EXTRACT_SUBREG (v8i64 VR512:$src), sub_xmm)))>; -def : Pat<(v4i32 (vselect VK4WM:$mask, - (extract_subvector (v16i32 VR512:$src), (iPTR 0)), - VR128X:$src0)), - (v4i32 (VMOVDQA32Z128rrk VR128X:$src0, VK4WM:$mask, - (EXTRACT_SUBREG (v16i32 VR512:$src), sub_xmm)))>; -def : Pat<(v2f64 (vselect VK2WM:$mask, - (extract_subvector (v8f64 VR512:$src), (iPTR 0)), - VR128X:$src0)), - (v2f64 (VMOVAPDZ128rrk VR128X:$src0, VK2WM:$mask, - (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm)))>; -def : Pat<(v4f32 (vselect VK4WM:$mask, - (extract_subvector (v16f32 VR512:$src), (iPTR 0)), - VR128X:$src0)), - (v4f32 (VMOVAPSZ128rrk VR128X:$src0, VK4WM:$mask, - (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm)))>; - -def : Pat<(v2i64 (vselect VK2WM:$mask, - (extract_subvector (v8i64 VR512:$src), (iPTR 0)), - (bitconvert (v4i32 immAllZerosV)))), - (v2i64 (VMOVDQA64Z128rrkz VK2WM:$mask, - (EXTRACT_SUBREG (v8i64 VR512:$src), sub_xmm)))>; -def : Pat<(v4i32 (vselect VK4WM:$mask, - (extract_subvector (v16i32 VR512:$src), (iPTR 0)), - (bitconvert (v4i32 immAllZerosV)))), - (v4i32 (VMOVDQA32Z128rrkz VK4WM:$mask, - (EXTRACT_SUBREG (v16i32 VR512:$src), sub_xmm)))>; -def : Pat<(v2f64 (vselect VK2WM:$mask, - (extract_subvector (v8f64 VR512:$src), (iPTR 0)), - (bitconvert (v4i32 immAllZerosV)))), - (v2f64 (VMOVAPDZ128rrkz VK2WM:$mask, - (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm)))>; -def : Pat<(v4f32 (vselect VK4WM:$mask, - (extract_subvector (v16f32 VR512:$src), (iPTR 0)), - (bitconvert (v4i32 immAllZerosV)))), - (v4f32 (VMOVAPSZ128rrkz VK4WM:$mask, - (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm)))>; +defm : masked_move_for_extract<"VMOVDQA64Z128", v8i64_info, v2i64x_info, v2i64x_info>; +defm : masked_move_for_extract<"VMOVDQA64Z128", v16i32_info, v4i32x_info, v2i64x_info>; +defm : masked_move_for_extract<"VMOVDQA64Z128", v32i16_info, v8i16x_info, v2i64x_info>; +defm : masked_move_for_extract<"VMOVDQA64Z128", v64i8_info, v16i8x_info, v2i64x_info>; +defm : masked_move_for_extract<"VMOVDQA32Z128", v8i64_info, v2i64x_info, v4i32x_info>; +defm : masked_move_for_extract<"VMOVDQA32Z128", v16i32_info, v4i32x_info, v4i32x_info>; +defm : masked_move_for_extract<"VMOVDQA32Z128", v32i16_info, v8i16x_info, v4i32x_info>; +defm : masked_move_for_extract<"VMOVDQA32Z128", v64i8_info, v16i8x_info, v4i32x_info>; +defm : masked_move_for_extract<"VMOVAPDZ128", v8f64_info, v2f64x_info, v2f64x_info>; +defm : masked_move_for_extract<"VMOVAPDZ128", v16f32_info, v4f32x_info, v2f64x_info>; +defm : masked_move_for_extract<"VMOVAPSZ128", v8f64_info, v2f64x_info, v4f32x_info>; +defm : masked_move_for_extract<"VMOVAPSZ128", v16f32_info, v4f32x_info, v4f32x_info>; // A masked extract from the first 256-bits of a 512-bit vector can be // implemented with masked move. -def : Pat<(v4i64 (vselect VK4WM:$mask, - (extract_subvector (v8i64 VR512:$src), (iPTR 0)), - VR256X:$src0)), - (v4i64 (VMOVDQA64Z256rrk VR256X:$src0, VK4WM:$mask, - (EXTRACT_SUBREG (v8i64 VR512:$src), sub_ymm)))>; -def : Pat<(v8i32 (vselect VK8WM:$mask, - (extract_subvector (v16i32 VR512:$src), (iPTR 0)), - VR256X:$src0)), - (v8i32 (VMOVDQA32Z256rrk VR256X:$src0, VK8WM:$mask, - (EXTRACT_SUBREG (v16i32 VR512:$src), sub_ymm)))>; -def : Pat<(v4f64 (vselect VK4WM:$mask, - (extract_subvector (v8f64 VR512:$src), (iPTR 0)), - VR256X:$src0)), - (v4f64 (VMOVAPDZ256rrk VR256X:$src0, VK4WM:$mask, - (EXTRACT_SUBREG (v8f64 VR512:$src), sub_ymm)))>; -def : Pat<(v8f32 (vselect VK8WM:$mask, - (extract_subvector (v16f32 VR512:$src), (iPTR 0)), - VR256X:$src0)), - (v8f32 (VMOVAPSZ256rrk VR256X:$src0, VK8WM:$mask, - (EXTRACT_SUBREG (v16f32 VR512:$src), sub_ymm)))>; - -def : Pat<(v4i64 (vselect VK4WM:$mask, - (extract_subvector (v8i64 VR512:$src), (iPTR 0)), - (bitconvert (v8i32 immAllZerosV)))), - (v4i64 (VMOVDQA64Z256rrkz VK4WM:$mask, - (EXTRACT_SUBREG (v8i64 VR512:$src), sub_ymm)))>; -def : Pat<(v8i32 (vselect VK8WM:$mask, - (extract_subvector (v16i32 VR512:$src), (iPTR 0)), - (bitconvert (v8i32 immAllZerosV)))), - (v8i32 (VMOVDQA32Z256rrkz VK8WM:$mask, - (EXTRACT_SUBREG (v16i32 VR512:$src), sub_ymm)))>; -def : Pat<(v4f64 (vselect VK4WM:$mask, - (extract_subvector (v8f64 VR512:$src), (iPTR 0)), - (bitconvert (v8i32 immAllZerosV)))), - (v4f64 (VMOVAPDZ256rrkz VK4WM:$mask, - (EXTRACT_SUBREG (v8f64 VR512:$src), sub_ymm)))>; -def : Pat<(v8f32 (vselect VK8WM:$mask, - (extract_subvector (v16f32 VR512:$src), (iPTR 0)), - (bitconvert (v8i32 immAllZerosV)))), - (v8f32 (VMOVAPSZ256rrkz VK8WM:$mask, - (EXTRACT_SUBREG (v16f32 VR512:$src), sub_ymm)))>; +defm : masked_move_for_extract<"VMOVDQA64Z256", v8i64_info, v4i64x_info, v4i64x_info>; +defm : masked_move_for_extract<"VMOVDQA64Z256", v16i32_info, v8i32x_info, v4i64x_info>; +defm : masked_move_for_extract<"VMOVDQA64Z256", v32i16_info, v16i16x_info, v4i64x_info>; +defm : masked_move_for_extract<"VMOVDQA64Z256", v64i8_info, v32i8x_info, v4i64x_info>; +defm : masked_move_for_extract<"VMOVDQA32Z256", v8i64_info, v4i64x_info, v8i32x_info>; +defm : masked_move_for_extract<"VMOVDQA32Z256", v16i32_info, v8i32x_info, v8i32x_info>; +defm : masked_move_for_extract<"VMOVDQA32Z256", v32i16_info, v16i16x_info, v8i32x_info>; +defm : masked_move_for_extract<"VMOVDQA32Z256", v64i8_info, v32i8x_info, v8i32x_info>; +defm : masked_move_for_extract<"VMOVAPDZ256", v8f64_info, v4f64x_info, v4f64x_info>; +defm : masked_move_for_extract<"VMOVAPDZ256", v16f32_info, v8f32x_info, v4f64x_info>; +defm : masked_move_for_extract<"VMOVAPSZ256", v8f64_info, v4f64x_info, v8f32x_info>; +defm : masked_move_for_extract<"VMOVAPSZ256", v16f32_info, v8f32x_info, v8f32x_info>; } // Move Int Doubleword to Packed Double Int diff --git a/llvm/test/CodeGen/X86/vector-shuffle-masked.ll b/llvm/test/CodeGen/X86/vector-shuffle-masked.ll index 3b82d191a0c6..33bed1861cf4 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-masked.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-masked.ll @@ -1017,6 +1017,32 @@ define <4 x double> @mask_extract_v8f64_v4f64_1_z(<8 x double> %a, i8 %mask) { ret <4 x double> %res } +define <8 x i32> @mask_cast_extract_v8i64_v8i32_0(<8 x i64> %a, <8 x i32> %passthru, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v8i64_v8i32_0: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpblendmd %ymm0, %ymm1, %ymm0 {%k1} +; CHECK-NEXT: retq + %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> + %shuffle.cast = bitcast <4 x i64> %shuffle to <8 x i32> + %mask.cast = bitcast i8 %mask to <8 x i1> + %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle.cast, <8 x i32> %passthru + ret <8 x i32> %res +} + +define <8 x i32> @mask_cast_extract_v8i64_v8i32_0_z(<8 x i64> %a, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v8i64_v8i32_0_z: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z} +; CHECK-NEXT: retq + %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> + %shuffle.cast = bitcast <4 x i64> %shuffle to <8 x i32> + %mask.cast = bitcast i8 %mask to <8 x i1> + %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle.cast, <8 x i32> zeroinitializer + ret <8 x i32> %res +} + define <8 x i32> @mask_cast_extract_v8i64_v8i32_1(<8 x i64> %a, <8 x i32> %passthru, i8 %mask) { ; CHECK-LABEL: mask_cast_extract_v8i64_v8i32_1: ; CHECK: # BB#0: @@ -1044,6 +1070,32 @@ define <8 x i32> @mask_cast_extract_v8i64_v8i32_1_z(<8 x i64> %a, i8 %mask) { ret <8 x i32> %res } +define <8 x float> @mask_cast_extract_v8f64_v8f32_0(<8 x double> %a, <8 x float> %passthru, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v8f64_v8f32_0: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vblendmps %ymm0, %ymm1, %ymm0 {%k1} +; CHECK-NEXT: retq + %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> + %shuffle.cast = bitcast <4 x double> %shuffle to <8 x float> + %mask.cast = bitcast i8 %mask to <8 x i1> + %res = select <8 x i1> %mask.cast, <8 x float> %shuffle.cast, <8 x float> %passthru + ret <8 x float> %res +} + +define <8 x float> @mask_cast_extract_v8f64_v8f32_0_z(<8 x double> %a, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v8f64_v8f32_0_z: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovaps %ymm0, %ymm0 {%k1} {z} +; CHECK-NEXT: retq + %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> + %shuffle.cast = bitcast <4 x double> %shuffle to <8 x float> + %mask.cast = bitcast i8 %mask to <8 x i1> + %res = select <8 x i1> %mask.cast, <8 x float> %shuffle.cast, <8 x float> zeroinitializer + ret <8 x float> %res +} + define <8 x float> @mask_cast_extract_v8f64_v8f32_1(<8 x double> %a, <8 x float> %passthru, i8 %mask) { ; CHECK-LABEL: mask_cast_extract_v8f64_v8f32_1: ; CHECK: # BB#0: @@ -1071,6 +1123,36 @@ define <8 x float> @mask_cast_extract_v8f64_v8f32_1_z(<8 x double> %a, i8 %mask) ret <8 x float> %res } +define <4 x i32> @mask_cast_extract_v8i64_v4i32_0(<8 x i64> %a, <4 x i32> %passthru, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v8i64_v4i32_0: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpblendmd %xmm0, %xmm1, %xmm0 {%k1} +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> + %shuffle.cast = bitcast <2 x i64> %shuffle to <4 x i32> + %mask.cast = bitcast i8 %mask to <8 x i1> + %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> + %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle.cast, <4 x i32> %passthru + ret <4 x i32> %res +} + +define <4 x i32> @mask_cast_extract_v8i64_v4i32_0_z(<8 x i64> %a, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v8i64_v4i32_0_z: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z} +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> + %shuffle.cast = bitcast <2 x i64> %shuffle to <4 x i32> + %mask.cast = bitcast i8 %mask to <8 x i1> + %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> + %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle.cast, <4 x i32> zeroinitializer + ret <4 x i32> %res +} + define <4 x i32> @mask_cast_extract_v8i64_v4i32_1(<8 x i64> %a, <4 x i32> %passthru, i8 %mask) { ; CHECK-LABEL: mask_cast_extract_v8i64_v4i32_1: ; CHECK: # BB#0: @@ -1102,6 +1184,36 @@ define <4 x i32> @mask_cast_extract_v8i64_v4i32_1_z(<8 x i64> %a, i8 %mask) { ret <4 x i32> %res } +define <4 x float> @mask_cast_extract_v8f64_v4f32_0(<8 x double> %a, <4 x float> %passthru, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v8f64_v4f32_0: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vblendmps %xmm0, %xmm1, %xmm0 {%k1} +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> + %shuffle.cast = bitcast <2 x double> %shuffle to <4 x float> + %mask.cast = bitcast i8 %mask to <8 x i1> + %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> + %res = select <4 x i1> %mask.extract, <4 x float> %shuffle.cast, <4 x float> %passthru + ret <4 x float> %res +} + +define <4 x float> @mask_cast_extract_v8f64_v4f32_0_z(<8 x double> %a, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v8f64_v4f32_0_z: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovaps %xmm0, %xmm0 {%k1} {z} +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> + %shuffle.cast = bitcast <2 x double> %shuffle to <4 x float> + %mask.cast = bitcast i8 %mask to <8 x i1> + %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> + %res = select <4 x i1> %mask.extract, <4 x float> %shuffle.cast, <4 x float> zeroinitializer + ret <4 x float> %res +} + define <4 x float> @mask_cast_extract_v8f64_v4f32_1(<8 x double> %a, <4 x float> %passthru, i8 %mask) { ; CHECK-LABEL: mask_cast_extract_v8f64_v4f32_1: ; CHECK: # BB#0: @@ -1133,6 +1245,34 @@ define <4 x float> @mask_cast_extract_v8f64_v4f32_1_z(<8 x double> %a, i8 %mask) ret <4 x float> %res } +define <4 x i64> @mask_cast_extract_v16i32_v4i64_0(<16 x i32> %a, <4 x i64> %passthru, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v16i32_v4i64_0: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpblendmq %ymm0, %ymm1, %ymm0 {%k1} +; CHECK-NEXT: retq + %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> + %shuffle.cast = bitcast <8 x i32> %shuffle to <4 x i64> + %mask.cast = bitcast i8 %mask to <8 x i1> + %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> + %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle.cast, <4 x i64> %passthru + ret <4 x i64> %res +} + +define <4 x i64> @mask_cast_extract_v16i32_v4i64_0_z(<16 x i32> %a, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v16i32_v4i64_0_z: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovdqa64 %ymm0, %ymm0 {%k1} {z} +; CHECK-NEXT: retq + %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> + %shuffle.cast = bitcast <8 x i32> %shuffle to <4 x i64> + %mask.cast = bitcast i8 %mask to <8 x i1> + %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> + %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle.cast, <4 x i64> zeroinitializer + ret <4 x i64> %res +} + define <4 x i64> @mask_cast_extract_v16i32_v4i64_1(<16 x i32> %a, <4 x i64> %passthru, i8 %mask) { ; CHECK-LABEL: mask_cast_extract_v16i32_v4i64_1: ; CHECK: # BB#0: @@ -1162,6 +1302,34 @@ define <4 x i64> @mask_cast_extract_v16i32_v4i64_1_z(<16 x i32> %a, i8 %mask) { ret <4 x i64> %res } +define <4 x double> @mask_cast_extract_v16f32_v4f64_0(<16 x float> %a, <4 x double> %passthru, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v16f32_v4f64_0: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vblendmpd %ymm0, %ymm1, %ymm0 {%k1} +; CHECK-NEXT: retq + %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> + %shuffle.cast = bitcast <8 x float> %shuffle to <4 x double> + %mask.cast = bitcast i8 %mask to <8 x i1> + %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> + %res = select <4 x i1> %mask.extract, <4 x double> %shuffle.cast, <4 x double> %passthru + ret <4 x double> %res +} + +define <4 x double> @mask_cast_extract_v16f32_v4f64_0_z(<16 x float> %a, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v16f32_v4f64_0_z: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovapd %ymm0, %ymm0 {%k1} {z} +; CHECK-NEXT: retq + %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> + %shuffle.cast = bitcast <8 x float> %shuffle to <4 x double> + %mask.cast = bitcast i8 %mask to <8 x i1> + %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> + %res = select <4 x i1> %mask.extract, <4 x double> %shuffle.cast, <4 x double> zeroinitializer + ret <4 x double> %res +} + define <4 x double> @mask_cast_extract_v16f32_v4f64_1(<16 x float> %a, <4 x double> %passthru, i8 %mask) { ; CHECK-LABEL: mask_cast_extract_v16f32_v4f64_1: ; CHECK: # BB#0: @@ -1191,6 +1359,36 @@ define <4 x double> @mask_cast_extract_v16f32_v4f64_1_z(<16 x float> %a, i8 %mas ret <4 x double> %res } +define <2 x i64> @mask_cast_extract_v16i32_v2i64_0(<16 x i32> %a, <2 x i64> %passthru, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v16i32_v2i64_0: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpblendmq %xmm0, %xmm1, %xmm0 {%k1} +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> + %shuffle.cast = bitcast <4 x i32> %shuffle to <2 x i64> + %mask.cast = bitcast i8 %mask to <8 x i1> + %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> + %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle.cast, <2 x i64> %passthru + ret <2 x i64> %res +} + +define <2 x i64> @mask_cast_extract_v16i32_v2i64_0_z(<16 x i32> %a, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v16i32_v2i64_0_z: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z} +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> + %shuffle.cast = bitcast <4 x i32> %shuffle to <2 x i64> + %mask.cast = bitcast i8 %mask to <8 x i1> + %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> + %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle.cast, <2 x i64> zeroinitializer + ret <2 x i64> %res +} + define <2 x i64> @mask_cast_extract_v16i32_v2i64_1(<16 x i32> %a, <2 x i64> %passthru, i8 %mask) { ; CHECK-LABEL: mask_cast_extract_v16i32_v2i64_1: ; CHECK: # BB#0: @@ -1222,6 +1420,36 @@ define <2 x i64> @mask_cast_extract_v16i32_v2i64_1_z(<16 x i32> %a, i8 %mask) { ret <2 x i64> %res } +define <2 x double> @mask_cast_extract_v16f32_v2f64_0(<16 x float> %a, <2 x double> %passthru, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v16f32_v2f64_0: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vblendmpd %xmm0, %xmm1, %xmm0 {%k1} +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> + %shuffle.cast = bitcast <4 x float> %shuffle to <2 x double> + %mask.cast = bitcast i8 %mask to <8 x i1> + %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> + %res = select <2 x i1> %mask.extract, <2 x double> %shuffle.cast, <2 x double> %passthru + ret <2 x double> %res +} + +define <2 x double> @mask_cast_extract_v16f32_v2f64_0_z(<16 x float> %a, i8 %mask) { +; CHECK-LABEL: mask_cast_extract_v16f32_v2f64_0_z: +; CHECK: # BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovapd %xmm0, %xmm0 {%k1} {z} +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> + %shuffle.cast = bitcast <4 x float> %shuffle to <2 x double> + %mask.cast = bitcast i8 %mask to <8 x i1> + %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> + %res = select <2 x i1> %mask.extract, <2 x double> %shuffle.cast, <2 x double> zeroinitializer + ret <2 x double> %res +} + define <2 x double> @mask_cast_extract_v16f32_v2f64_1(<16 x float> %a, <2 x double> %passthru, i8 %mask) { ; CHECK-LABEL: mask_cast_extract_v16f32_v2f64_1: ; CHECK: # BB#0: