forked from OSchip/llvm-project
1343 lines
102 KiB
C
1343 lines
102 KiB
C
// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
|
|
// RUN: %clang_cc1 -triple aarch64-arm-none-eabi -target-feature +neon -target-feature +bf16 \
|
|
// RUN: -O2 -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK64
|
|
// RUN: %clang_cc1 -triple armv8.6a-arm-none-eabi -target-feature +neon -target-feature +bf16 -mfloat-abi hard \
|
|
// RUN: -O2 -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK32
|
|
|
|
// REQUIRES: arm-registered-target,aarch64-registered-target
|
|
|
|
#include "arm_neon.h"
|
|
|
|
// CHECK-LABEL: @test_vld1_bf16(
|
|
// CHECK-NEXT: entry:
|
|
// CHECK-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to <4 x bfloat>*
|
|
// CHECK-NEXT: [[TMP1:%.*]] = load <4 x bfloat>, <4 x bfloat>* [[TMP0]], align 2
|
|
// CHECK-NEXT: ret <4 x bfloat> [[TMP1]]
|
|
//
|
|
bfloat16x4_t test_vld1_bf16(bfloat16_t const *ptr) {
|
|
return vld1_bf16(ptr);
|
|
}
|
|
|
|
// CHECK-LABEL: @test_vld1q_bf16(
|
|
// CHECK-NEXT: entry:
|
|
// CHECK-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to <8 x bfloat>*
|
|
// CHECK-NEXT: [[TMP1:%.*]] = load <8 x bfloat>, <8 x bfloat>* [[TMP0]], align 2
|
|
// CHECK-NEXT: ret <8 x bfloat> [[TMP1]]
|
|
//
|
|
bfloat16x8_t test_vld1q_bf16(bfloat16_t const *ptr) {
|
|
return vld1q_bf16(ptr);
|
|
}
|
|
|
|
// CHECK-LABEL: @test_vld1_lane_bf16(
|
|
// CHECK-NEXT: entry:
|
|
// CHECK-NEXT: [[TMP0:%.*]] = load bfloat, bfloat* [[PTR:%.*]], align 2
|
|
// CHECK-NEXT: [[VLD1_LANE:%.*]] = insertelement <4 x bfloat> [[SRC:%.*]], bfloat [[TMP0]], i64 0
|
|
// CHECK-NEXT: ret <4 x bfloat> [[VLD1_LANE]]
|
|
//
|
|
bfloat16x4_t test_vld1_lane_bf16(bfloat16_t const *ptr, bfloat16x4_t src) {
|
|
return vld1_lane_bf16(ptr, src, 0);
|
|
}
|
|
|
|
// CHECK-LABEL: @test_vld1q_lane_bf16(
|
|
// CHECK-NEXT: entry:
|
|
// CHECK-NEXT: [[TMP0:%.*]] = load bfloat, bfloat* [[PTR:%.*]], align 2
|
|
// CHECK-NEXT: [[VLD1_LANE:%.*]] = insertelement <8 x bfloat> [[SRC:%.*]], bfloat [[TMP0]], i64 7
|
|
// CHECK-NEXT: ret <8 x bfloat> [[VLD1_LANE]]
|
|
//
|
|
bfloat16x8_t test_vld1q_lane_bf16(bfloat16_t const *ptr, bfloat16x8_t src) {
|
|
return vld1q_lane_bf16(ptr, src, 7);
|
|
}
|
|
|
|
// CHECK-LABEL: @test_vld1_dup_bf16(
|
|
// CHECK-NEXT: entry:
|
|
// CHECK-NEXT: [[TMP0:%.*]] = load bfloat, bfloat* [[PTR:%.*]], align 2
|
|
// CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x bfloat> undef, bfloat [[TMP0]], i64 0
|
|
// CHECK-NEXT: [[LANE:%.*]] = shufflevector <4 x bfloat> [[TMP1]], <4 x bfloat> poison, <4 x i32> zeroinitializer
|
|
// CHECK-NEXT: ret <4 x bfloat> [[LANE]]
|
|
//
|
|
bfloat16x4_t test_vld1_dup_bf16(bfloat16_t const *ptr) {
|
|
return vld1_dup_bf16(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld1_bf16_x2(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VLD1XN:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld1x2.v4bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 0
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T:%.*]] poison, <4 x bfloat> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_1_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld1_bf16_x2(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VLD1XN:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld1x2.v4bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 0
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 1
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_0_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_1_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <2 x i32>] poison, <2 x i32> [[TMP0]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP1]], 1
|
|
// CHECK32-NEXT: ret [2 x <2 x i32>] [[DOTFCA_1_INSERT]]
|
|
//
|
|
bfloat16x4x2_t test_vld1_bf16_x2(bfloat16_t const *ptr) {
|
|
return vld1_bf16_x2(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld1q_bf16_x2(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VLD1XN:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld1x2.v8bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 0
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T:%.*]] poison, <8 x bfloat> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld1q_bf16_x2(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VLD1XN:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld1x2.v8bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 0
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 1
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_0_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_1_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <4 x i32>] poison, <4 x i32> [[TMP0]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP1]], 1
|
|
// CHECK32-NEXT: ret [2 x <4 x i32>] [[DOTFCA_1_INSERT]]
|
|
//
|
|
bfloat16x8x2_t test_vld1q_bf16_x2(bfloat16_t const *ptr) {
|
|
return vld1q_bf16_x2(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld1_bf16_x3(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VLD1XN:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld1x3.v4bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 0
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 1
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T:%.*]] poison, <4 x bfloat> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD1XN_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_2_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld1_bf16_x3(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VLD1XN:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld1x3.v4bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 0
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 1
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 2
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_0_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_1_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_2_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <2 x i32>] poison, <2 x i32> [[TMP0]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP1]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP2]], 2
|
|
// CHECK32-NEXT: ret [3 x <2 x i32>] [[DOTFCA_2_INSERT]]
|
|
//
|
|
bfloat16x4x3_t test_vld1_bf16_x3(bfloat16_t const *ptr) {
|
|
return vld1_bf16_x3(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld1q_bf16_x3(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VLD1XN:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld1x3.v8bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 0
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 1
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T:%.*]] poison, <8 x bfloat> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD1XN_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_2_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld1q_bf16_x3(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VLD1XN:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld1x3.v8bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 0
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 1
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 2
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_0_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_1_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_2_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <4 x i32>] poison, <4 x i32> [[TMP0]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP1]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP2]], 2
|
|
// CHECK32-NEXT: ret [3 x <4 x i32>] [[DOTFCA_2_INSERT]]
|
|
//
|
|
bfloat16x8x3_t test_vld1q_bf16_x3(bfloat16_t const *ptr) {
|
|
return vld1q_bf16_x3(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld1_bf16_x4(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VLD1XN:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld1x4.v4bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 0
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 1
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 2
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 3
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T:%.*]] poison, <4 x bfloat> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD1XN_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_2_INSERT]], <4 x bfloat> [[VLD1XN_FCA_3_EXTRACT]], 0, 3
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_3_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld1_bf16_x4(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VLD1XN:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld1x4.v4bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 0
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 1
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 2
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 3
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_0_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_1_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_2_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_3_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <2 x i32>] poison, <2 x i32> [[TMP0]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP1]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP2]], 2
|
|
// CHECK32-NEXT: [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_2_INSERT]], <2 x i32> [[TMP3]], 3
|
|
// CHECK32-NEXT: ret [4 x <2 x i32>] [[DOTFCA_3_INSERT]]
|
|
//
|
|
bfloat16x4x4_t test_vld1_bf16_x4(bfloat16_t const *ptr) {
|
|
return vld1_bf16_x4(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld1q_bf16_x4(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VLD1XN:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld1x4.v8bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 0
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 1
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 2
|
|
// CHECK64-NEXT: [[VLD1XN_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 3
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T:%.*]] poison, <8 x bfloat> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD1XN_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x bfloat> [[VLD1XN_FCA_3_EXTRACT]], 0, 3
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_3_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld1q_bf16_x4(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VLD1XN:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld1x4.v8bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 0
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 1
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 2
|
|
// CHECK32-NEXT: [[VLD1XN_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 3
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_0_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_1_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_2_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_3_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <4 x i32>] poison, <4 x i32> [[TMP0]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP1]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP2]], 2
|
|
// CHECK32-NEXT: [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_2_INSERT]], <4 x i32> [[TMP3]], 3
|
|
// CHECK32-NEXT: ret [4 x <4 x i32>] [[DOTFCA_3_INSERT]]
|
|
//
|
|
bfloat16x8x4_t test_vld1q_bf16_x4(bfloat16_t const *ptr) {
|
|
return vld1q_bf16_x4(ptr);
|
|
}
|
|
|
|
// CHECK-LABEL: @test_vld1q_dup_bf16(
|
|
// CHECK-NEXT: entry:
|
|
// CHECK-NEXT: [[TMP0:%.*]] = load bfloat, bfloat* [[PTR:%.*]], align 2
|
|
// CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x bfloat> undef, bfloat [[TMP0]], i64 0
|
|
// CHECK-NEXT: [[LANE:%.*]] = shufflevector <8 x bfloat> [[TMP1]], <8 x bfloat> poison, <8 x i32> zeroinitializer
|
|
// CHECK-NEXT: ret <8 x bfloat> [[LANE]]
|
|
//
|
|
bfloat16x8_t test_vld1q_dup_bf16(bfloat16_t const *ptr) {
|
|
return vld1q_dup_bf16(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld2_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to <4 x bfloat>*
|
|
// CHECK64-NEXT: [[VLD2:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld2.v4bf16.p0v4bf16(<4 x bfloat>* [[TMP0]])
|
|
// CHECK64-NEXT: [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2]], 0
|
|
// CHECK64-NEXT: [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2]], 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T:%.*]] poison, <4 x bfloat> [[VLD2_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD2_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_1_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld2_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD2_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld2.v4bf16.p0i8(i8* [[TMP0]], i32 2)
|
|
// CHECK32-NEXT: [[VLD2_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_V]], 0
|
|
// CHECK32-NEXT: [[VLD2_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_V]], 1
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD2_V_FCA_0_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD2_V_FCA_1_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <2 x i32>] poison, <2 x i32> [[TMP1]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP2]], 1
|
|
// CHECK32-NEXT: ret [2 x <2 x i32>] [[DOTFCA_1_INSERT]]
|
|
//
|
|
bfloat16x4x2_t test_vld2_bf16(bfloat16_t const *ptr) {
|
|
return vld2_bf16(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld2q_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to <8 x bfloat>*
|
|
// CHECK64-NEXT: [[VLD2:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld2.v8bf16.p0v8bf16(<8 x bfloat>* [[TMP0]])
|
|
// CHECK64-NEXT: [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2]], 0
|
|
// CHECK64-NEXT: [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2]], 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T:%.*]] poison, <8 x bfloat> [[VLD2_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD2_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld2q_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD2Q_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld2.v8bf16.p0i8(i8* [[TMP0]], i32 2)
|
|
// CHECK32-NEXT: [[VLD2Q_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2Q_V]], 0
|
|
// CHECK32-NEXT: [[VLD2Q_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2Q_V]], 1
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD2Q_V_FCA_0_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD2Q_V_FCA_1_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <4 x i32>] poison, <4 x i32> [[TMP1]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP2]], 1
|
|
// CHECK32-NEXT: ret [2 x <4 x i32>] [[DOTFCA_1_INSERT]]
|
|
//
|
|
bfloat16x8x2_t test_vld2q_bf16(bfloat16_t const *ptr) {
|
|
return vld2q_bf16(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld2_lane_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[SRC_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[SRC_COERCE]], 1
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: [[VLD2_LANE:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld2lane.v4bf16.p0i8(<4 x bfloat> [[SRC_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[SRC_COERCE_FCA_1_EXTRACT]], i64 1, i8* [[TMP0]])
|
|
// CHECK64-NEXT: [[VLD2_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_LANE]], 0
|
|
// CHECK64-NEXT: [[VLD2_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_LANE]], 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T:%.*]] poison, <4 x bfloat> [[VLD2_LANE_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD2_LANE_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_1_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld2_lane_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[SRC_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[SRC_COERCE]], 1
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD2_LANE_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld2lane.v4bf16.p0i8(i8* [[TMP2]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], i32 1, i32 2)
|
|
// CHECK32-NEXT: [[VLD2_LANE_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_LANE_V]], 0
|
|
// CHECK32-NEXT: [[VLD2_LANE_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_LANE_V]], 1
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <4 x bfloat> [[VLD2_LANE_V_FCA_0_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP4:%.*]] = bitcast <4 x bfloat> [[VLD2_LANE_V_FCA_1_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <2 x i32>] poison, <2 x i32> [[TMP3]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP4]], 1
|
|
// CHECK32-NEXT: ret [2 x <2 x i32>] [[DOTFCA_1_INSERT]]
|
|
//
|
|
bfloat16x4x2_t test_vld2_lane_bf16(bfloat16_t const *ptr, bfloat16x4x2_t src) {
|
|
return vld2_lane_bf16(ptr, src, 1);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld2q_lane_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[SRC_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[SRC_COERCE]], 1
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: [[VLD2_LANE:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld2lane.v8bf16.p0i8(<8 x bfloat> [[SRC_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[SRC_COERCE_FCA_1_EXTRACT]], i64 7, i8* [[TMP0]])
|
|
// CHECK64-NEXT: [[VLD2_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2_LANE]], 0
|
|
// CHECK64-NEXT: [[VLD2_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2_LANE]], 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T:%.*]] poison, <8 x bfloat> [[VLD2_LANE_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD2_LANE_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld2q_lane_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[SRC_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[SRC_COERCE]], 1
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD2Q_LANE_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld2lane.v8bf16.p0i8(i8* [[TMP2]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], i32 7, i32 2)
|
|
// CHECK32-NEXT: [[VLD2Q_LANE_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2Q_LANE_V]], 0
|
|
// CHECK32-NEXT: [[VLD2Q_LANE_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2Q_LANE_V]], 1
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <8 x bfloat> [[VLD2Q_LANE_V_FCA_0_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP4:%.*]] = bitcast <8 x bfloat> [[VLD2Q_LANE_V_FCA_1_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <4 x i32>] poison, <4 x i32> [[TMP3]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP4]], 1
|
|
// CHECK32-NEXT: ret [2 x <4 x i32>] [[DOTFCA_1_INSERT]]
|
|
//
|
|
bfloat16x8x2_t test_vld2q_lane_bf16(bfloat16_t const *ptr, bfloat16x8x2_t src) {
|
|
return vld2q_lane_bf16(ptr, src, 7);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld3_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to <4 x bfloat>*
|
|
// CHECK64-NEXT: [[VLD3:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld3.v4bf16.p0v4bf16(<4 x bfloat>* [[TMP0]])
|
|
// CHECK64-NEXT: [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3]], 0
|
|
// CHECK64-NEXT: [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3]], 1
|
|
// CHECK64-NEXT: [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3]], 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T:%.*]] poison, <4 x bfloat> [[VLD3_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD3_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD3_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_2_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld3_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD3_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld3.v4bf16.p0i8(i8* [[TMP0]], i32 2)
|
|
// CHECK32-NEXT: [[VLD3_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_V]], 0
|
|
// CHECK32-NEXT: [[VLD3_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_V]], 1
|
|
// CHECK32-NEXT: [[VLD3_V_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_V]], 2
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD3_V_FCA_0_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD3_V_FCA_1_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <4 x bfloat> [[VLD3_V_FCA_2_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <2 x i32>] poison, <2 x i32> [[TMP1]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP2]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP3]], 2
|
|
// CHECK32-NEXT: ret [3 x <2 x i32>] [[DOTFCA_2_INSERT]]
|
|
//
|
|
bfloat16x4x3_t test_vld3_bf16(bfloat16_t const *ptr) {
|
|
return vld3_bf16(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld3q_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to <8 x bfloat>*
|
|
// CHECK64-NEXT: [[VLD3:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld3.v8bf16.p0v8bf16(<8 x bfloat>* [[TMP0]])
|
|
// CHECK64-NEXT: [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3]], 0
|
|
// CHECK64-NEXT: [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3]], 1
|
|
// CHECK64-NEXT: [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3]], 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T:%.*]] poison, <8 x bfloat> [[VLD3_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD3_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD3_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_2_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld3q_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD3Q_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld3.v8bf16.p0i8(i8* [[TMP0]], i32 2)
|
|
// CHECK32-NEXT: [[VLD3Q_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_V]], 0
|
|
// CHECK32-NEXT: [[VLD3Q_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_V]], 1
|
|
// CHECK32-NEXT: [[VLD3Q_V_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_V]], 2
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD3Q_V_FCA_0_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD3Q_V_FCA_1_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <8 x bfloat> [[VLD3Q_V_FCA_2_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <4 x i32>] poison, <4 x i32> [[TMP1]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP2]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP3]], 2
|
|
// CHECK32-NEXT: ret [3 x <4 x i32>] [[DOTFCA_2_INSERT]]
|
|
//
|
|
bfloat16x8x3_t test_vld3q_bf16(bfloat16_t const *ptr) {
|
|
return vld3q_bf16(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld3_lane_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[SRC_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[SRC_COERCE]], 1
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[SRC_COERCE]], 2
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: [[VLD3_LANE:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld3lane.v4bf16.p0i8(<4 x bfloat> [[SRC_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[SRC_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[SRC_COERCE_FCA_2_EXTRACT]], i64 1, i8* [[TMP0]])
|
|
// CHECK64-NEXT: [[VLD3_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_LANE]], 0
|
|
// CHECK64-NEXT: [[VLD3_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_LANE]], 1
|
|
// CHECK64-NEXT: [[VLD3_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_LANE]], 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T:%.*]] poison, <4 x bfloat> [[VLD3_LANE_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD3_LANE_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD3_LANE_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_2_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld3_lane_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[SRC_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[SRC_COERCE]], 1
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[SRC_COERCE]], 2
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD3_LANE_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld3lane.v4bf16.p0i8(i8* [[TMP3]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]], i32 1, i32 2)
|
|
// CHECK32-NEXT: [[VLD3_LANE_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_LANE_V]], 0
|
|
// CHECK32-NEXT: [[VLD3_LANE_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_LANE_V]], 1
|
|
// CHECK32-NEXT: [[VLD3_LANE_V_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_LANE_V]], 2
|
|
// CHECK32-NEXT: [[TMP4:%.*]] = bitcast <4 x bfloat> [[VLD3_LANE_V_FCA_0_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP5:%.*]] = bitcast <4 x bfloat> [[VLD3_LANE_V_FCA_1_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP6:%.*]] = bitcast <4 x bfloat> [[VLD3_LANE_V_FCA_2_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <2 x i32>] poison, <2 x i32> [[TMP4]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP5]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP6]], 2
|
|
// CHECK32-NEXT: ret [3 x <2 x i32>] [[DOTFCA_2_INSERT]]
|
|
//
|
|
bfloat16x4x3_t test_vld3_lane_bf16(bfloat16_t const *ptr, bfloat16x4x3_t src) {
|
|
return vld3_lane_bf16(ptr, src, 1);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld3q_lane_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[SRC_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[SRC_COERCE]], 1
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[SRC_COERCE]], 2
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: [[VLD3_LANE:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld3lane.v8bf16.p0i8(<8 x bfloat> [[SRC_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[SRC_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[SRC_COERCE_FCA_2_EXTRACT]], i64 7, i8* [[TMP0]])
|
|
// CHECK64-NEXT: [[VLD3_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3_LANE]], 0
|
|
// CHECK64-NEXT: [[VLD3_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3_LANE]], 1
|
|
// CHECK64-NEXT: [[VLD3_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3_LANE]], 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T:%.*]] poison, <8 x bfloat> [[VLD3_LANE_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD3_LANE_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD3_LANE_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_2_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld3q_lane_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[SRC_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[SRC_COERCE]], 1
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[SRC_COERCE]], 2
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD3Q_LANE_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld3lane.v8bf16.p0i8(i8* [[TMP3]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]], i32 7, i32 2)
|
|
// CHECK32-NEXT: [[VLD3Q_LANE_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_LANE_V]], 0
|
|
// CHECK32-NEXT: [[VLD3Q_LANE_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_LANE_V]], 1
|
|
// CHECK32-NEXT: [[VLD3Q_LANE_V_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_LANE_V]], 2
|
|
// CHECK32-NEXT: [[TMP4:%.*]] = bitcast <8 x bfloat> [[VLD3Q_LANE_V_FCA_0_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP5:%.*]] = bitcast <8 x bfloat> [[VLD3Q_LANE_V_FCA_1_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP6:%.*]] = bitcast <8 x bfloat> [[VLD3Q_LANE_V_FCA_2_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <4 x i32>] poison, <4 x i32> [[TMP4]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP5]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP6]], 2
|
|
// CHECK32-NEXT: ret [3 x <4 x i32>] [[DOTFCA_2_INSERT]]
|
|
//
|
|
bfloat16x8x3_t test_vld3q_lane_bf16(bfloat16_t const *ptr, bfloat16x8x3_t src) {
|
|
return vld3q_lane_bf16(ptr, src, 7);
|
|
// return vld3q_lane_bf16(ptr, src, 8);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld4_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to <4 x bfloat>*
|
|
// CHECK64-NEXT: [[VLD4:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld4.v4bf16.p0v4bf16(<4 x bfloat>* [[TMP0]])
|
|
// CHECK64-NEXT: [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 0
|
|
// CHECK64-NEXT: [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 1
|
|
// CHECK64-NEXT: [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 2
|
|
// CHECK64-NEXT: [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 3
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T:%.*]] poison, <4 x bfloat> [[VLD4_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD4_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD4_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_2_INSERT]], <4 x bfloat> [[VLD4_FCA_3_EXTRACT]], 0, 3
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_3_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld4_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD4_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld4.v4bf16.p0i8(i8* [[TMP0]], i32 2)
|
|
// CHECK32-NEXT: [[VLD4_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_V]], 0
|
|
// CHECK32-NEXT: [[VLD4_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_V]], 1
|
|
// CHECK32-NEXT: [[VLD4_V_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_V]], 2
|
|
// CHECK32-NEXT: [[VLD4_V_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_V]], 3
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD4_V_FCA_0_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD4_V_FCA_1_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <4 x bfloat> [[VLD4_V_FCA_2_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP4:%.*]] = bitcast <4 x bfloat> [[VLD4_V_FCA_3_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <2 x i32>] poison, <2 x i32> [[TMP1]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP2]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP3]], 2
|
|
// CHECK32-NEXT: [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_2_INSERT]], <2 x i32> [[TMP4]], 3
|
|
// CHECK32-NEXT: ret [4 x <2 x i32>] [[DOTFCA_3_INSERT]]
|
|
//
|
|
bfloat16x4x4_t test_vld4_bf16(bfloat16_t const *ptr) {
|
|
return vld4_bf16(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld4q_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to <8 x bfloat>*
|
|
// CHECK64-NEXT: [[VLD4:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld4.v8bf16.p0v8bf16(<8 x bfloat>* [[TMP0]])
|
|
// CHECK64-NEXT: [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 0
|
|
// CHECK64-NEXT: [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 1
|
|
// CHECK64-NEXT: [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 2
|
|
// CHECK64-NEXT: [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 3
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T:%.*]] poison, <8 x bfloat> [[VLD4_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD4_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD4_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x bfloat> [[VLD4_FCA_3_EXTRACT]], 0, 3
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_3_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld4q_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD4Q_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld4.v8bf16.p0i8(i8* [[TMP0]], i32 2)
|
|
// CHECK32-NEXT: [[VLD4Q_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_V]], 0
|
|
// CHECK32-NEXT: [[VLD4Q_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_V]], 1
|
|
// CHECK32-NEXT: [[VLD4Q_V_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_V]], 2
|
|
// CHECK32-NEXT: [[VLD4Q_V_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_V]], 3
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD4Q_V_FCA_0_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD4Q_V_FCA_1_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <8 x bfloat> [[VLD4Q_V_FCA_2_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP4:%.*]] = bitcast <8 x bfloat> [[VLD4Q_V_FCA_3_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <4 x i32>] poison, <4 x i32> [[TMP1]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP2]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP3]], 2
|
|
// CHECK32-NEXT: [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_2_INSERT]], <4 x i32> [[TMP4]], 3
|
|
// CHECK32-NEXT: ret [4 x <4 x i32>] [[DOTFCA_3_INSERT]]
|
|
//
|
|
bfloat16x8x4_t test_vld4q_bf16(bfloat16_t const *ptr) {
|
|
return vld4q_bf16(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld4_lane_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[SRC_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[SRC_COERCE]], 1
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[SRC_COERCE]], 2
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[SRC_COERCE]], 3
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: [[VLD4_LANE:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld4lane.v4bf16.p0i8(<4 x bfloat> [[SRC_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[SRC_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[SRC_COERCE_FCA_2_EXTRACT]], <4 x bfloat> [[SRC_COERCE_FCA_3_EXTRACT]], i64 1, i8* [[TMP0]])
|
|
// CHECK64-NEXT: [[VLD4_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE]], 0
|
|
// CHECK64-NEXT: [[VLD4_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE]], 1
|
|
// CHECK64-NEXT: [[VLD4_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE]], 2
|
|
// CHECK64-NEXT: [[VLD4_LANE_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE]], 3
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T:%.*]] poison, <4 x bfloat> [[VLD4_LANE_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD4_LANE_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD4_LANE_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_2_INSERT]], <4 x bfloat> [[VLD4_LANE_FCA_3_EXTRACT]], 0, 3
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_3_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld4_lane_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[SRC_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[SRC_COERCE]], 1
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[SRC_COERCE]], 2
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[SRC_COERCE]], 3
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_3_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP4:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD4_LANE_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld4lane.v4bf16.p0i8(i8* [[TMP4]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]], <4 x bfloat> [[TMP3]], i32 1, i32 2)
|
|
// CHECK32-NEXT: [[VLD4_LANE_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE_V]], 0
|
|
// CHECK32-NEXT: [[VLD4_LANE_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE_V]], 1
|
|
// CHECK32-NEXT: [[VLD4_LANE_V_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE_V]], 2
|
|
// CHECK32-NEXT: [[VLD4_LANE_V_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE_V]], 3
|
|
// CHECK32-NEXT: [[TMP5:%.*]] = bitcast <4 x bfloat> [[VLD4_LANE_V_FCA_0_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP6:%.*]] = bitcast <4 x bfloat> [[VLD4_LANE_V_FCA_1_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP7:%.*]] = bitcast <4 x bfloat> [[VLD4_LANE_V_FCA_2_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP8:%.*]] = bitcast <4 x bfloat> [[VLD4_LANE_V_FCA_3_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <2 x i32>] poison, <2 x i32> [[TMP5]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP6]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP7]], 2
|
|
// CHECK32-NEXT: [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_2_INSERT]], <2 x i32> [[TMP8]], 3
|
|
// CHECK32-NEXT: ret [4 x <2 x i32>] [[DOTFCA_3_INSERT]]
|
|
//
|
|
bfloat16x4x4_t test_vld4_lane_bf16(bfloat16_t const *ptr, bfloat16x4x4_t src) {
|
|
return vld4_lane_bf16(ptr, src, 1);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld4q_lane_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[SRC_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[SRC_COERCE]], 1
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[SRC_COERCE]], 2
|
|
// CHECK64-NEXT: [[SRC_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[SRC_COERCE]], 3
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: [[VLD4_LANE:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld4lane.v8bf16.p0i8(<8 x bfloat> [[SRC_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[SRC_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[SRC_COERCE_FCA_2_EXTRACT]], <8 x bfloat> [[SRC_COERCE_FCA_3_EXTRACT]], i64 7, i8* [[TMP0]])
|
|
// CHECK64-NEXT: [[VLD4_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4_LANE]], 0
|
|
// CHECK64-NEXT: [[VLD4_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4_LANE]], 1
|
|
// CHECK64-NEXT: [[VLD4_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4_LANE]], 2
|
|
// CHECK64-NEXT: [[VLD4_LANE_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4_LANE]], 3
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T:%.*]] poison, <8 x bfloat> [[VLD4_LANE_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD4_LANE_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD4_LANE_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x bfloat> [[VLD4_LANE_FCA_3_EXTRACT]], 0, 3
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_3_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld4q_lane_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[SRC_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[SRC_COERCE]], 1
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[SRC_COERCE]], 2
|
|
// CHECK32-NEXT: [[SRC_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[SRC_COERCE]], 3
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_3_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP4:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD4Q_LANE_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld4lane.v8bf16.p0i8(i8* [[TMP4]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]], <8 x bfloat> [[TMP3]], i32 7, i32 2)
|
|
// CHECK32-NEXT: [[VLD4Q_LANE_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_LANE_V]], 0
|
|
// CHECK32-NEXT: [[VLD4Q_LANE_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_LANE_V]], 1
|
|
// CHECK32-NEXT: [[VLD4Q_LANE_V_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_LANE_V]], 2
|
|
// CHECK32-NEXT: [[VLD4Q_LANE_V_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_LANE_V]], 3
|
|
// CHECK32-NEXT: [[TMP5:%.*]] = bitcast <8 x bfloat> [[VLD4Q_LANE_V_FCA_0_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP6:%.*]] = bitcast <8 x bfloat> [[VLD4Q_LANE_V_FCA_1_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP7:%.*]] = bitcast <8 x bfloat> [[VLD4Q_LANE_V_FCA_2_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP8:%.*]] = bitcast <8 x bfloat> [[VLD4Q_LANE_V_FCA_3_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <4 x i32>] poison, <4 x i32> [[TMP5]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP6]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP7]], 2
|
|
// CHECK32-NEXT: [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_2_INSERT]], <4 x i32> [[TMP8]], 3
|
|
// CHECK32-NEXT: ret [4 x <4 x i32>] [[DOTFCA_3_INSERT]]
|
|
//
|
|
bfloat16x8x4_t test_vld4q_lane_bf16(bfloat16_t const *ptr, bfloat16x8x4_t src) {
|
|
return vld4q_lane_bf16(ptr, src, 7);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld2_dup_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VLD2:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld2r.v4bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2]], 0
|
|
// CHECK64-NEXT: [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2]], 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T:%.*]] poison, <4 x bfloat> [[VLD2_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD2_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_1_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld2_dup_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD2_DUP_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld2dup.v4bf16.p0i8(i8* [[TMP0]], i32 2)
|
|
// CHECK32-NEXT: [[VLD2_DUP_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_DUP_V]], 0
|
|
// CHECK32-NEXT: [[VLD2_DUP_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_DUP_V]], 1
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD2_DUP_V_FCA_0_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD2_DUP_V_FCA_1_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <2 x i32>] poison, <2 x i32> [[TMP1]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP2]], 1
|
|
// CHECK32-NEXT: ret [2 x <2 x i32>] [[DOTFCA_1_INSERT]]
|
|
//
|
|
bfloat16x4x2_t test_vld2_dup_bf16(bfloat16_t const *ptr) {
|
|
return vld2_dup_bf16(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld2q_dup_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VLD2:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld2r.v8bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2]], 0
|
|
// CHECK64-NEXT: [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2]], 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T:%.*]] poison, <8 x bfloat> [[VLD2_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD2_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld2q_dup_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD2Q_DUP_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld2dup.v8bf16.p0i8(i8* [[TMP0]], i32 2)
|
|
// CHECK32-NEXT: [[VLD2Q_DUP_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2Q_DUP_V]], 0
|
|
// CHECK32-NEXT: [[VLD2Q_DUP_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2Q_DUP_V]], 1
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD2Q_DUP_V_FCA_0_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD2Q_DUP_V_FCA_1_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <4 x i32>] poison, <4 x i32> [[TMP1]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP2]], 1
|
|
// CHECK32-NEXT: ret [2 x <4 x i32>] [[DOTFCA_1_INSERT]]
|
|
//
|
|
bfloat16x8x2_t test_vld2q_dup_bf16(bfloat16_t const *ptr) {
|
|
return vld2q_dup_bf16(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld3_dup_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VLD3:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld3r.v4bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3]], 0
|
|
// CHECK64-NEXT: [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3]], 1
|
|
// CHECK64-NEXT: [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3]], 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T:%.*]] poison, <4 x bfloat> [[VLD3_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD3_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD3_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_2_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld3_dup_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD3_DUP_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld3dup.v4bf16.p0i8(i8* [[TMP0]], i32 2)
|
|
// CHECK32-NEXT: [[VLD3_DUP_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_DUP_V]], 0
|
|
// CHECK32-NEXT: [[VLD3_DUP_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_DUP_V]], 1
|
|
// CHECK32-NEXT: [[VLD3_DUP_V_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_DUP_V]], 2
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD3_DUP_V_FCA_0_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD3_DUP_V_FCA_1_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <4 x bfloat> [[VLD3_DUP_V_FCA_2_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <2 x i32>] poison, <2 x i32> [[TMP1]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP2]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP3]], 2
|
|
// CHECK32-NEXT: ret [3 x <2 x i32>] [[DOTFCA_2_INSERT]]
|
|
//
|
|
bfloat16x4x3_t test_vld3_dup_bf16(bfloat16_t const *ptr) {
|
|
return vld3_dup_bf16(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld3q_dup_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VLD3:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld3r.v8bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3]], 0
|
|
// CHECK64-NEXT: [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3]], 1
|
|
// CHECK64-NEXT: [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3]], 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T:%.*]] poison, <8 x bfloat> [[VLD3_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD3_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD3_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_2_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld3q_dup_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD3Q_DUP_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld3dup.v8bf16.p0i8(i8* [[TMP0]], i32 2)
|
|
// CHECK32-NEXT: [[VLD3Q_DUP_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_DUP_V]], 0
|
|
// CHECK32-NEXT: [[VLD3Q_DUP_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_DUP_V]], 1
|
|
// CHECK32-NEXT: [[VLD3Q_DUP_V_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_DUP_V]], 2
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD3Q_DUP_V_FCA_0_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD3Q_DUP_V_FCA_1_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <8 x bfloat> [[VLD3Q_DUP_V_FCA_2_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <4 x i32>] poison, <4 x i32> [[TMP1]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP2]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP3]], 2
|
|
// CHECK32-NEXT: ret [3 x <4 x i32>] [[DOTFCA_2_INSERT]]
|
|
//
|
|
bfloat16x8x3_t test_vld3q_dup_bf16(bfloat16_t const *ptr) {
|
|
return vld3q_dup_bf16(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld4_dup_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VLD4:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld4r.v4bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 0
|
|
// CHECK64-NEXT: [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 1
|
|
// CHECK64-NEXT: [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 2
|
|
// CHECK64-NEXT: [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 3
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T:%.*]] poison, <4 x bfloat> [[VLD4_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD4_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD4_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_2_INSERT]], <4 x bfloat> [[VLD4_FCA_3_EXTRACT]], 0, 3
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_3_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld4_dup_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD4_DUP_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld4dup.v4bf16.p0i8(i8* [[TMP0]], i32 2)
|
|
// CHECK32-NEXT: [[VLD4_DUP_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_DUP_V]], 0
|
|
// CHECK32-NEXT: [[VLD4_DUP_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_DUP_V]], 1
|
|
// CHECK32-NEXT: [[VLD4_DUP_V_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_DUP_V]], 2
|
|
// CHECK32-NEXT: [[VLD4_DUP_V_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_DUP_V]], 3
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD4_DUP_V_FCA_0_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD4_DUP_V_FCA_1_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <4 x bfloat> [[VLD4_DUP_V_FCA_2_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[TMP4:%.*]] = bitcast <4 x bfloat> [[VLD4_DUP_V_FCA_3_EXTRACT]] to <2 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <2 x i32>] poison, <2 x i32> [[TMP1]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP2]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP3]], 2
|
|
// CHECK32-NEXT: [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_2_INSERT]], <2 x i32> [[TMP4]], 3
|
|
// CHECK32-NEXT: ret [4 x <2 x i32>] [[DOTFCA_3_INSERT]]
|
|
//
|
|
bfloat16x4x4_t test_vld4_dup_bf16(bfloat16_t const *ptr) {
|
|
return vld4_dup_bf16(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vld4q_dup_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VLD4:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld4r.v8bf16.p0bf16(bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 0
|
|
// CHECK64-NEXT: [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 1
|
|
// CHECK64-NEXT: [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 2
|
|
// CHECK64-NEXT: [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 3
|
|
// CHECK64-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T:%.*]] poison, <8 x bfloat> [[VLD4_FCA_0_EXTRACT]], 0, 0
|
|
// CHECK64-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD4_FCA_1_EXTRACT]], 0, 1
|
|
// CHECK64-NEXT: [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD4_FCA_2_EXTRACT]], 0, 2
|
|
// CHECK64-NEXT: [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x bfloat> [[VLD4_FCA_3_EXTRACT]], 0, 3
|
|
// CHECK64-NEXT: ret [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_3_INSERT]]
|
|
//
|
|
// CHECK32-LABEL: @test_vld4q_dup_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: [[VLD4Q_DUP_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld4dup.v8bf16.p0i8(i8* [[TMP0]], i32 2)
|
|
// CHECK32-NEXT: [[VLD4Q_DUP_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_DUP_V]], 0
|
|
// CHECK32-NEXT: [[VLD4Q_DUP_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_DUP_V]], 1
|
|
// CHECK32-NEXT: [[VLD4Q_DUP_V_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_DUP_V]], 2
|
|
// CHECK32-NEXT: [[VLD4Q_DUP_V_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_DUP_V]], 3
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD4Q_DUP_V_FCA_0_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD4Q_DUP_V_FCA_1_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <8 x bfloat> [[VLD4Q_DUP_V_FCA_2_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[TMP4:%.*]] = bitcast <8 x bfloat> [[VLD4Q_DUP_V_FCA_3_EXTRACT]] to <4 x i32>
|
|
// CHECK32-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <4 x i32>] poison, <4 x i32> [[TMP1]], 0
|
|
// CHECK32-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP2]], 1
|
|
// CHECK32-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP3]], 2
|
|
// CHECK32-NEXT: [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_2_INSERT]], <4 x i32> [[TMP4]], 3
|
|
// CHECK32-NEXT: ret [4 x <4 x i32>] [[DOTFCA_3_INSERT]]
|
|
//
|
|
bfloat16x8x4_t test_vld4q_dup_bf16(bfloat16_t const *ptr) {
|
|
return vld4q_dup_bf16(ptr);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst1_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to <4 x bfloat>*
|
|
// CHECK64-NEXT: store <4 x bfloat> [[VAL:%.*]], <4 x bfloat>* [[TMP0]], align 2
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst1_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst1.p0i8.v4bf16(i8* [[TMP0]], <4 x bfloat> [[VAL:%.*]], i32 2)
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst1_bf16(bfloat16_t *ptr, bfloat16x4_t val) {
|
|
vst1_bf16(ptr, val);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst1q_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to <8 x bfloat>*
|
|
// CHECK64-NEXT: store <8 x bfloat> [[VAL:%.*]], <8 x bfloat>* [[TMP0]], align 2
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst1q_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst1.p0i8.v8bf16(i8* [[TMP0]], <8 x bfloat> [[VAL:%.*]], i32 2)
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst1q_bf16(bfloat16_t *ptr, bfloat16x8_t val) {
|
|
vst1q_bf16(ptr, val);
|
|
}
|
|
|
|
// CHECK-LABEL: @test_vst1_lane_bf16(
|
|
// CHECK-NEXT: entry:
|
|
// CHECK-NEXT: [[TMP0:%.*]] = extractelement <4 x bfloat> [[VAL:%.*]], i64 1
|
|
// CHECK-NEXT: store bfloat [[TMP0]], bfloat* [[PTR:%.*]], align 2
|
|
// CHECK-NEXT: ret void
|
|
//
|
|
void test_vst1_lane_bf16(bfloat16_t *ptr, bfloat16x4_t val) {
|
|
vst1_lane_bf16(ptr, val, 1);
|
|
}
|
|
|
|
// CHECK-LABEL: @test_vst1q_lane_bf16(
|
|
// CHECK-NEXT: entry:
|
|
// CHECK-NEXT: [[TMP0:%.*]] = extractelement <8 x bfloat> [[VAL:%.*]], i64 7
|
|
// CHECK-NEXT: store bfloat [[TMP0]], bfloat* [[PTR:%.*]], align 2
|
|
// CHECK-NEXT: ret void
|
|
//
|
|
void test_vst1q_lane_bf16(bfloat16_t *ptr, bfloat16x8_t val) {
|
|
vst1q_lane_bf16(ptr, val, 7);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst1_bf16_x2(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st1x2.v4bf16.p0bf16(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst1_bf16_x2(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst1x2.p0bf16.v4bf16(bfloat* [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]])
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst1_bf16_x2(bfloat16_t *ptr, bfloat16x4x2_t val) {
|
|
vst1_bf16_x2(ptr, val);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst1q_bf16_x2(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st1x2.v8bf16.p0bf16(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst1q_bf16_x2(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst1x2.p0bf16.v8bf16(bfloat* [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]])
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst1q_bf16_x2(bfloat16_t *ptr, bfloat16x8x2_t val) {
|
|
vst1q_bf16_x2(ptr, val);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst1_bf16_x3(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE]], 2
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st1x3.v4bf16.p0bf16(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst1_bf16_x3(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE]], 2
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst1x3.p0bf16.v4bf16(bfloat* [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]])
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst1_bf16_x3(bfloat16_t *ptr, bfloat16x4x3_t val) {
|
|
vst1_bf16_x3(ptr, val);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst1q_bf16_x3(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE]], 2
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st1x3.v8bf16.p0bf16(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst1q_bf16_x3(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE]], 2
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst1x3.p0bf16.v8bf16(bfloat* [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]])
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst1q_bf16_x3(bfloat16_t *ptr, bfloat16x8x3_t val) {
|
|
vst1q_bf16_x3(ptr, val);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst1_bf16_x4(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 2
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 3
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st1x4.v4bf16.p0bf16(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_3_EXTRACT]], bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst1_bf16_x4(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 2
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 3
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_3_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst1x4.p0bf16.v4bf16(bfloat* [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]], <4 x bfloat> [[TMP3]])
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst1_bf16_x4(bfloat16_t *ptr, bfloat16x4x4_t val) {
|
|
vst1_bf16_x4(ptr, val);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst1q_bf16_x4(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 2
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 3
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st1x4.v8bf16.p0bf16(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_3_EXTRACT]], bfloat* [[PTR:%.*]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst1q_bf16_x4(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 2
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 3
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_3_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst1x4.p0bf16.v8bf16(bfloat* [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]], <8 x bfloat> [[TMP3]])
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst1q_bf16_x4(bfloat16_t *ptr, bfloat16x8x4_t val) {
|
|
vst1q_bf16_x4(ptr, val);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst2_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st2.v4bf16.p0i8(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], i8* [[TMP0]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst2_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst2.p0i8.v4bf16(i8* [[TMP2]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], i32 2)
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst2_bf16(bfloat16_t *ptr, bfloat16x4x2_t val) {
|
|
vst2_bf16(ptr, val);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst2q_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st2.v8bf16.p0i8(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], i8* [[TMP0]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst2q_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst2.p0i8.v8bf16(i8* [[TMP2]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], i32 2)
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst2q_bf16(bfloat16_t *ptr, bfloat16x8x2_t val) {
|
|
vst2q_bf16(ptr, val);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst2_lane_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st2lane.v4bf16.p0i8(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], i64 1, i8* [[TMP0]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst2_lane_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst2lane.p0i8.v4bf16(i8* [[TMP2]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], i32 1, i32 2)
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst2_lane_bf16(bfloat16_t *ptr, bfloat16x4x2_t val) {
|
|
vst2_lane_bf16(ptr, val, 1);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst2q_lane_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st2lane.v8bf16.p0i8(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], i64 7, i8* [[TMP0]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst2q_lane_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst2lane.p0i8.v8bf16(i8* [[TMP2]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], i32 7, i32 2)
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst2q_lane_bf16(bfloat16_t *ptr, bfloat16x8x2_t val) {
|
|
vst2q_lane_bf16(ptr, val, 7);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst3_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE]], 2
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st3.v4bf16.p0i8(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], i8* [[TMP0]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst3_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE]], 2
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst3.p0i8.v4bf16(i8* [[TMP3]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]], i32 2)
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst3_bf16(bfloat16_t *ptr, bfloat16x4x3_t val) {
|
|
vst3_bf16(ptr, val);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst3q_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE]], 2
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st3.v8bf16.p0i8(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], i8* [[TMP0]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst3q_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE]], 2
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst3.p0i8.v8bf16(i8* [[TMP3]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]], i32 2)
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst3q_bf16(bfloat16_t *ptr, bfloat16x8x3_t val) {
|
|
vst3q_bf16(ptr, val);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst3_lane_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE]], 2
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st3lane.v4bf16.p0i8(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], i64 1, i8* [[TMP0]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst3_lane_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE]], 2
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst3lane.p0i8.v4bf16(i8* [[TMP3]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]], i32 1, i32 2)
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst3_lane_bf16(bfloat16_t *ptr, bfloat16x4x3_t val) {
|
|
vst3_lane_bf16(ptr, val, 1);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst3q_lane_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE]], 2
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st3lane.v8bf16.p0i8(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], i64 7, i8* [[TMP0]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst3q_lane_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE]], 2
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst3lane.p0i8.v8bf16(i8* [[TMP3]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]], i32 7, i32 2)
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst3q_lane_bf16(bfloat16_t *ptr, bfloat16x8x3_t val) {
|
|
vst3q_lane_bf16(ptr, val, 7);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst4_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 2
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 3
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st4.v4bf16.p0i8(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_3_EXTRACT]], i8* [[TMP0]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst4_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 2
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 3
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_3_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP4:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst4.p0i8.v4bf16(i8* [[TMP4]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]], <4 x bfloat> [[TMP3]], i32 2)
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst4_bf16(bfloat16_t *ptr, bfloat16x4x4_t val) {
|
|
vst4_bf16(ptr, val);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst4q_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 2
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 3
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st4.v8bf16.p0i8(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_3_EXTRACT]], i8* [[TMP0]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst4q_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 2
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 3
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_3_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP4:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst4.p0i8.v8bf16(i8* [[TMP4]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]], <8 x bfloat> [[TMP3]], i32 2)
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst4q_bf16(bfloat16_t *ptr, bfloat16x8x4_t val) {
|
|
vst4q_bf16(ptr, val);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst4_lane_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 2
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 3
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st4lane.v4bf16.p0i8(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_3_EXTRACT]], i64 1, i8* [[TMP0]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst4_lane_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 2
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 3
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_3_EXTRACT]] to <4 x bfloat>
|
|
// CHECK32-NEXT: [[TMP4:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst4lane.p0i8.v4bf16(i8* [[TMP4]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]], <4 x bfloat> [[TMP3]], i32 1, i32 2)
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst4_lane_bf16(bfloat16_t *ptr, bfloat16x4x4_t val) {
|
|
vst4_lane_bf16(ptr, val, 1);
|
|
}
|
|
|
|
// CHECK64-LABEL: @test_vst4q_lane_bf16(
|
|
// CHECK64-NEXT: entry:
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 1
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 2
|
|
// CHECK64-NEXT: [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 3
|
|
// CHECK64-NEXT: [[TMP0:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK64-NEXT: tail call void @llvm.aarch64.neon.st4lane.v8bf16.p0i8(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_3_EXTRACT]], i64 7, i8* [[TMP0]])
|
|
// CHECK64-NEXT: ret void
|
|
//
|
|
// CHECK32-LABEL: @test_vst4q_lane_bf16(
|
|
// CHECK32-NEXT: entry:
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE:%.*]], 0
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 1
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 2
|
|
// CHECK32-NEXT: [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 3
|
|
// CHECK32-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_3_EXTRACT]] to <8 x bfloat>
|
|
// CHECK32-NEXT: [[TMP4:%.*]] = bitcast bfloat* [[PTR:%.*]] to i8*
|
|
// CHECK32-NEXT: tail call void @llvm.arm.neon.vst4lane.p0i8.v8bf16(i8* [[TMP4]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]], <8 x bfloat> [[TMP3]], i32 7, i32 2)
|
|
// CHECK32-NEXT: ret void
|
|
//
|
|
void test_vst4q_lane_bf16(bfloat16_t *ptr, bfloat16x8x4_t val) {
|
|
vst4q_lane_bf16(ptr, val, 7);
|
|
}
|