diff --git a/mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/fp32/sub_fp32.c b/mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/fp32/sub_fp32.c index b2e80c0a605..69d10742a90 100644 --- a/mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/fp32/sub_fp32.c +++ b/mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/fp32/sub_fp32.c @@ -1,5 +1,5 @@ /** - * Copyright 2021 Huawei Technologies Co., Ltd + * Copyright 2021-2022 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,37 +15,31 @@ */ #include "nnacl/fp32/sub_fp32.h" #include "nnacl/intrinsics/ms_simd_instructions.h" +#ifdef ENABLE_AVX512 +#include "nnacl/avx512/sub_fp32_avx512.h" +#endif -// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1) -#define SimdElementOptSubCoreCalc1(block_size, block_num, in0, in1, out, size, index) \ - do { \ - MS_FLOAT_32xN(block_num) vin0_opt_##block_num = MS_MOVN_F32(block_size, in0[0]); \ - for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \ - MS_FLOAT_32xN(block_num) vin1 = MS_LD_F32(block_size, in1 + index); \ - MS_FLOAT_32xN(block_num) vout = MS_SUB_F32(block_size, vin0_opt_##block_num, vin1); \ - MS_ST_F32(block_size, out + index, vout); \ - } \ - } while (0) +#ifdef ENABLE_AVX +#include "nnacl/avx/sub_fp32_avx.h" +#endif -#define SimdElementOptSubCoreCalc2(block_size, block_num, in0, in1, out, size, index) \ - do { \ - MS_FLOAT_32xN(block_num) vin1_opt_##block_num = MS_MOVN_F32(block_size, in1[0]); \ - for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \ - MS_FLOAT_32xN(block_num) vin0 = MS_LD_F32(block_size, in0 + index); \ - MS_FLOAT_32xN(block_num) vout = MS_SUB_F32(block_size, vin0, vin1_opt_##block_num); \ - MS_ST_F32(block_size, out + index, vout); \ - } \ - } while (0) +#ifdef ENABLE_SSE +#include "nnacl/sse/sub_fp32_sse.h" +#endif + +#ifdef ENABLE_ARM +#include "nnacl/neon/sub_fp32_neon.h" +#endif int ElementOptSub(const float *in0, const float *in1, float *out, int size, const ArithmeticParameter *param) { int index = 0; if (param->in_elements_num0_ == 1) { - MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubCoreCalc1, in0, in1, out, size, index); + SIMD_RUN_NO_SCALAR(ElementOptSubNum0, index, in0, in1, out, size); for (; index < size; index++) { out[index] = in0[0] - in1[index]; } } else { - MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubCoreCalc2, in0, in1, out, size, index); + SIMD_RUN_NO_SCALAR(ElementOptSubNum1, index, in0, in1, out, size); for (; index < size; index++) { out[index] = in0[index] - in1[0]; } @@ -53,36 +47,15 @@ int ElementOptSub(const float *in0, const float *in1, float *out, int size, cons return NNACL_OK; } -// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1) -#define SimdElementOptSubIntCoreCalc1(block_size, block_num, in0, in1, out, size, index) \ - do { \ - MS_INT_32xN(block_num) vin0_opt_##block_num = MS_MOVN_EPI32(block_size, in0[0]); \ - for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \ - MS_INT_32xN(block_num) vin1 = MS_LD_EPI32(block_size, in1 + index); \ - MS_INT_32xN(block_num) vout = MS_SUB_EPI32(block_size, vin0_opt_##block_num, vin1); \ - MS_ST_EPI32(block_size, out + index, vout); \ - } \ - } while (0) - -#define SimdElementOptSubIntCoreCalc2(block_size, block_num, in0, in1, out, size, index) \ - do { \ - MS_INT_32xN(block_num) vin1_opt_##block_num = MS_MOVN_EPI32(block_size, in1[0]); \ - for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \ - MS_INT_32xN(block_num) vin0 = MS_LD_EPI32(block_size, in0 + index); \ - MS_INT_32xN(block_num) vout = MS_SUB_EPI32(block_size, vin0, vin1_opt_##block_num); \ - MS_ST_EPI32(block_size, out + index, vout); \ - } \ - } while (0) - int ElementOptSubInt(const int *in0, const int *in1, int *out, int size, const ArithmeticParameter *param) { int index = 0; if (param->in_elements_num0_ == 1) { - MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubIntCoreCalc1, in0, in1, out, size, index); + SIMD_RUN_NO_SCALAR(ElementOptSubIntNum0, index, in0, in1, out, size); for (; index < size; index++) { out[index] = in0[0] - in1[index]; } } else { - MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubIntCoreCalc2, in0, in1, out, size, index); + SIMD_RUN_NO_SCALAR(ElementOptSubIntNum1, index, in0, in1, out, size); for (; index < size; index++) { out[index] = in0[index] - in1[0]; } @@ -90,38 +63,15 @@ int ElementOptSubInt(const int *in0, const int *in1, int *out, int size, const A return NNACL_OK; } -// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1) -#define SimdElementOptSubReluCoreCalc1(block_size, block_num, in0, in1, out, size, index) \ - do { \ - MS_FLOAT_32xN(block_num) vin0_opt_##block_num = MS_MOVN_F32(block_size, in0[0]); \ - for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \ - MS_FLOAT_32xN(block_num) vin1 = MS_LD_F32(block_size, in1 + index); \ - MS_FLOAT_32xN(block_num) vout = \ - MS_MAX_N_F32(block_size, MS_SUB_F32(block_size, vin0_opt_##block_num, vin1), 0.0f); \ - MS_ST_F32(block_size, out + index, vout); \ - } \ - } while (0) - -#define SimdElementOptSubReluCoreCalc2(block_size, block_num, in0, in1, out, size, index) \ - do { \ - MS_FLOAT_32xN(block_num) vin1_opt_##block_num = MS_MOVN_F32(block_size, in1[0]); \ - for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \ - MS_FLOAT_32xN(block_num) vin0 = MS_LD_F32(block_size, in0 + index); \ - MS_FLOAT_32xN(block_num) vout = \ - MS_MAX_N_F32(block_size, MS_SUB_F32(block_size, vin0, vin1_opt_##block_num), 0.0f); \ - MS_ST_F32(block_size, out + index, vout); \ - } \ - } while (0) - int ElementOptSubRelu(const float *in0, const float *in1, float *out, int size, const ArithmeticParameter *param) { int index = 0; if (param->in_elements_num0_ == 1) { - MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubReluCoreCalc1, in0, in1, out, size, index); + SIMD_RUN_NO_SCALAR(ElementOptSubReluNum0, index, in0, in1, out, size); for (; index < size; index++) { out[index] = MSMAX(in0[0] - in1[index], 0); } } else { - MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubReluCoreCalc2, in0, in1, out, size, index); + SIMD_RUN_NO_SCALAR(ElementOptSubReluNum1, index, in0, in1, out, size); for (; index < size; index++) { out[index] = MSMAX(in0[index] - in1[0], 0); } @@ -129,38 +79,15 @@ int ElementOptSubRelu(const float *in0, const float *in1, float *out, int size, return NNACL_OK; } -// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1) -#define SimdElementOptSubRelu6CoreCalc1(block_size, block_num, in0, in1, out, size, index) \ - do { \ - MS_FLOAT_32xN(block_num) vin0_opt_##block_num = MS_MOVN_F32(block_size, in0[0]); \ - for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \ - MS_FLOAT_32xN(block_num) vin1 = MS_LD_F32(block_size, in1 + index); \ - MS_FLOAT_32xN(block_num) vout = MS_MIN_N_F32( \ - block_size, MS_MAX_N_F32(block_size, MS_SUB_F32(block_size, vin0_opt_##block_num, vin1), 0.0f), 6.0f); \ - MS_ST_F32(block_size, out + index, vout); \ - } \ - } while (0) - -#define SimdElementOptSubRelu6CoreCalc2(block_size, block_num, in0, in1, out, size, index) \ - do { \ - MS_FLOAT_32xN(block_num) vin1_opt_##block_num = MS_MOVN_F32(block_size, in1[0]); \ - for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \ - MS_FLOAT_32xN(block_num) vin0 = MS_LD_F32(block_size, in0 + index); \ - MS_FLOAT_32xN(block_num) vout = MS_MIN_N_F32( \ - block_size, MS_MAX_N_F32(block_size, MS_SUB_F32(block_size, vin0, vin1_opt_##block_num), 0.0f), 6.0f); \ - MS_ST_F32(block_size, out + index, vout); \ - } \ - } while (0) - int ElementOptSubRelu6(const float *in0, const float *in1, float *out, int size, const ArithmeticParameter *param) { int index = 0; if (param->in_elements_num0_ == 1) { - MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubRelu6CoreCalc1, in0, in1, out, size, index); + SIMD_RUN_NO_SCALAR(ElementOptSubRelu6Num0, index, in0, in1, out, size); for (; index < size; index++) { out[index] = MSMIN(MSMAX(in0[0] - in1[index], 0), 6); } } else { - MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubRelu6CoreCalc2, in0, in1, out, size, index); + SIMD_RUN_NO_SCALAR(ElementOptSubRelu6Num1, index, in0, in1, out, size); for (; index < size; index++) { out[index] = MSMIN(MSMAX(in0[index] - in1[0], 0), 6); } @@ -168,57 +95,30 @@ int ElementOptSubRelu6(const float *in0, const float *in1, float *out, int size, return NNACL_OK; } -// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1) -#define SimdElementSubCoreCalc(block_size, block_num, in0, in1, out, size, index) \ - for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \ - MS_FLOAT_32xN(block_num) vin0 = MS_LD_F32(block_size, in0 + index); \ - MS_FLOAT_32xN(block_num) vin1 = MS_LD_F32(block_size, in1 + index); \ - MS_FLOAT_32xN(block_num) vout = MS_SUB_F32(block_size, vin0, vin1); \ - MS_ST_F32(block_size, out + index, vout); \ - } - int ElementSub(const float *in0, const float *in1, float *out, int size) { int index = 0; - MS_SIMD_RUN_NO_SCALAR(SimdElementSubCoreCalc, in0, in1, out, size, index); + SIMD_RUN_NO_SCALAR(ElementSub, index, in0, in1, out, size); for (; index < size; index++) { out[index] = in0[index] - in1[index]; } return NNACL_OK; } -// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1) -#define SimdElementSubIntCoreCalc(block_size, block_num, in0, in1, out, size, index) \ - for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \ - MS_INT_32xN(block_num) vin0 = MS_LD_EPI32(block_size, in0 + index); \ - MS_INT_32xN(block_num) vin1 = MS_LD_EPI32(block_size, in1 + index); \ - MS_INT_32xN(block_num) vout = MS_SUB_EPI32(block_size, vin0, vin1); \ - MS_ST_EPI32(block_size, out + index, vout); \ - } - int ElementSubInt(const int *in0, const int *in1, int *out, int size) { int index = 0; - MS_SIMD_RUN_NO_SCALAR(SimdElementSubIntCoreCalc, in0, in1, out, size, index); + SIMD_RUN_NO_SCALAR(ElementSubInt, index, in0, in1, out, size); for (; index < size; index++) { out[index] = in0[index] - in1[index]; } return NNACL_OK; } -// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1) -#define SimdElementSubReluCoreCalc(block_size, block_num, in0, in1, out, size, index) \ - for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \ - MS_FLOAT_32xN(block_num) vin0 = MS_LD_F32(block_size, in0 + index); \ - MS_FLOAT_32xN(block_num) vin1 = MS_LD_F32(block_size, in1 + index); \ - MS_FLOAT_32xN(block_num) vout = MS_MAX_N_F32(block_size, MS_SUB_F32(block_size, vin0, vin1), 0.0f); \ - MS_ST_F32(block_size, out + index, vout); \ - } - int ElementSubRelu(const float *in0, const float *in1, float *out, int size) { int index = 0; - MS_SIMD_RUN_NO_SCALAR(SimdElementSubReluCoreCalc, in0, in1, out, size, index); + SIMD_RUN_NO_SCALAR(ElementSubRelu, index, in0, in1, out, size); for (; index < size; index++) { float res = in0[index] - in1[index]; out[index] = res > 0 ? res : 0; @@ -226,20 +126,10 @@ int ElementSubRelu(const float *in0, const float *in1, float *out, int size) { return NNACL_OK; } -// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1) -#define SimdElementSubRelu6CoreCalc(block_size, block_num, in0, in1, out, size, index) \ - for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \ - MS_FLOAT_32xN(block_num) vin0 = MS_LD_F32(block_size, in0 + index); \ - MS_FLOAT_32xN(block_num) vin1 = MS_LD_F32(block_size, in1 + index); \ - MS_FLOAT_32xN(block_num) vout = \ - MS_MIN_N_F32(block_size, MS_MAX_N_F32(block_size, MS_SUB_F32(block_size, vin0, vin1), 0.0f), 6.0f); \ - MS_ST_F32(block_size, out + index, vout); \ - } - int ElementSubRelu6(const float *in0, const float *in1, float *out, int size) { int index = 0; - MS_SIMD_RUN_NO_SCALAR(SimdElementSubRelu6CoreCalc, in0, in1, out, size, index); + SIMD_RUN_NO_SCALAR(ElementSubRelu6, index, in0, in1, out, size); for (; index < size; index++) { out[index] = MSMIN(MSMAX(in0[index] - in1[index], 0), 6); } diff --git a/mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/fp32/sub_fp32_simd.h.in b/mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/fp32/sub_fp32_simd.h.in new file mode 100644 index 00000000000..1163796362c --- /dev/null +++ b/mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/fp32/sub_fp32_simd.h.in @@ -0,0 +1,160 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_NNACL_FP32_SUB_@SIMD_INSTRUCTION@_H_ +#define MINDSPORE_NNACL_FP32_SUB_@SIMD_INSTRUCTION@_H_ + +#include "nnacl/intrinsics/ms_simd_instructions.h" +#include "nnacl/intrinsics/ms_simd_@SIMD_INSTRUCTION_LOWER@_instructions.h" + +#ifdef __cplusplus +extern "C" { +#endif +@SIMD_INSTRUCTION_BEGIN@ + +static inline int ElementOptSubNum0@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out, + int size) { + SIMD_F32 vin0_opt = SIMD_MOV_F32(in0[0]); + for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) { + SIMD_F32 vin1 = SIMD_LD_F32(in1 + index); + SIMD_F32 vout = SIMD_SUB_F32(vin0_opt, vin1); + SIMD_ST_F32(out + index, vout); + } + return index; +} + +static inline int ElementOptSubNum1@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out, + int size) { + SIMD_F32 vin1_opt_ = SIMD_MOV_F32(in1[0]); + for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) { + SIMD_F32 vin0 = SIMD_LD_F32(in0 + index); + SIMD_F32 vout = SIMD_SUB_F32(vin0, vin1_opt_); + SIMD_ST_F32(out + index, vout); + } + return index; +} + +static inline int ElementOptSubIntNum0@SIMD_INSTRUCTION@(int index, const int *in0, const int *in1, int *out, int size) { + SIMD_EPI32 vin0_opt = SIMD_MOV_EPI32(in0[0]); + for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) { + SIMD_EPI32 vin1 = SIMD_LD_EPI32(in1 + index); + SIMD_EPI32 vout = SIMD_SUB_EPI32(vin0_opt, vin1); + SIMD_ST_EPI32(out + index, vout); + } + return index; +} + +static inline int ElementOptSubIntNum1@SIMD_INSTRUCTION@(int index, const int *in0, const int *in1, int *out, int size) { + SIMD_EPI32 vin1_opt_ = SIMD_MOV_EPI32(in1[0]); + for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) { + SIMD_EPI32 vin0 = SIMD_LD_EPI32(in0 + index); + SIMD_EPI32 vout = SIMD_SUB_EPI32(vin0, vin1_opt_); + SIMD_ST_EPI32(out + index, vout); + } + return index; +} + +static inline int ElementOptSubReluNum0@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out, + int size) { + SIMD_F32 vin0_opt = SIMD_MOV_F32(in0[0]); + for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) { + SIMD_F32 vin1 = SIMD_LD_F32(in1 + index); + SIMD_F32 vout = SIMD_MAX_N_F32(SIMD_SUB_F32(vin0_opt, vin1), 0.0f); + SIMD_ST_F32(out + index, vout); + } + return index; +} + +static inline int ElementOptSubReluNum1@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out, + int size) { + SIMD_F32 vin1_opt_ = SIMD_MOV_F32(in1[0]); + for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) { + SIMD_F32 vin0 = SIMD_LD_F32(in0 + index); + SIMD_F32 vout = SIMD_MAX_N_F32(SIMD_SUB_F32(vin0, vin1_opt_), 0.0f); + SIMD_ST_F32(out + index, vout); + } + return index; +} + +static inline int ElementOptSubRelu6Num0@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out, + int size) { + SIMD_F32 vin0_opt = SIMD_MOV_F32(in0[0]); + for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) { + SIMD_F32 vin1 = SIMD_LD_F32(in1 + index); + SIMD_F32 vout = SIMD_MIN_N_F32(SIMD_MAX_N_F32(SIMD_SUB_F32(vin0_opt, vin1), 0.0f), 6.0f); + SIMD_ST_F32(out + index, vout); + } + return index; +} + +static inline int ElementOptSubRelu6Num1@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out, + int size) { + SIMD_F32 vin1_opt_ = SIMD_MOV_F32(in1[0]); + for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) { + SIMD_F32 vin0 = SIMD_LD_F32(in0 + index); + SIMD_F32 vout = SIMD_MIN_N_F32(SIMD_MAX_N_F32(SIMD_SUB_F32(vin0, vin1_opt_), 0.0f), 6.0f); + SIMD_ST_F32(out + index, vout); + } + return index; +} + +static inline int ElementSub@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out, int size) { + for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) { + SIMD_F32 vin0 = SIMD_LD_F32(in0 + index); + SIMD_F32 vin1 = SIMD_LD_F32(in1 + index); + SIMD_F32 vout = SIMD_SUB_F32(vin0, vin1); + SIMD_ST_F32(out + index, vout); + } + return index; +} + +static inline int ElementSubInt@SIMD_INSTRUCTION@(int index, const int *in0, const int *in1, int *out, int size) { + for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) { + SIMD_EPI32 vin0 = SIMD_LD_EPI32(in0 + index); + SIMD_EPI32 vin1 = SIMD_LD_EPI32(in1 + index); + SIMD_EPI32 vout = SIMD_SUB_EPI32(vin0, vin1); + SIMD_ST_EPI32(out + index, vout); + } + return index; +} + +static inline int ElementSubRelu@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out, + int size) { + for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) { + SIMD_F32 vin0 = SIMD_LD_F32(in0 + index); + SIMD_F32 vin1 = SIMD_LD_F32(in1 + index); + SIMD_F32 vout = SIMD_MAX_N_F32(SIMD_SUB_F32(vin0, vin1), 0.0f); + SIMD_ST_F32(out + index, vout); + } + return index; +} + +static inline int ElementSubRelu6@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out, + int size) { + for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) { + SIMD_F32 vin0 = SIMD_LD_F32(in0 + index); + SIMD_F32 vin1 = SIMD_LD_F32(in1 + index); + SIMD_F32 vout = SIMD_MIN_N_F32(SIMD_MAX_N_F32(SIMD_SUB_F32(vin0, vin1), 0.0f), 6.0f); + SIMD_ST_F32(out + index, vout); + } + return index; +} + +@SIMD_INSTRUCTION_END@ +#ifdef __cplusplus +}; +#endif +#endif