Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

avx512 self feel, support sub op

This commit is contained in:
greatpan 2022-05-26 14:21:51 +08:00
parent e960f5beb7
commit 80b052d0b1
2 changed files with 186 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/fp32/sub_fp32.c
View File

@ -1,5 +1,5 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
* Copyright 2021-2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -15,37 +15,31 @@
*/
#include "nnacl/fp32/sub_fp32.h"
#include "nnacl/intrinsics/ms_simd_instructions.h"
#ifdef ENABLE_AVX512
#include "nnacl/avx512/sub_fp32_avx512.h"
#endif
// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1)
#define SimdElementOptSubCoreCalc1(block_size, block_num, in0, in1, out, size, index) \
do { \
MS_FLOAT_32xN(block_num) vin0_opt_##block_num = MS_MOVN_F32(block_size, in0[0]); \
for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \
MS_FLOAT_32xN(block_num) vin1 = MS_LD_F32(block_size, in1 + index); \
MS_FLOAT_32xN(block_num) vout = MS_SUB_F32(block_size, vin0_opt_##block_num, vin1); \
MS_ST_F32(block_size, out + index, vout); \
} \
} while (0)
#ifdef ENABLE_AVX
#include "nnacl/avx/sub_fp32_avx.h"
#endif
#define SimdElementOptSubCoreCalc2(block_size, block_num, in0, in1, out, size, index) \
do { \
MS_FLOAT_32xN(block_num) vin1_opt_##block_num = MS_MOVN_F32(block_size, in1[0]); \
for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \
MS_FLOAT_32xN(block_num) vin0 = MS_LD_F32(block_size, in0 + index); \
MS_FLOAT_32xN(block_num) vout = MS_SUB_F32(block_size, vin0, vin1_opt_##block_num); \
MS_ST_F32(block_size, out + index, vout); \
} \
} while (0)
#ifdef ENABLE_SSE
#include "nnacl/sse/sub_fp32_sse.h"
#endif
#ifdef ENABLE_ARM
#include "nnacl/neon/sub_fp32_neon.h"
#endif
int ElementOptSub(const float *in0, const float *in1, float *out, int size, const ArithmeticParameter *param) {
int index = 0;
if (param->in_elements_num0_ == 1) {
MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubCoreCalc1, in0, in1, out, size, index);
SIMD_RUN_NO_SCALAR(ElementOptSubNum0, index, in0, in1, out, size);
for (; index < size; index++) {
out[index] = in0[0] - in1[index];
}
} else {
MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubCoreCalc2, in0, in1, out, size, index);
SIMD_RUN_NO_SCALAR(ElementOptSubNum1, index, in0, in1, out, size);
for (; index < size; index++) {
out[index] = in0[index] - in1[0];
}
@ -53,36 +47,15 @@ int ElementOptSub(const float *in0, const float *in1, float *out, int size, cons
return NNACL_OK;
}
// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1)
#define SimdElementOptSubIntCoreCalc1(block_size, block_num, in0, in1, out, size, index) \
do { \
MS_INT_32xN(block_num) vin0_opt_##block_num = MS_MOVN_EPI32(block_size, in0[0]); \
for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \
MS_INT_32xN(block_num) vin1 = MS_LD_EPI32(block_size, in1 + index); \
MS_INT_32xN(block_num) vout = MS_SUB_EPI32(block_size, vin0_opt_##block_num, vin1); \
MS_ST_EPI32(block_size, out + index, vout); \
} \
} while (0)
#define SimdElementOptSubIntCoreCalc2(block_size, block_num, in0, in1, out, size, index) \
do { \
MS_INT_32xN(block_num) vin1_opt_##block_num = MS_MOVN_EPI32(block_size, in1[0]); \
for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \
MS_INT_32xN(block_num) vin0 = MS_LD_EPI32(block_size, in0 + index); \
MS_INT_32xN(block_num) vout = MS_SUB_EPI32(block_size, vin0, vin1_opt_##block_num); \
MS_ST_EPI32(block_size, out + index, vout); \
} \
} while (0)
int ElementOptSubInt(const int *in0, const int *in1, int *out, int size, const ArithmeticParameter *param) {
int index = 0;
if (param->in_elements_num0_ == 1) {
MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubIntCoreCalc1, in0, in1, out, size, index);
SIMD_RUN_NO_SCALAR(ElementOptSubIntNum0, index, in0, in1, out, size);
for (; index < size; index++) {
out[index] = in0[0] - in1[index];
}
} else {
MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubIntCoreCalc2, in0, in1, out, size, index);
SIMD_RUN_NO_SCALAR(ElementOptSubIntNum1, index, in0, in1, out, size);
for (; index < size; index++) {
out[index] = in0[index] - in1[0];
}
@ -90,38 +63,15 @@ int ElementOptSubInt(const int *in0, const int *in1, int *out, int size, const A
return NNACL_OK;
}
// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1)
#define SimdElementOptSubReluCoreCalc1(block_size, block_num, in0, in1, out, size, index) \
do { \
MS_FLOAT_32xN(block_num) vin0_opt_##block_num = MS_MOVN_F32(block_size, in0[0]); \
for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \
MS_FLOAT_32xN(block_num) vin1 = MS_LD_F32(block_size, in1 + index); \
MS_FLOAT_32xN(block_num) vout = \
MS_MAX_N_F32(block_size, MS_SUB_F32(block_size, vin0_opt_##block_num, vin1), 0.0f); \
MS_ST_F32(block_size, out + index, vout); \
} \
} while (0)
#define SimdElementOptSubReluCoreCalc2(block_size, block_num, in0, in1, out, size, index) \
do { \
MS_FLOAT_32xN(block_num) vin1_opt_##block_num = MS_MOVN_F32(block_size, in1[0]); \
for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \
MS_FLOAT_32xN(block_num) vin0 = MS_LD_F32(block_size, in0 + index); \
MS_FLOAT_32xN(block_num) vout = \
MS_MAX_N_F32(block_size, MS_SUB_F32(block_size, vin0, vin1_opt_##block_num), 0.0f); \
MS_ST_F32(block_size, out + index, vout); \
} \
} while (0)
int ElementOptSubRelu(const float *in0, const float *in1, float *out, int size, const ArithmeticParameter *param) {
int index = 0;
if (param->in_elements_num0_ == 1) {
MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubReluCoreCalc1, in0, in1, out, size, index);
SIMD_RUN_NO_SCALAR(ElementOptSubReluNum0, index, in0, in1, out, size);
for (; index < size; index++) {
out[index] = MSMAX(in0[0] - in1[index], 0);
}
} else {
MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubReluCoreCalc2, in0, in1, out, size, index);
SIMD_RUN_NO_SCALAR(ElementOptSubReluNum1, index, in0, in1, out, size);
for (; index < size; index++) {
out[index] = MSMAX(in0[index] - in1[0], 0);
}
@ -129,38 +79,15 @@ int ElementOptSubRelu(const float *in0, const float *in1, float *out, int size,
return NNACL_OK;
}
// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1)
#define SimdElementOptSubRelu6CoreCalc1(block_size, block_num, in0, in1, out, size, index) \
do { \
MS_FLOAT_32xN(block_num) vin0_opt_##block_num = MS_MOVN_F32(block_size, in0[0]); \
for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \
MS_FLOAT_32xN(block_num) vin1 = MS_LD_F32(block_size, in1 + index); \
MS_FLOAT_32xN(block_num) vout = MS_MIN_N_F32( \
block_size, MS_MAX_N_F32(block_size, MS_SUB_F32(block_size, vin0_opt_##block_num, vin1), 0.0f), 6.0f); \
MS_ST_F32(block_size, out + index, vout); \
} \
} while (0)
#define SimdElementOptSubRelu6CoreCalc2(block_size, block_num, in0, in1, out, size, index) \
do { \
MS_FLOAT_32xN(block_num) vin1_opt_##block_num = MS_MOVN_F32(block_size, in1[0]); \
for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \
MS_FLOAT_32xN(block_num) vin0 = MS_LD_F32(block_size, in0 + index); \
MS_FLOAT_32xN(block_num) vout = MS_MIN_N_F32( \
block_size, MS_MAX_N_F32(block_size, MS_SUB_F32(block_size, vin0, vin1_opt_##block_num), 0.0f), 6.0f); \
MS_ST_F32(block_size, out + index, vout); \
} \
} while (0)
int ElementOptSubRelu6(const float *in0, const float *in1, float *out, int size, const ArithmeticParameter *param) {
int index = 0;
if (param->in_elements_num0_ == 1) {
MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubRelu6CoreCalc1, in0, in1, out, size, index);
SIMD_RUN_NO_SCALAR(ElementOptSubRelu6Num0, index, in0, in1, out, size);
for (; index < size; index++) {
out[index] = MSMIN(MSMAX(in0[0] - in1[index], 0), 6);
}
} else {
MS_SIMD_RUN_NO_SCALAR(SimdElementOptSubRelu6CoreCalc2, in0, in1, out, size, index);
SIMD_RUN_NO_SCALAR(ElementOptSubRelu6Num1, index, in0, in1, out, size);
for (; index < size; index++) {
out[index] = MSMIN(MSMAX(in0[index] - in1[0], 0), 6);
}
@ -168,57 +95,30 @@ int ElementOptSubRelu6(const float *in0, const float *in1, float *out, int size,
return NNACL_OK;
}
// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1)
#define SimdElementSubCoreCalc(block_size, block_num, in0, in1, out, size, index) \
for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \
MS_FLOAT_32xN(block_num) vin0 = MS_LD_F32(block_size, in0 + index); \
MS_FLOAT_32xN(block_num) vin1 = MS_LD_F32(block_size, in1 + index); \
MS_FLOAT_32xN(block_num) vout = MS_SUB_F32(block_size, vin0, vin1); \
MS_ST_F32(block_size, out + index, vout); \
}
int ElementSub(const float *in0, const float *in1, float *out, int size) {
int index = 0;
MS_SIMD_RUN_NO_SCALAR(SimdElementSubCoreCalc, in0, in1, out, size, index);
SIMD_RUN_NO_SCALAR(ElementSub, index, in0, in1, out, size);
for (; index < size; index++) {
out[index] = in0[index] - in1[index];
}
return NNACL_OK;
}
// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1)
#define SimdElementSubIntCoreCalc(block_size, block_num, in0, in1, out, size, index) \
for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \
MS_INT_32xN(block_num) vin0 = MS_LD_EPI32(block_size, in0 + index); \
MS_INT_32xN(block_num) vin1 = MS_LD_EPI32(block_size, in1 + index); \
MS_INT_32xN(block_num) vout = MS_SUB_EPI32(block_size, vin0, vin1); \
MS_ST_EPI32(block_size, out + index, vout); \
}
int ElementSubInt(const int *in0, const int *in1, int *out, int size) {
int index = 0;
MS_SIMD_RUN_NO_SCALAR(SimdElementSubIntCoreCalc, in0, in1, out, size, index);
SIMD_RUN_NO_SCALAR(ElementSubInt, index, in0, in1, out, size);
for (; index < size; index++) {
out[index] = in0[index] - in1[index];
}
return NNACL_OK;
}
// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1)
#define SimdElementSubReluCoreCalc(block_size, block_num, in0, in1, out, size, index) \
for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \
MS_FLOAT_32xN(block_num) vin0 = MS_LD_F32(block_size, in0 + index); \
MS_FLOAT_32xN(block_num) vin1 = MS_LD_F32(block_size, in1 + index); \
MS_FLOAT_32xN(block_num) vout = MS_MAX_N_F32(block_size, MS_SUB_F32(block_size, vin0, vin1), 0.0f); \
MS_ST_F32(block_size, out + index, vout); \
}
int ElementSubRelu(const float *in0, const float *in1, float *out, int size) {
int index = 0;
MS_SIMD_RUN_NO_SCALAR(SimdElementSubReluCoreCalc, in0, in1, out, size, index);
SIMD_RUN_NO_SCALAR(ElementSubRelu, index, in0, in1, out, size);
for (; index < size; index++) {
float res = in0[index] - in1[index];
out[index] = res > 0 ? res : 0;
@ -226,20 +126,10 @@ int ElementSubRelu(const float *in0, const float *in1, float *out, int size) {
return NNACL_OK;
}
// 32 bits, block_size : (512/256/128/32), block_num : (16/8/4/1)
#define SimdElementSubRelu6CoreCalc(block_size, block_num, in0, in1, out, size, index) \
for (int block_max_size = size - block_num + 1; index < block_max_size; index += block_num) { \
MS_FLOAT_32xN(block_num) vin0 = MS_LD_F32(block_size, in0 + index); \
MS_FLOAT_32xN(block_num) vin1 = MS_LD_F32(block_size, in1 + index); \
MS_FLOAT_32xN(block_num) vout = \
MS_MIN_N_F32(block_size, MS_MAX_N_F32(block_size, MS_SUB_F32(block_size, vin0, vin1), 0.0f), 6.0f); \
MS_ST_F32(block_size, out + index, vout); \
}
int ElementSubRelu6(const float *in0, const float *in1, float *out, int size) {
int index = 0;
MS_SIMD_RUN_NO_SCALAR(SimdElementSubRelu6CoreCalc, in0, in1, out, size, index);
SIMD_RUN_NO_SCALAR(ElementSubRelu6, index, in0, in1, out, size);
for (; index < size; index++) {
out[index] = MSMIN(MSMAX(in0[index] - in1[index], 0), 6);
}

160
mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/fp32/sub_fp32_simd.h.in Normal file
View File

@ -0,0 +1,160 @@
/**
* Copyright 2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_NNACL_FP32_SUB_@SIMD_INSTRUCTION@_H_
#define MINDSPORE_NNACL_FP32_SUB_@SIMD_INSTRUCTION@_H_
#include "nnacl/intrinsics/ms_simd_instructions.h"
#include "nnacl/intrinsics/ms_simd_@SIMD_INSTRUCTION_LOWER@_instructions.h"
#ifdef __cplusplus
extern "C" {
#endif
@SIMD_INSTRUCTION_BEGIN@
static inline int ElementOptSubNum0@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out,
int size) {
SIMD_F32 vin0_opt = SIMD_MOV_F32(in0[0]);
for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) {
SIMD_F32 vin1 = SIMD_LD_F32(in1 + index);
SIMD_F32 vout = SIMD_SUB_F32(vin0_opt, vin1);
SIMD_ST_F32(out + index, vout);
}
return index;
}
static inline int ElementOptSubNum1@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out,
int size) {
SIMD_F32 vin1_opt_ = SIMD_MOV_F32(in1[0]);
for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) {
SIMD_F32 vin0 = SIMD_LD_F32(in0 + index);
SIMD_F32 vout = SIMD_SUB_F32(vin0, vin1_opt_);
SIMD_ST_F32(out + index, vout);
}
return index;
}
static inline int ElementOptSubIntNum0@SIMD_INSTRUCTION@(int index, const int *in0, const int *in1, int *out, int size) {
SIMD_EPI32 vin0_opt = SIMD_MOV_EPI32(in0[0]);
for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) {
SIMD_EPI32 vin1 = SIMD_LD_EPI32(in1 + index);
SIMD_EPI32 vout = SIMD_SUB_EPI32(vin0_opt, vin1);
SIMD_ST_EPI32(out + index, vout);
}
return index;
}
static inline int ElementOptSubIntNum1@SIMD_INSTRUCTION@(int index, const int *in0, const int *in1, int *out, int size) {
SIMD_EPI32 vin1_opt_ = SIMD_MOV_EPI32(in1[0]);
for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) {
SIMD_EPI32 vin0 = SIMD_LD_EPI32(in0 + index);
SIMD_EPI32 vout = SIMD_SUB_EPI32(vin0, vin1_opt_);
SIMD_ST_EPI32(out + index, vout);
}
return index;
}
static inline int ElementOptSubReluNum0@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out,
int size) {
SIMD_F32 vin0_opt = SIMD_MOV_F32(in0[0]);
for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) {
SIMD_F32 vin1 = SIMD_LD_F32(in1 + index);
SIMD_F32 vout = SIMD_MAX_N_F32(SIMD_SUB_F32(vin0_opt, vin1), 0.0f);
SIMD_ST_F32(out + index, vout);
}
return index;
}
static inline int ElementOptSubReluNum1@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out,
int size) {
SIMD_F32 vin1_opt_ = SIMD_MOV_F32(in1[0]);
for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) {
SIMD_F32 vin0 = SIMD_LD_F32(in0 + index);
SIMD_F32 vout = SIMD_MAX_N_F32(SIMD_SUB_F32(vin0, vin1_opt_), 0.0f);
SIMD_ST_F32(out + index, vout);
}
return index;
}
static inline int ElementOptSubRelu6Num0@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out,
int size) {
SIMD_F32 vin0_opt = SIMD_MOV_F32(in0[0]);
for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) {
SIMD_F32 vin1 = SIMD_LD_F32(in1 + index);
SIMD_F32 vout = SIMD_MIN_N_F32(SIMD_MAX_N_F32(SIMD_SUB_F32(vin0_opt, vin1), 0.0f), 6.0f);
SIMD_ST_F32(out + index, vout);
}
return index;
}
static inline int ElementOptSubRelu6Num1@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out,
int size) {
SIMD_F32 vin1_opt_ = SIMD_MOV_F32(in1[0]);
for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) {
SIMD_F32 vin0 = SIMD_LD_F32(in0 + index);
SIMD_F32 vout = SIMD_MIN_N_F32(SIMD_MAX_N_F32(SIMD_SUB_F32(vin0, vin1_opt_), 0.0f), 6.0f);
SIMD_ST_F32(out + index, vout);
}
return index;
}
static inline int ElementSub@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out, int size) {
for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) {
SIMD_F32 vin0 = SIMD_LD_F32(in0 + index);
SIMD_F32 vin1 = SIMD_LD_F32(in1 + index);
SIMD_F32 vout = SIMD_SUB_F32(vin0, vin1);
SIMD_ST_F32(out + index, vout);
}
return index;
}
static inline int ElementSubInt@SIMD_INSTRUCTION@(int index, const int *in0, const int *in1, int *out, int size) {
for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) {
SIMD_EPI32 vin0 = SIMD_LD_EPI32(in0 + index);
SIMD_EPI32 vin1 = SIMD_LD_EPI32(in1 + index);
SIMD_EPI32 vout = SIMD_SUB_EPI32(vin0, vin1);
SIMD_ST_EPI32(out + index, vout);
}
return index;
}
static inline int ElementSubRelu@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out,
int size) {
for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) {
SIMD_F32 vin0 = SIMD_LD_F32(in0 + index);
SIMD_F32 vin1 = SIMD_LD_F32(in1 + index);
SIMD_F32 vout = SIMD_MAX_N_F32(SIMD_SUB_F32(vin0, vin1), 0.0f);
SIMD_ST_F32(out + index, vout);
}
return index;
}
static inline int ElementSubRelu6@SIMD_INSTRUCTION@(int index, const float *in0, const float *in1, float *out,
int size) {
for (int block_max_size = size - BLOCK_NUM + 1; index < block_max_size; index += BLOCK_NUM) {
SIMD_F32 vin0 = SIMD_LD_F32(in0 + index);
SIMD_F32 vin1 = SIMD_LD_F32(in1 + index);
SIMD_F32 vout = SIMD_MIN_N_F32(SIMD_MAX_N_F32(SIMD_SUB_F32(vin0, vin1), 0.0f), 6.0f);
SIMD_ST_F32(out + index, vout);
}
return index;
}
@SIMD_INSTRUCTION_END@
#ifdef __cplusplus
};
#endif
#endif