From 9d9d2e1ff41f4240d3df83e3688f7de847682f8b Mon Sep 17 00:00:00 2001 From: yangruoqi713 Date: Mon, 18 Jan 2021 14:54:54 +0800 Subject: [PATCH] [MSLITE][DEVELOP] add cpu fp32 op: crop_and_resize --- mindspore/lite/nnacl/fp32/resize_fp32.c | 146 ++++++++------ mindspore/lite/nnacl/fp32/resize_fp32.h | 13 +- mindspore/lite/nnacl/resize_parameter.h | 7 + mindspore/lite/schema/model.fbs | 1 + mindspore/lite/schema/ops.fbs | 5 + mindspore/lite/src/ops/crop_and_resize.cc | 111 +++++++++++ mindspore/lite/src/ops/crop_and_resize.h | 47 +++++ .../ops/populate/crop_and_resize_populate.cc | 40 ++++ .../kernel/arm/fp32/crop_and_resize_fp32.cc | 178 ++++++++++++++++++ .../kernel/arm/fp32/crop_and_resize_fp32.h | 60 ++++++ .../runtime/kernel/arm/fp32/resize_fp32.cc | 45 ++--- .../src/runtime/kernel/arm/fp32/resize_fp32.h | 3 - .../runtime/kernel/arm/fp32/upsample_fp32.cc | 4 +- 13 files changed, 559 insertions(+), 101 deletions(-) create mode 100644 mindspore/lite/src/ops/crop_and_resize.cc create mode 100644 mindspore/lite/src/ops/crop_and_resize.h create mode 100644 mindspore/lite/src/ops/populate/crop_and_resize_populate.cc create mode 100644 mindspore/lite/src/runtime/kernel/arm/fp32/crop_and_resize_fp32.cc create mode 100644 mindspore/lite/src/runtime/kernel/arm/fp32/crop_and_resize_fp32.h diff --git a/mindspore/lite/nnacl/fp32/resize_fp32.c b/mindspore/lite/nnacl/fp32/resize_fp32.c index 5f073c16b72..2cb73340a73 100644 --- a/mindspore/lite/nnacl/fp32/resize_fp32.c +++ b/mindspore/lite/nnacl/fp32/resize_fp32.c @@ -57,13 +57,13 @@ int PrepareCropAndResizeBilinear(const int *input_shape, const float *boxes, con x_rights == NULL || y_bottom_weights == NULL || x_left_weights == NULL) { return NNACL_NULL_PTR; } - int in_b = input_shape[0]; int in_h = input_shape[1]; int in_w = input_shape[2]; + int batch = output_shape[0]; int new_height = output_shape[1]; int new_width = output_shape[2]; - for (int i = 0; i < in_b; i++) { + for (int i = 0; i < batch; i++) { int b = box_idx[i]; const float *box = boxes + b * 4; int start_h = box[0] * (in_h - 1); @@ -145,10 +145,63 @@ int InterpCol(const float *bottom_line, const float *top_line, float *output, in return 0; } +void Bilinear(const float *input_data, float *output_data, const int *input_shape, const int *output_shape, + const int *y_bottom, const int *y_top, const int *x_left, const int *x_right, + const float *y_bottom_weight, const float *x_left_weight, float *line0, float *line1, const int h_begin, + const int h_end) { + int in_w = input_shape[2]; + int in_c = input_shape[3]; + int new_width = output_shape[2]; + int h_stride = new_width * in_c; + + bool cache_line_used[2] = {false, false}; + int cache_line_num[2] = {-1, -1}; + float *const cache_line_ptr[2] = {line0, line1}; + float *current_line_ptr[2] = {line0, line1}; + int current_line_num[2] = {-1, -1}; + + for (int h = h_begin; h < h_end; h++) { + current_line_num[0] = y_bottom[h]; + current_line_num[1] = y_top[h]; + + for (int i = 0; i < 2; i++) { + cache_line_used[i] = false; + } + // search if we cached + for (int j = 0; j < 2; j++) { + bool find = false; + for (int k = 0; k < 2; k++) { + if (current_line_num[j] == cache_line_num[k]) { + cache_line_used[k] = true; + current_line_ptr[j] = cache_line_ptr[k]; + find = true; + break; + } + } + + if (!find) { + const float *line = input_data + current_line_num[j] * in_w * in_c; + for (int k = 0; k < 2; k++) { + if (!cache_line_used[k]) { + cache_line_num[k] = current_line_num[j]; + cache_line_used[k] = true; + current_line_ptr[j] = cache_line_ptr[k]; + InterpRow(line, current_line_ptr[j], new_width, x_left_weight, x_left, x_right, in_c); + break; + } + } + } + } + // do col interp + InterpCol(current_line_ptr[0], current_line_ptr[1], output_data + h * h_stride, new_width, y_bottom_weight[h], + in_c); + } +} + int ResizeBilinear(const float *input_data, float *output_data, const int *input_shape, const int *output_shape, const int *y_bottoms, const int *y_tops, const int *x_lefts, const int *x_rights, const float *y_bottom_weights, const float *x_left_weights, float *line0, float *line1, - const int h_begin, const int h_end, bool is_crop) { + const int h_begin, const int h_end) { if (input_data == NULL || output_data == NULL || input_shape == NULL || output_shape == NULL || y_bottoms == NULL || y_tops == NULL || x_lefts == NULL || x_rights == NULL || y_bottom_weights == NULL || x_left_weights == NULL) { return NNACL_NULL_PTR; @@ -158,75 +211,48 @@ int ResizeBilinear(const float *input_data, float *output_data, const int *input int in_h = input_shape[1]; int in_w = input_shape[2]; int in_c = input_shape[3]; - int new_height = output_shape[1]; int new_width = output_shape[2]; - int h_stride = new_width * in_c; - - const int *y_bottom = y_bottoms; - const int *y_top = y_tops; - const float *y_bottom_weight = y_bottom_weights; - const int *x_left = x_lefts; - const int *x_right = x_rights; - const float *x_left_weight = x_left_weights; for (int b = 0; b < in_b; b++) { - if (is_crop) { - y_bottom = y_bottoms + b * new_height; - y_top = y_tops + b * new_height; - y_bottom_weight = y_bottom_weights + b * new_height; - x_left = x_lefts + b * new_width; - x_right = x_rights + b * new_width; - x_left_weight = x_left_weights + b * new_width; - } const float *input = input_data + b * in_h * in_w * in_c; float *output = output_data + b * new_height * new_width * in_c; - bool cache_line_used[2] = {false, false}; - int cache_line_num[2] = {-1, -1}; - float *const cache_line_ptr[2] = {line0, line1}; - float *current_line_ptr[2] = {line0, line1}; - int current_line_num[2] = {-1, -1}; - for (int h = h_begin; h < h_end; h++) { - current_line_num[0] = y_bottom[h]; - current_line_num[1] = y_top[h]; - - for (int i = 0; i < 2; i++) { - cache_line_used[i] = false; - } - // search if we cached - for (int j = 0; j < 2; j++) { - bool find = false; - for (int k = 0; k < 2; k++) { - if (current_line_num[j] == cache_line_num[k]) { - cache_line_used[k] = true; - current_line_ptr[j] = cache_line_ptr[k]; - find = true; - break; - } - } - - if (!find) { - const float *line = input + current_line_num[j] * in_w * in_c; - for (int k = 0; k < 2; k++) { - if (!cache_line_used[k]) { - cache_line_num[k] = current_line_num[j]; - cache_line_used[k] = true; - current_line_ptr[j] = cache_line_ptr[k]; - InterpRow(line, current_line_ptr[j], new_width, x_left_weight, x_left, x_right, in_c); - break; - } - } - } - } - - // do col interp - InterpCol(current_line_ptr[0], current_line_ptr[1], output + h * h_stride, new_width, y_bottom_weight[h], in_c); + Bilinear(input, output, input_shape, output_shape, y_bottoms, y_tops, x_lefts, x_rights, y_bottom_weights, + x_left_weights, line0, line1, h_begin, h_end); } } return NNACL_OK; } +int CropAndResizeBilinear(const float *input_data, float *output_data, const int *input_shape, const int *output_shape, + const int *y_bottoms, const int *y_tops, const int *x_lefts, const int *x_rights, + const float *y_bottom_weights, const float *x_left_weights, float *line0, float *line1, + const int h_begin, const int h_end) { + if (input_data == NULL || output_data == NULL || input_shape == NULL || output_shape == NULL || y_bottoms == NULL || + y_tops == NULL || x_lefts == NULL || x_rights == NULL || y_bottom_weights == NULL || x_left_weights == NULL) { + return NNACL_NULL_PTR; + } + int batch = output_shape[0]; + int new_height = output_shape[1]; + int new_width = output_shape[2]; + int new_channel = output_shape[3]; + + for (int b = 0; b < batch; b++) { + const int *y_bottom = y_bottoms + b * new_height; + const int *y_top = y_tops + b * new_height; + const float *y_bottom_weight = y_bottom_weights + b * new_height; + const int *x_left = x_lefts + b * new_width; + const int *x_right = x_rights + b * new_width; + const float *x_left_weight = x_left_weights + b * new_width; + float *output = output_data + b * new_height * new_width * new_channel; + + Bilinear(input_data, output, input_shape, output_shape, y_bottom, y_top, x_left, x_right, y_bottom_weight, + x_left_weight, line0, line1, h_begin, h_end); + } + return NNACL_OK; +} + int ResizeNearestNeighbor(const float *input_data, float *output_data, const int *input_shape, const int *output_shape, CalculateOriginalCoordinate calculate, int coordinate_transform_mode, int tid, int thread_num) { diff --git a/mindspore/lite/nnacl/fp32/resize_fp32.h b/mindspore/lite/nnacl/fp32/resize_fp32.h index bd6e683f19e..873c1ffa20f 100644 --- a/mindspore/lite/nnacl/fp32/resize_fp32.h +++ b/mindspore/lite/nnacl/fp32/resize_fp32.h @@ -31,14 +31,19 @@ int PrepareResizeBilinear(const int *input_shape, const int *output_shape, Calcu int *y_bottoms, int *y_tops, int *x_lefts, int *x_rights, float *y_bottom_weights, float *x_left_weights); +int ResizeBilinear(const float *input_data, float *output_data, const int *input_shape, const int *output_shape, + const int *y_bottoms, const int *y_tops, const int *x_lefts, const int *x_rights, + const float *y_bottom_weights, const float *x_left_weights, float *line0, float *line1, + const int h_begin, const int h_end); + int PrepareCropAndResizeBilinear(const int *input_shape, const float *boxes, const int *box_idx, const int *output_shape, int *y_bottoms, int *y_tops, int *x_lefts, int *x_rights, float *y_bottom_weights, float *x_left_weights); -int ResizeBilinear(const float *input_data, float *output_data, const int *input_shape, const int *output_shape, - const int *y_bottoms, const int *y_tops, const int *x_lefts, const int *x_rights, - const float *y_bottom_weights, const float *x_left_weights, float *line0, float *line1, - const int h_begin, const int h_end, bool is_crop); +int CropAndResizeBilinear(const float *input_data, float *output_data, const int *input_shape, const int *output_shape, + const int *y_bottoms, const int *y_tops, const int *x_lefts, const int *x_rights, + const float *y_bottom_weights, const float *x_left_weights, float *line0, float *line1, + const int h_begin, const int h_end); int ResizeNearestNeighbor(const float *input_data, float *output_data, const int *input_shape, const int *output_shape, CalculateOriginalCoordinate calculate, int coordinate_transform_mode, int tid, diff --git a/mindspore/lite/nnacl/resize_parameter.h b/mindspore/lite/nnacl/resize_parameter.h index 115464d0e5d..7fed0bcd3c5 100644 --- a/mindspore/lite/nnacl/resize_parameter.h +++ b/mindspore/lite/nnacl/resize_parameter.h @@ -26,4 +26,11 @@ typedef struct ResizeParameter { int coordinate_transform_mode_; bool preserve_aspect_ratio_; } ResizeParameter; + +typedef struct CropAndResizeParameter { + // primitive parameter + OpParameter op_parameter_; + int method_; + float extrapolation_value_; +} CropAndResizeParameter; #endif // MINDSPORE_LITE_NNACL_RESIZE_PARAMETER_H_ diff --git a/mindspore/lite/schema/model.fbs b/mindspore/lite/schema/model.fbs index 719a10ffedc..e37bc7741c4 100644 --- a/mindspore/lite/schema/model.fbs +++ b/mindspore/lite/schema/model.fbs @@ -270,6 +270,7 @@ union PrimitiveType { InvertPermutation, Size, RandomStandardNormal, + CropAndResize, } enum QuantType: int { diff --git a/mindspore/lite/schema/ops.fbs b/mindspore/lite/schema/ops.fbs index 732d3bdae2e..1615d616cd8 100644 --- a/mindspore/lite/schema/ops.fbs +++ b/mindspore/lite/schema/ops.fbs @@ -1255,3 +1255,8 @@ table RandomStandardNormal { seed : int; seed2 : int; } + +table CropAndResize { + method : ResizeMethod; + extrapolation_value : float; +} diff --git a/mindspore/lite/src/ops/crop_and_resize.cc b/mindspore/lite/src/ops/crop_and_resize.cc new file mode 100644 index 00000000000..76badc02355 --- /dev/null +++ b/mindspore/lite/src/ops/crop_and_resize.cc @@ -0,0 +1,111 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/ops/crop_and_resize.h" + +#ifndef PRIMITIVE_WRITEABLE +#include "src/ops/ops_register.h" +#endif + +namespace mindspore { +namespace lite { +#ifdef PRIMITIVE_WRITEABLE +int CropAndResize::GetMethod() const { return this->primitive_->value.AsCropAndResize()->method; } +float CropAndResize::GetExtrapolationValue() const { + return this->primitive_->value.AsCropAndResize()->extrapolation_value; +} + +void CropAndResize::SetMethod(int method) { + this->primitive_->value.AsCropAndResize()->method = (schema::ResizeMethod)method; +} +void CropAndResize::SetExtrapolationValue(float value) { + this->primitive_->value.AsCropAndResize()->extrapolation_value = value; +} +#else + +int CropAndResize::GetMethod() const { return this->primitive_->value_as_CropAndResize()->method(); } +float CropAndResize::GetExtrapolationValue() const { + return this->primitive_->value_as_CropAndResize()->extrapolation_value(); +} +int CropAndResize::UnPackToFlatBuilder(const schema::Primitive *primitive, flatbuffers::FlatBufferBuilder *fbb) { + MS_ASSERT(nullptr != primitive); + MS_ASSERT(nullptr != fbb); + auto attr = primitive->value_as_CropAndResize(); + if (attr == nullptr) { + MS_LOG(ERROR) << "value_as_CropAndResize return nullptr"; + return RET_ERROR; + } + auto val_offset = schema::CreateCropAndResize(*fbb, attr->method(), attr->extrapolation_value()); + auto prim_offset = schema::CreatePrimitive(*fbb, schema::PrimitiveType_CropAndResize, val_offset.o); + fbb->Finish(prim_offset); + return RET_OK; +} + +PrimitiveC *CropAndResizeCreator(const schema::Primitive *primitive) { + return PrimitiveC::NewPrimitiveC(primitive); +} +Registry CropAndResizeRegistry(schema::PrimitiveType_CropAndResize, CropAndResizeCreator); +#endif + +namespace { +constexpr int kInputRank = 4; +} // namespace +int CropAndResize::InferShape(std::vector inputs_, std::vector outputs_) { + MS_ASSERT(this->primitive_ != nullptr); + if (inputs_.size() != 4) { + MS_LOG(ERROR) << "Input tensor num should be 4 for crop_an_resize."; + return RET_ERROR; + } + auto input = inputs_.front(); + if (input == nullptr) { + return RET_ERROR; + } + if (!input->shape().empty() && input->shape().size() != kInputRank) { + MS_LOG(ERROR) << "Size of input shape is wrong."; + return RET_ERROR; + } + if (input->format() != schema::Format_NHWC) { + MS_LOG(ERROR) << "Crop_an_resize op only support NHWC format."; + return RET_ERROR; + } + + auto output = outputs_.front(); + if (output == nullptr) { + return RET_NULL_PTR; + } + output->set_data_type(input->data_type()); + output->set_format(input->format()); + if (!infer_flag()) { + return RET_INFER_INVALID; + } + + std::vector output_shape; + auto boxes_tensor = inputs_[1]; + output_shape.push_back(boxes_tensor->shape()[0]); + auto shape_tensor = inputs_[3]; + auto data = reinterpret_cast(shape_tensor->data_c()); + if (data == nullptr) { + MS_LOG(INFO) << "The data of 4th input tensor(shape tensor) for crop_an_resize op is nullptr."; + return RET_INFER_INVALID; + } + output_shape.push_back(data[0]); + output_shape.push_back(data[1]); + output_shape.push_back(input->Channel()); + output->set_shape(output_shape); + return RET_OK; +} +} // namespace lite +} // namespace mindspore diff --git a/mindspore/lite/src/ops/crop_and_resize.h b/mindspore/lite/src/ops/crop_and_resize.h new file mode 100644 index 00000000000..6cbfa98b8bb --- /dev/null +++ b/mindspore/lite/src/ops/crop_and_resize.h @@ -0,0 +1,47 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LITE_MINDSPORE_LITE_C_OPS_CROP_AND_RESIZE_H_ +#define LITE_MINDSPORE_LITE_C_OPS_CROP_AND_RESIZE_H_ + +#include +#include +#include + +#include "src/ops/primitive_c.h" + +namespace mindspore { +namespace lite { +class CropAndResize : public PrimitiveC { + public: + CropAndResize() = default; + ~CropAndResize() = default; +#ifdef PRIMITIVE_WRITEABLE + MS_DECLARE_PARENT(CropAndResize, PrimitiveC); + explicit CropAndResize(schema::PrimitiveT *primitive) : PrimitiveC(primitive) {} + void SetMethod(int method); + void SetExtrapolationValue(float value); +#else + int UnPackToFlatBuilder(const schema::Primitive *primitive, flatbuffers::FlatBufferBuilder *fbb) override; +#endif + int InferShape(std::vector inputs_, std::vector outputs_) override; + int GetMethod() const; + float GetExtrapolationValue() const; +}; +} // namespace lite +} // namespace mindspore + +#endif // LITE_MINDSPORE_LITE_C_OPS_CROP_AND_RESIZE_H_ diff --git a/mindspore/lite/src/ops/populate/crop_and_resize_populate.cc b/mindspore/lite/src/ops/populate/crop_and_resize_populate.cc new file mode 100644 index 00000000000..e87ef703faf --- /dev/null +++ b/mindspore/lite/src/ops/populate/crop_and_resize_populate.cc @@ -0,0 +1,40 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/ops/crop_and_resize.h" +#include "src/ops/primitive_c.h" +#include "src/ops/populate/populate_register.h" +#include "nnacl/resize_parameter.h" +namespace mindspore { +namespace lite { +OpParameter *PopulateCropAndResizeParameter(const mindspore::lite::PrimitiveC *primitive) { + CropAndResizeParameter *crop_resize_param = + reinterpret_cast(malloc(sizeof(CropAndResizeParameter))); + if (crop_resize_param == nullptr) { + MS_LOG(ERROR) << "malloc CropAndResizeParameter failed."; + return nullptr; + } + memset(crop_resize_param, 0, sizeof(CropAndResizeParameter)); + crop_resize_param->op_parameter_.type_ = primitive->Type(); + auto param = reinterpret_cast(const_cast(primitive)); + crop_resize_param->method_ = static_cast(param->GetMethod()); + crop_resize_param->extrapolation_value_ = param->GetExtrapolationValue(); + return reinterpret_cast(crop_resize_param); +} + +Registry CropAndResizeParameterRegistry(schema::PrimitiveType_CropAndResize, PopulateCropAndResizeParameter); +} // namespace lite +} // namespace mindspore diff --git a/mindspore/lite/src/runtime/kernel/arm/fp32/crop_and_resize_fp32.cc b/mindspore/lite/src/runtime/kernel/arm/fp32/crop_and_resize_fp32.cc new file mode 100644 index 00000000000..3b49691b53c --- /dev/null +++ b/mindspore/lite/src/runtime/kernel/arm/fp32/crop_and_resize_fp32.cc @@ -0,0 +1,178 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/runtime/kernel/arm/fp32/crop_and_resize_fp32.h" +#include "schema/model_generated.h" +#include "src/kernel_registry.h" +#include "src/runtime/runtime_api.h" +#include "nnacl/fp32/resize_fp32.h" + +using mindspore::kernel::KERNEL_ARCH::kCPU; +using mindspore::lite::KernelRegistrar; +using mindspore::lite::RET_ERROR; +using mindspore::lite::RET_INVALID_OP_ATTR; +using mindspore::lite::RET_NULL_PTR; +using mindspore::lite::RET_OK; +using mindspore::schema::PrimitiveType_CropAndResize; + +namespace mindspore::kernel { +int CropAndResizeCPUKernel::Init() { + if (!InferShapeDone()) { + return RET_OK; + } + return ReSize(); +} + +int CropAndResizeCPUKernel::ReSize() { + new_height_ = out_tensors_.at(0)->shape()[1]; + new_width_ = out_tensors_.at(0)->shape()[2]; + return RET_OK; +} + +int CropAndResizeCPUKernel::MallocTmpBuffer() { + // Malloc buffer to save coordinate. + // For mode CROP_AND_RESIZE, different output batches require different cache coordinates. + int c = in_tensors_.at(0)->Channel(); + y_bottoms_ = reinterpret_cast(context_->allocator->Malloc(sizeof(int) * new_height_ * batch_)); + if (y_bottoms_ == nullptr) { + MS_LOG(ERROR) << "malloc data failed"; + return RET_NULL_PTR; + } + y_tops_ = reinterpret_cast(context_->allocator->Malloc(sizeof(int) * new_height_ * batch_)); + if (y_tops_ == nullptr) { + MS_LOG(ERROR) << "malloc data failed"; + return RET_NULL_PTR; + } + y_bottom_weights_ = reinterpret_cast(context_->allocator->Malloc(sizeof(float) * new_height_ * batch_)); + if (y_bottom_weights_ == nullptr) { + MS_LOG(ERROR) << "malloc data failed"; + return RET_NULL_PTR; + } + + x_lefts_ = reinterpret_cast(context_->allocator->Malloc(sizeof(int) * new_width_ * batch_)); + if (x_lefts_ == nullptr) { + MS_LOG(ERROR) << "malloc data failed"; + return RET_NULL_PTR; + } + x_rights_ = reinterpret_cast(context_->allocator->Malloc(sizeof(int) * new_width_ * batch_)); + if (x_rights_ == nullptr) { + MS_LOG(ERROR) << "malloc data failed"; + return RET_NULL_PTR; + } + x_left_weights_ = reinterpret_cast(context_->allocator->Malloc(sizeof(float) * new_width_ * batch_)); + if (x_left_weights_ == nullptr) { + MS_LOG(ERROR) << "malloc data failed"; + return RET_NULL_PTR; + } + line_buffer_ = + reinterpret_cast(context_->allocator->Malloc(sizeof(float) * new_width_ * c * 2 * context_->thread_num_)); + if (line_buffer_ == nullptr) { + MS_LOG(ERROR) << "malloc data failed"; + return RET_NULL_PTR; + } + return RET_OK; +} + +void CropAndResizeCPUKernel::FreeTmpBuffer() { + context_->allocator->Free(y_bottoms_); + context_->allocator->Free(y_tops_); + context_->allocator->Free(y_bottom_weights_); + context_->allocator->Free(x_lefts_); + context_->allocator->Free(x_rights_); + context_->allocator->Free(x_left_weights_); + context_->allocator->Free(line_buffer_); +} + +int CropAndResizeImpl(void *cdata, int task_id) { + auto resize = reinterpret_cast(cdata); + auto error_code = resize->RunImpl(task_id); + if (error_code != RET_OK) { + MS_LOG(ERROR) << "CropAndResize Run error task_id[" << task_id << "] error_code[" << error_code << "]"; + return RET_ERROR; + } + return RET_OK; +} + +int CropAndResizeCPUKernel::RunImpl(int task_id) { + auto input = in_tensors_.at(0); + auto input_data = reinterpret_cast(input->data_c()); + if (input_data == nullptr) { + return RET_NULL_PTR; + } + auto output_data = reinterpret_cast(out_tensors_.at(0)->data_c()); + if (output_data == nullptr) { + return RET_NULL_PTR; + } + auto input_shape = input->shape(); + int unit = UP_DIV(new_height_, context_->thread_num_); + int h_begin = unit * task_id; + int h_end = MSMIN(h_begin + unit, new_height_); + int c = in_tensors_.at(0)->shape().at(3); + float *line0 = line_buffer_ + new_width_ * c * 2 * task_id; + float *line1 = line0 + new_width_ * c; + int ret = 0; + if (is_crop_) { + ret = CropAndResizeBilinear(input_data, output_data, input_shape.data(), out_tensors_.at(0)->shape().data(), + y_bottoms_, y_tops_, x_lefts_, x_rights_, y_bottom_weights_, x_left_weights_, line0, + line1, h_begin, h_end); + } else { + ret = + ResizeBilinear(input_data, output_data, input_shape.data(), out_tensors_.at(0)->shape().data(), y_bottoms_, + y_tops_, x_lefts_, x_rights_, y_bottom_weights_, x_left_weights_, line0, line1, h_begin, h_end); + } + return ret; +} + +int CropAndResizeCPUKernel::Run() { + auto ret = MallocTmpBuffer(); + if (ret != RET_OK) { + FreeTmpBuffer(); + return ret; + } + + auto input = in_tensors_.at(0); + auto input_shape = input->shape(); + // if boxes tensor data is nullptr, crop_and_resize can be seen as resize with coordinate transformation mode + // ALIGN_CORNERS + if (in_tensors_.at(1)->ElementsNum() == 0 || in_tensors_.at(1)->data_c() == nullptr) { + batch_ = 1; + is_crop_ = false; + ret = PrepareResizeBilinear(input_shape.data(), out_tensors_.at(0)->shape().data(), CalculateAlignCorners, + y_bottoms_, y_tops_, x_lefts_, x_rights_, y_bottom_weights_, x_left_weights_); + } else { + batch_ = out_tensors_[0]->Batch(); + auto boxes = reinterpret_cast(in_tensors_.at(1)->data_c()); + auto box_idx = reinterpret_cast(in_tensors_.at(2)->data_c()); + ret = PrepareCropAndResizeBilinear(input_shape.data(), boxes, box_idx, out_tensors_.at(0)->shape().data(), + y_bottoms_, y_tops_, x_lefts_, x_rights_, y_bottom_weights_, x_left_weights_); + } + if (ret != RET_OK) { + FreeTmpBuffer(); + return ret; + } + + int error_code = ParallelLaunch(this->context_->thread_pool_, CropAndResizeImpl, this, context_->thread_num_); + if (error_code != RET_OK) { + MS_LOG(ERROR) << "CropAndResize run error, error_code[" << error_code << "]"; + FreeTmpBuffer(); + return RET_ERROR; + } + FreeTmpBuffer(); + return RET_OK; +} + +REG_KERNEL(kCPU, kNumberTypeFloat32, PrimitiveType_CropAndResize, LiteKernelCreator) +} // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp32/crop_and_resize_fp32.h b/mindspore/lite/src/runtime/kernel/arm/fp32/crop_and_resize_fp32.h new file mode 100644 index 00000000000..96197248412 --- /dev/null +++ b/mindspore/lite/src/runtime/kernel/arm/fp32/crop_and_resize_fp32.h @@ -0,0 +1,60 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_CROP_AND_RESIZE_H_ +#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_CROP_AND_RESIZE_H_ + +#include +#include "include/errorcode.h" +#include "nnacl/resize_parameter.h" +#include "src/lite_kernel.h" + +namespace mindspore::kernel { +class CropAndResizeCPUKernel : public LiteKernel { + public: + CropAndResizeCPUKernel(OpParameter *parameter, const std::vector &inputs, + const std::vector &outputs, const lite::InnerContext *ctx, + const mindspore::lite::PrimitiveC *primitive) + : LiteKernel(parameter, inputs, outputs, ctx, primitive) { + param_ = reinterpret_cast(op_parameter_); + } + + ~CropAndResizeCPUKernel() = default; + + int Init() override; + int ReSize() override; + int Run() override; + int RunImpl(int task_id); + + protected: + int MallocTmpBuffer(); + void FreeTmpBuffer(); + + CropAndResizeParameter *param_; + int batch_; + int new_height_; + int new_width_; + bool is_crop_ = true; + int *y_tops_ = nullptr; + int *y_bottoms_ = nullptr; + int *x_lefts_ = nullptr; + int *x_rights_ = nullptr; + float *y_bottom_weights_ = nullptr; + float *x_left_weights_ = nullptr; + float *line_buffer_ = nullptr; +}; +} // namespace mindspore::kernel + +#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_CROP_AND_RESIZE_H_ diff --git a/mindspore/lite/src/runtime/kernel/arm/fp32/resize_fp32.cc b/mindspore/lite/src/runtime/kernel/arm/fp32/resize_fp32.cc index b8a056591f0..ce3bca0bd68 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp32/resize_fp32.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp32/resize_fp32.cc @@ -43,8 +43,6 @@ int ResizeCPUKernel::Init() { case schema::CoordinateTransformMode_HALF_PIXEL: calculate_ = CalculateHalfPixel; break; - case schema::CoordinateTransformMode_TF_CROP_AND_RESIZE: - break; default: MS_LOG(ERROR) << "Do not support coordinate transform mode. Mode is" << schema::EnumNameCoordinateTransformMode( @@ -75,15 +73,8 @@ int ResizeCPUKernel::ReSize() { auto input = in_tensors_.at(0); auto input_shape = input->shape(); - if (coordinate_transform_mode_ == schema::CoordinateTransformMode_TF_CROP_AND_RESIZE) { - auto boxes = reinterpret_cast(in_tensors_.at(1)->data_c()); - auto box_idx = reinterpret_cast(in_tensors_.at(2)->data_c()); - ret = PrepareCropAndResizeBilinear(input_shape.data(), boxes, box_idx, out_tensors_.at(0)->shape().data(), - y_bottoms_, y_tops_, x_lefts_, x_rights_, y_bottom_weights_, x_left_weights_); - } else { - ret = PrepareResizeBilinear(input_shape.data(), out_tensors_.at(0)->shape().data(), calculate_, y_bottoms_, - y_tops_, x_lefts_, x_rights_, y_bottom_weights_, x_left_weights_); - } + ret = PrepareResizeBilinear(input_shape.data(), out_tensors_.at(0)->shape().data(), calculate_, y_bottoms_, y_tops_, + x_lefts_, x_rights_, y_bottom_weights_, x_left_weights_); if (ret != RET_OK) { FreeTmpBuffer(); } @@ -92,42 +83,36 @@ int ResizeCPUKernel::ReSize() { } int ResizeCPUKernel::MallocTmpBuffer() { - int b = in_tensors_.at(0)->Batch(); - // Malloc buffer to save coordinate. For mode CROP_AND_RESIZE, different batches require different cache coordinates. - // For other modes, different batches have different cache coordinates. - if (coordinate_transform_mode_ != schema::CoordinateTransformMode_TF_CROP_AND_RESIZE) { - b = 1; - } int c = in_tensors_.at(0)->Channel(); int h = new_height_; int w = new_width_; - y_bottoms_ = reinterpret_cast(malloc(sizeof(int) * h * b)); + y_bottoms_ = reinterpret_cast(malloc(sizeof(int) * h)); if (y_bottoms_ == nullptr) { MS_LOG(ERROR) << "malloc data failed"; return RET_NULL_PTR; } - y_tops_ = reinterpret_cast(malloc(sizeof(int) * h * b)); + y_tops_ = reinterpret_cast(malloc(sizeof(int) * h)); if (y_tops_ == nullptr) { MS_LOG(ERROR) << "malloc data failed"; return RET_NULL_PTR; } - y_bottom_weights_ = reinterpret_cast(malloc(sizeof(float) * h * b)); + y_bottom_weights_ = reinterpret_cast(malloc(sizeof(float) * h)); if (y_bottom_weights_ == nullptr) { MS_LOG(ERROR) << "malloc data failed"; return RET_NULL_PTR; } - x_lefts_ = reinterpret_cast(malloc(sizeof(int) * w * b)); + x_lefts_ = reinterpret_cast(malloc(sizeof(int) * w)); if (x_lefts_ == nullptr) { MS_LOG(ERROR) << "malloc data failed"; return RET_NULL_PTR; } - x_rights_ = reinterpret_cast(malloc(sizeof(int) * w * b)); + x_rights_ = reinterpret_cast(malloc(sizeof(int) * w)); if (x_rights_ == nullptr) { MS_LOG(ERROR) << "malloc data failed"; return RET_NULL_PTR; } - x_left_weights_ = reinterpret_cast(malloc(sizeof(float) * w * b)); + x_left_weights_ = reinterpret_cast(malloc(sizeof(float) * w)); if (x_left_weights_ == nullptr) { MS_LOG(ERROR) << "malloc data failed"; return RET_NULL_PTR; @@ -137,9 +122,9 @@ int ResizeCPUKernel::MallocTmpBuffer() { MS_LOG(ERROR) << "malloc data failed"; return RET_NULL_PTR; } - return RET_OK; } + void ResizeCPUKernel::FreeTmpBuffer() { if (y_bottoms_ != nullptr) { free(y_bottoms_); @@ -184,11 +169,11 @@ int ResizeImpl(void *cdata, int task_id) { int ResizeCPUKernel::RunImpl(int task_id) { auto input = in_tensors_.at(0); - auto input_data = reinterpret_cast(input->MutableData()); + auto input_data = reinterpret_cast(input->data_c()); if (input_data == nullptr) { return RET_NULL_PTR; } - auto output_data = reinterpret_cast(out_tensors_.at(0)->MutableData()); + auto output_data = reinterpret_cast(out_tensors_.at(0)->data_c()); if (output_data == nullptr) { return RET_NULL_PTR; } @@ -205,11 +190,9 @@ int ResizeCPUKernel::RunImpl(int task_id) { int c = in_tensors_.at(0)->shape().at(3); float *line0 = line_buffer_ + new_width_ * c * 2 * task_id; float *line1 = line0 + new_width_ * c; - - bool is_crop = coordinate_transform_mode_ == schema::CoordinateTransformMode_TF_CROP_AND_RESIZE; - ret = ResizeBilinear(input_data, output_data, input_shape.data(), out_tensors_.at(0)->shape().data(), y_bottoms_, - y_tops_, x_lefts_, x_rights_, y_bottom_weights_, x_left_weights_, line0, line1, h_begin, - h_end, is_crop); + ret = + ResizeBilinear(input_data, output_data, input_shape.data(), out_tensors_.at(0)->shape().data(), y_bottoms_, + y_tops_, x_lefts_, x_rights_, y_bottom_weights_, x_left_weights_, line0, line1, h_begin, h_end); break; } case static_cast(schema::ResizeMethod_NEAREST): { diff --git a/mindspore/lite/src/runtime/kernel/arm/fp32/resize_fp32.h b/mindspore/lite/src/runtime/kernel/arm/fp32/resize_fp32.h index 203312ca5b3..979ec547d91 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp32/resize_fp32.h +++ b/mindspore/lite/src/runtime/kernel/arm/fp32/resize_fp32.h @@ -23,9 +23,6 @@ #include "src/lite_kernel.h" #include "src/runtime/kernel/arm/base/resize_base.h" -using mindspore::schema::PrimitiveType_Resize; -using mindspore::schema::ResizeMethod; - namespace mindspore::kernel { class ResizeCPUKernel : public ResizeBaseCPUKernel { public: diff --git a/mindspore/lite/src/runtime/kernel/arm/fp32/upsample_fp32.cc b/mindspore/lite/src/runtime/kernel/arm/fp32/upsample_fp32.cc index ffd758359dd..5fa002b4477 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp32/upsample_fp32.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp32/upsample_fp32.cc @@ -105,9 +105,7 @@ int UpsampleCPUKernel::RunImpl(int task_id) { float *line0 = line_buffer_ + new_width_ * c * 2 * task_id; float *line1 = line0 + new_width_ * c; ret = ResizeBilinear(input_data, output_data, input_shape.data(), out_tensor->shape().data(), y_bottoms_, y_tops_, - x_lefts_, x_rights_, y_bottom_weights_, x_left_weights_, line0, line1, n_h_begin, n_h_end, - false); - + x_lefts_, x_rights_, y_bottom_weights_, x_left_weights_, line0, line1, n_h_begin, n_h_end); break; } case static_cast(schema::ResizeMethod_NEAREST): {