From 90553fef08fbde89b014f83e11ed15932f63090e Mon Sep 17 00:00:00 2001 From: zhaozhenlong Date: Sat, 1 Aug 2020 15:59:41 +0800 Subject: [PATCH] add lite op PriorBox --- mindspore/lite/schema/model.fbs | 3 +- mindspore/lite/schema/ops.fbs | 14 ++ mindspore/lite/src/ops/ops.cc | 2 + mindspore/lite/src/ops/ops.h | 9 +- mindspore/lite/src/ops/prior_box.cc | 62 ++++++ mindspore/lite/src/populate_parameter.cc | 57 ++++++ .../src/runtime/kernel/arm/base/prior_box.cc | 193 ++++++++++++++++++ .../src/runtime/kernel/arm/base/prior_box.h | 53 +++++ .../runtime/kernel/arm/opclib/prior_box.cc | 30 +++ .../src/runtime/kernel/arm/opclib/prior_box.h | 45 ++++ 10 files changed, 466 insertions(+), 2 deletions(-) create mode 100644 mindspore/lite/src/ops/prior_box.cc create mode 100644 mindspore/lite/src/runtime/kernel/arm/base/prior_box.cc create mode 100644 mindspore/lite/src/runtime/kernel/arm/base/prior_box.h create mode 100644 mindspore/lite/src/runtime/kernel/arm/opclib/prior_box.cc create mode 100644 mindspore/lite/src/runtime/kernel/arm/opclib/prior_box.h diff --git a/mindspore/lite/schema/model.fbs b/mindspore/lite/schema/model.fbs index b40135ebbaf..b71fa966c12 100644 --- a/mindspore/lite/schema/model.fbs +++ b/mindspore/lite/schema/model.fbs @@ -173,7 +173,8 @@ union PrimitiveType { Div, Where, OneHot, - Lstm + Lstm, + PriorBox } enum QuantType: int { diff --git a/mindspore/lite/schema/ops.fbs b/mindspore/lite/schema/ops.fbs index f0b50bfca03..52fb99aabe7 100644 --- a/mindspore/lite/schema/ops.fbs +++ b/mindspore/lite/schema/ops.fbs @@ -722,3 +722,17 @@ table OneHot { table Lstm{ bidirection: bool = false; } + +table PriorBox { + min_sizes: [int]; + max_sizes: [int]; + aspect_ratios: [float]; + variances: [float]; + image_size_w: int; + image_size_h: int; + step_w: float; + step_h: float; + clip: bool = true; + flip: bool = true; + offset: float; +} diff --git a/mindspore/lite/src/ops/ops.cc b/mindspore/lite/src/ops/ops.cc index 4a41cc9f239..2fdc8681d72 100644 --- a/mindspore/lite/src/ops/ops.cc +++ b/mindspore/lite/src/ops/ops.cc @@ -131,6 +131,8 @@ Primitive *Primitive::CreatePrimitive(schema::Primitive *primitive) { return new lite::Resize(const_cast(primitive)); case schema::PrimitiveType_OneHot: return new lite::OneHot(const_cast(primitive)); + case schema::PrimitiveType_PriorBox: + return new lite::PriorBox(const_cast(primitive)); default: break; } diff --git a/mindspore/lite/src/ops/ops.h b/mindspore/lite/src/ops/ops.h index def9c3ee724..a67a9753413 100644 --- a/mindspore/lite/src/ops/ops.h +++ b/mindspore/lite/src/ops/ops.h @@ -660,7 +660,14 @@ class StridedSlice : public Primitive { std::vector new_axis_mask_; std::vector shrink_axis_mask_; }; + +class PriorBox : public Primitive { + public: + explicit PriorBox(schema::Primitive *primitive) : Primitive(primitive) {} + const schema::PriorBox *GetAttrbute() const { return this->primitive->value_as_PriorBox(); } + int InferShape(std::vector inputs, std::vector outputs) override; +}; + } // namespace lite } // namespace mindspore #endif // MINDSPORE_LITE_SRC_OPS_OPS_H_ - diff --git a/mindspore/lite/src/ops/prior_box.cc b/mindspore/lite/src/ops/prior_box.cc new file mode 100644 index 00000000000..96e5b654135 --- /dev/null +++ b/mindspore/lite/src/ops/prior_box.cc @@ -0,0 +1,62 @@ +/** + * Copyright 2019-2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/ops/ops.h" +#include "include/errorcode.h" +#include "utils/log_adapter.h" +#include "src/ir/tensor.h" + +namespace mindspore::lite { +namespace { +constexpr int kPriorBoxPoints = 4; +constexpr int kPriorBoxN = 1; +constexpr int kPriorBoxW = 1; +constexpr int kPriorBoxC = 2; +} // namespace + +int PriorBox::InferShape(std::vector inputs_, std::vector outputs_) { + auto param = GetAttrbute(); + MS_ASSERT(param != nullptr); + std::vector different_aspect_ratios{1.0f}; + auto aspect_ratios = param->aspect_ratios(); + MS_ASSERT(aspect_ratios != nullptr); + for (auto i = 0; i < aspect_ratios->size(); i++) { + float ratio = (*aspect_ratios)[i]; + bool exist = std::any_of(different_aspect_ratios.begin(), different_aspect_ratios.end(), [&](float v) { + return abs(ratio - v) < 1e-6; + }); + if (!exist) { + different_aspect_ratios.emplace_back(ratio); + if (param->flip()) { + different_aspect_ratios.emplace_back(1.0f / ratio); + } + } + } + int32_t num_priors_box = param->min_sizes()->size() * different_aspect_ratios.size() + param->max_sizes()->size(); + auto input = inputs_.at(0); + MS_ASSERT(input != nullptr); + int32_t h = input->Height() * input->Width() * num_priors_box * kPriorBoxPoints; + + std::vector output_shape{kPriorBoxN, h, kPriorBoxW, kPriorBoxC}; + auto output = outputs_.at(0); + MS_ASSERT(output != nullptr); + + output->set_shape(output_shape); + output->set_data_type(kNumberTypeFloat32); + output->SetFormat(input->GetFormat()); + return RET_OK; +} +} // namespace mindspore::lite diff --git a/mindspore/lite/src/populate_parameter.cc b/mindspore/lite/src/populate_parameter.cc index 87c099e6848..1221b994204 100644 --- a/mindspore/lite/src/populate_parameter.cc +++ b/mindspore/lite/src/populate_parameter.cc @@ -62,6 +62,7 @@ #include "src/runtime/kernel/arm/opclib/fp32/unsqueeze.h" #include "src/runtime/kernel/arm/opclib/fp32/one_hot.h" #include "src/runtime/kernel/arm/opclib/fp32/strided_slice.h" +#include "src/runtime/kernel/arm/base/prior_box.h" namespace mindspore::kernel { FillParameter *PopulateFillParam(const lite::Primitive *primitive) { @@ -990,6 +991,60 @@ OpParameter *PopulateAddNParam(const lite::Primitive *primitive) { return parameter; } +PriorBoxParameter *PopulatePriorBoxParameter(const lite::Primitive *primitive) { + PriorBoxParameter *param = new (std::nothrow) PriorBoxParameter(); + if (param == nullptr) { + MS_LOG(ERROR) << "new PriorBoxParameter failed."; + return nullptr; + } + param->op_parameter_.type_ = primitive->Type(); + auto prior_box_param = primitive->Value()->value_as_PriorBox(); + + if (prior_box_param->min_sizes()->size() > PRIOR_BOX_MAX_NUM) { + MS_LOG(ERROR) << "PriorBox min_sizes size exceeds max num " << PRIOR_BOX_MAX_NUM << ", got " + << prior_box_param->min_sizes(); + delete (param); + return nullptr; + } + param->min_sizes_size = prior_box_param->min_sizes()->size(); + if (prior_box_param->max_sizes()->size() > PRIOR_BOX_MAX_NUM) { + MS_LOG(ERROR) << "PriorBox max_sizes size exceeds max num " << PRIOR_BOX_MAX_NUM << ", got " + << prior_box_param->max_sizes(); + delete (param); + return nullptr; + } + param->max_sizes_size = prior_box_param->max_sizes()->size(); + (void)memcpy(param->max_sizes, prior_box_param->max_sizes()->data(), + prior_box_param->max_sizes()->size() * sizeof(int32_t)); + (void)memcpy(param->min_sizes, prior_box_param->min_sizes()->data(), + prior_box_param->min_sizes()->size() * sizeof(int32_t)); + + if (prior_box_param->aspect_ratios()->size() > PRIOR_BOX_MAX_NUM) { + MS_LOG(ERROR) << "PriorBox aspect_ratios size exceeds max num " << PRIOR_BOX_MAX_NUM << ", got " + << prior_box_param->aspect_ratios(); + delete (param); + return nullptr; + } + param->aspect_ratios_size = prior_box_param->aspect_ratios()->size(); + (void)memcpy(param->aspect_ratios, prior_box_param->aspect_ratios()->data(), + prior_box_param->aspect_ratios()->size() * sizeof(float)); + if (prior_box_param->variances()->size() != PRIOR_BOX_VAR_NUM) { + MS_LOG(ERROR) << "PriorBox variances size should be " << PRIOR_BOX_VAR_NUM << ", got " + << prior_box_param->variances()->size(); + delete (param); + return nullptr; + } + (void)memcpy(param->variances, prior_box_param->variances()->data(), PRIOR_BOX_VAR_NUM * sizeof(float)); + param->flip = prior_box_param->flip(); + param->clip = prior_box_param->clip(); + param->offset = prior_box_param->offset(); + param->image_size_h = prior_box_param->image_size_h(); + param->image_size_w = prior_box_param->image_size_w(); + param->step_h = prior_box_param->step_h(); + param->step_w = prior_box_param->step_w(); + return param; +} + OpParameter *PopulateParameter(const lite::Primitive *primitive) { MS_EXCEPTION_IF_NULL(primitive); auto op_type = primitive->Type(); @@ -1109,6 +1164,8 @@ OpParameter *PopulateParameter(const lite::Primitive *primitive) { return reinterpret_cast(PopulateOneHotParameter(primitive)); case schema::PrimitiveType_AddN: return reinterpret_cast(PopulateAddNParam(primitive)); + case schema::PrimitiveType_PriorBox: + return reinterpret_cast(PopulatePriorBoxParameter(primitive)); default: break; } diff --git a/mindspore/lite/src/runtime/kernel/arm/base/prior_box.cc b/mindspore/lite/src/runtime/kernel/arm/base/prior_box.cc new file mode 100644 index 00000000000..e09ec8889c5 --- /dev/null +++ b/mindspore/lite/src/runtime/kernel/arm/base/prior_box.cc @@ -0,0 +1,193 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "src/runtime/kernel/arm/base/prior_box.h" +#include "schema/model_generated.h" +#include "src/kernel_factory.h" +#include "include/errorcode.h" +#include "include/context.h" +#include "src/runtime/runtime_api.h" + +using mindspore::lite::KernelRegistrar; +using mindspore::lite::RET_ERROR; +using mindspore::lite::RET_NULL_PTR; +using mindspore::lite::RET_OK; +using mindspore::schema::PrimitiveType_PriorBox; + +namespace mindspore::kernel { +namespace { +constexpr int kInputNum = 2; +constexpr int kOutputNum = 1; +} // namespace +int PriorBoxCPUKernel::Init() { + if (prior_box_param_ == nullptr) { + MS_LOG(ERROR) << "PriorBoxParameter nullptr"; + return RET_NULL_PTR; + } + MS_ASSERT(inputs_.size() == kInputNum); + MS_ASSERT(outputs_.size() == kOutputNum); + + auto ret = GeneratePriorBox(); + + return ret; +} + +int PriorBoxCPUKernel::GeneratePriorBox() { + const int fmap_w = inputs_[0]->Width(); + const int fmap_h = inputs_[0]->Height(); + + const int image_w = prior_box_param_->image_size_w > 0 ? prior_box_param_->image_size_w : inputs_[1]->Width(); + const int image_h = prior_box_param_->image_size_h > 0 ? prior_box_param_->image_size_h : inputs_[1]->Height(); + + const float step_w = + prior_box_param_->step_w > 0.0f ? prior_box_param_->step_w : static_cast(image_w) / fmap_w; + const float step_h = + prior_box_param_->step_h > 0.0f ? prior_box_param_->step_h : static_cast(image_h) / fmap_h; + + std::vector different_aspect_ratios{1.0f}; + auto aspect_ratios = prior_box_param_->aspect_ratios; + MS_ASSERT(aspect_ratios != nullptr); + for (auto i = 0; i < prior_box_param_->aspect_ratios_size; i++) { + float ratio = aspect_ratios[i]; + bool exist = std::any_of(different_aspect_ratios.begin(), different_aspect_ratios.end(), + [&](float v) { return abs(ratio - v) < 1e-6; }); + if (!exist) { + different_aspect_ratios.emplace_back(ratio); + if (prior_box_param_->flip) { + different_aspect_ratios.emplace_back(1.0f / ratio); + } + } + } + + for (int i = 0; i < fmap_h; i++) { + float cy = i + prior_box_param_->offset; + for (int j = 0; j < fmap_w; j++) { + float cx = j + prior_box_param_->offset; + for (auto k = 0; k < prior_box_param_->min_sizes_size; k++) { + float min_size = prior_box_param_->min_sizes[k]; + output_.emplace_back((cx - min_size / step_w * 0.5f) / fmap_w); + output_.emplace_back((cy - min_size / step_h * 0.5f) / fmap_h); + output_.emplace_back((cx + min_size / step_w * 0.5f) / fmap_w); + output_.emplace_back((cy + min_size / step_h * 0.5f) / fmap_h); + + if (prior_box_param_->max_sizes_size > 0) { + float max_size = prior_box_param_->max_sizes[k]; + float prime = sqrt(min_size * max_size); + output_.emplace_back((cx - prime / step_w * 0.5f) / fmap_w); + output_.emplace_back((cy - prime / step_h * 0.5f) / fmap_h); + output_.emplace_back((cx + prime / step_w * 0.5f) / fmap_w); + output_.emplace_back((cy + prime / step_h * 0.5f) / fmap_h); + } + + for (auto v : different_aspect_ratios) { + if (abs(v - 1.0f) < 1e-6) { + continue; + } + float as_square_root = sqrt(v); + float box_w = min_size * as_square_root; + float box_h = min_size / as_square_root; + output_.emplace_back((cx - box_w / step_w * 0.5f) / fmap_w); + output_.emplace_back((cy - box_h / step_h * 0.5f) / fmap_h); + output_.emplace_back((cx + box_w / step_w * 0.5f) / fmap_w); + output_.emplace_back((cy + box_h / step_h * 0.5f) / fmap_h); + } + } + } + } + + // do clip + if (prior_box_param_->clip) { + for (auto item : output_) { + if (item > 1.0f) { + item = 1.0f; + } + if (item < 0.0f) { + item = 0.0f; + } + } + } + + // variance + for (auto i = 0; i < outputs_[0]->Height() / PRIOR_BOX_VAR_NUM; i++) { + for (auto j = 0; j < PRIOR_BOX_VAR_NUM; j++) { + output_.emplace_back(prior_box_param_->variances[j]); + } + } + return RET_OK; +} + +int PriorBoxCPUKernel::PriorBoxImpl(int task_id) { + auto src = output_.data(); + auto output = outputs_.at(0); + if (output == nullptr) { + return RET_NULL_PTR; + } + auto ret = PriorBox(src, reinterpret_cast(output->Data()), output_.size(), task_id, thread_count_); + return ret; +} + +int RunPriorBox(int task_id, LiteParallelGroupEnv *penv, void *cdata) { + auto prior_box = reinterpret_cast(cdata); + + auto error_code = prior_box->PriorBoxImpl(task_id); + if (error_code != RET_OK) { + MS_LOG(ERROR) << "Resize Run error task_id[" << task_id << "] error_code[" << error_code << "]"; + return RET_ERROR; + } + return RET_OK; +} + +int PriorBoxCPUKernel::Run() { + int error_code = LiteBackendParallelLaunch(RunPriorBox, this, thread_count_); + if (error_code != RET_OK) { + MS_LOG(ERROR) << "PriorBox run error, error_code[" << error_code << "]"; + return RET_ERROR; + } + return RET_OK; +} + +kernel::LiteKernel *CpuPriorBoxKernelCreator(const std::vector &inputs, + const std::vector &outputs, + OpParameter *opParameter, const Context *ctx, + const kernel::KernelKey &desc) { + if (opParameter == nullptr) { + MS_LOG(ERROR) << "Input opParameter is nullptr!"; + return nullptr; + } + if (desc.type != schema::PrimitiveType_PriorBox) { + MS_LOG(ERROR) << "PriorBox invalid desc type " << desc.type; + return nullptr; + } + auto *kernel = new (std::nothrow) PriorBoxCPUKernel(opParameter, inputs, outputs, ctx); + if (kernel == nullptr) { + MS_LOG(ERROR) << "new PriorBoxCPUKernel fail!"; + return nullptr; + } + auto ret = kernel->Init(); + if (ret != RET_OK) { + delete kernel; + MS_LOG(ERROR) << "Init kernel failed, name: " << opParameter->name_ << ", type: " + << schema::EnumNamePrimitiveType(static_cast(opParameter->type_)); + return nullptr; + } + return kernel; +} + +REG_KERNEL(kCPU, kNumberTypeFloat32, PrimitiveType_PriorBox, CpuPriorBoxKernelCreator) +REG_KERNEL(kCPU, kNumberTypeInt8, PrimitiveType_PriorBox, CpuPriorBoxKernelCreator) +} // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/base/prior_box.h b/mindspore/lite/src/runtime/kernel/arm/base/prior_box.h new file mode 100644 index 00000000000..3fe698f930f --- /dev/null +++ b/mindspore/lite/src/runtime/kernel/arm/base/prior_box.h @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_BASE_PRIOR_BOX_H_ +#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_BASE_PRIOR_BOX_H_ + +#include +#include "src/lite_kernel.h" +#include "src/runtime/kernel/arm/opclib/reshape_parameter.h" +#include "src/runtime/kernel/arm/opclib/prior_box.h" + +using mindspore::lite::Context; + +namespace mindspore::kernel { +class PriorBoxCPUKernel : public LiteKernel { + public: + PriorBoxCPUKernel(OpParameter *parameter, const std::vector &inputs, + const std::vector &outputs, const Context *ctx) + : LiteKernel(parameter, inputs, outputs), ctx_(ctx), thread_count_(ctx->threadNum) { + prior_box_param_ = reinterpret_cast(opParameter); + } + ~PriorBoxCPUKernel() = default; + + int Init() override; + int ReSize() override { return 0; } + int Run() override; + int PriorBoxImpl(int task_id); + + protected: + int thread_count_; + const Context *ctx_; + + private: + std::vector output_; + PriorBoxParameter *prior_box_param_; + int GeneratePriorBox(); +}; +} // namespace mindspore::kernel + +#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_BASE_PRIOR_BOX_H_ diff --git a/mindspore/lite/src/runtime/kernel/arm/opclib/prior_box.cc b/mindspore/lite/src/runtime/kernel/arm/opclib/prior_box.cc new file mode 100644 index 00000000000..233393af54e --- /dev/null +++ b/mindspore/lite/src/runtime/kernel/arm/opclib/prior_box.cc @@ -0,0 +1,30 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "src/runtime/kernel/arm/opclib/errorcode.h" +#include "src/runtime/kernel/arm/opclib/prior_box.h" + +int PriorBox(const float *input_data, float *output_data, const size_t size, const int tid, const int thread_num) { + size_t unit_size = size / thread_num; + if (tid == thread_num - 1) { + size_t tail_size = size - unit_size * tid; + (void)memcpy(output_data + tid * unit_size, input_data + tid * unit_size, tail_size * sizeof(float)); + } else { + (void)memcpy(output_data + tid * unit_size, input_data + tid * unit_size, unit_size * sizeof(float)); + } + return OPCLIB_OK; +} diff --git a/mindspore/lite/src/runtime/kernel/arm/opclib/prior_box.h b/mindspore/lite/src/runtime/kernel/arm/opclib/prior_box.h new file mode 100644 index 00000000000..ce7fae2b4fc --- /dev/null +++ b/mindspore/lite/src/runtime/kernel/arm/opclib/prior_box.h @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_OPCLIB_PRIOR_BOX_H_ +#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_OPCLIB_PRIOR_BOX_H_ + +#ifdef ENABLE_NEON +#include +#endif +#include +#include "src/runtime/kernel/arm/opclib/op_base.h" +#define PRIOR_BOX_MAX_NUM 8 +#define PRIOR_BOX_VAR_NUM 4 +struct PriorBoxParameter { + OpParameter op_parameter_; + int32_t min_sizes_size; + int32_t min_sizes[PRIOR_BOX_MAX_NUM]; + int32_t max_sizes_size; + int32_t max_sizes[PRIOR_BOX_MAX_NUM]; + int32_t aspect_ratios_size; + float aspect_ratios[PRIOR_BOX_MAX_NUM]; + float variances[PRIOR_BOX_VAR_NUM]; + int32_t image_size_w; + int32_t image_size_h; + float step_w; + float step_h; + bool clip; + bool flip; + float offset; +}; + +int PriorBox(const float *input_data, float *output_data, const size_t size, const int tid, const int thread_num); +#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_OPCLIB_PRIOR_BOX_H_