From 63343d10f22931508c861db7b62986c3271213ed Mon Sep 17 00:00:00 2001 From: zhengjun10 Date: Sat, 29 Aug 2020 12:37:44 +0800 Subject: [PATCH] add infershape and trans format optimize --- mindspore/lite/tools/common/node_util.cc | 6 + mindspore/lite/tools/common/node_util.h | 2 + .../tools/converter/graphdef_transform.cc | 11 +- .../legacy_optimizer/graph/CMakeLists.txt | 4 +- .../legacy_optimizer/graph/infershape_pass.cc | 123 ++++++++++++++++++ .../legacy_optimizer/graph/infershape_pass.h | 40 ++++++ ...ns_pass.cc => trans_format_insert_pass.cc} | 19 ++- ...rans_pass.h => trans_format_insert_pass.h} | 6 +- .../graph/trans_format_remove_pass.cc | 49 +++++++ .../graph/trans_format_remove_pass.h | 40 ++++++ mindspore/lite/tools/converter/optimizer.cc | 4 +- 11 files changed, 289 insertions(+), 15 deletions(-) create mode 100644 mindspore/lite/tools/converter/legacy_optimizer/graph/infershape_pass.cc create mode 100644 mindspore/lite/tools/converter/legacy_optimizer/graph/infershape_pass.h rename mindspore/lite/tools/converter/legacy_optimizer/graph/{eltwise_format_trans_pass.cc => trans_format_insert_pass.cc} (90%) rename mindspore/lite/tools/converter/legacy_optimizer/graph/{eltwise_format_trans_pass.h => trans_format_insert_pass.h} (90%) create mode 100644 mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_remove_pass.cc create mode 100644 mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_remove_pass.h diff --git a/mindspore/lite/tools/common/node_util.cc b/mindspore/lite/tools/common/node_util.cc index e112b307331..de8cf2c3052 100644 --- a/mindspore/lite/tools/common/node_util.cc +++ b/mindspore/lite/tools/common/node_util.cc @@ -52,6 +52,12 @@ static const std::vector int8OpList = { schema::PrimitiveType_Squeeze, schema::PrimitiveType_Sub, schema::PrimitiveType_TopK, schema::PrimitiveType_Unsqueeze}; +static const std::vector needInsertOpList = { + schema::PrimitiveType_Eltwise, schema::PrimitiveType_Activation, + schema::PrimitiveType_Concat, schema::PrimitiveType_Power}; + +std::vector GetInsertOpList() { return needInsertOpList; } + std::vector Getfp32FullOpList() { return fp32FullOpList; } std::vector GetNhwcOpList() { return nhwcOpList; } diff --git a/mindspore/lite/tools/common/node_util.h b/mindspore/lite/tools/common/node_util.h index 6f619b3a243..840fde44128 100644 --- a/mindspore/lite/tools/common/node_util.h +++ b/mindspore/lite/tools/common/node_util.h @@ -30,6 +30,8 @@ namespace lite { using STATUS = int; STATUS BroadCastQuantParam(schema::MetaGraphT *graphT, const std::unique_ptr &node); +std::vector GetInsertOpList(); + std::vector GetNhwcOpList(); std::vector Getfp32FullOpList(); diff --git a/mindspore/lite/tools/converter/graphdef_transform.cc b/mindspore/lite/tools/converter/graphdef_transform.cc index b74d9b57e8c..7b7a8cc6552 100644 --- a/mindspore/lite/tools/converter/graphdef_transform.cc +++ b/mindspore/lite/tools/converter/graphdef_transform.cc @@ -25,11 +25,13 @@ #include "tools/converter/legacy_optimizer/fusion/format_trans_transpose_fusion_pass.h" #include "tools/converter/legacy_optimizer/fusion/quant_cast_fusion_pass.h" #include "tools/converter/legacy_optimizer/fusion/mul_add_fusion_pass.h" +#include "tools/converter/legacy_optimizer/graph/trans_format_remove_pass.h" +#include "tools/converter/legacy_optimizer/graph/infershape_pass.h" #include "tools/converter/legacy_optimizer/graph/batchnorm_convert_scale_pass.h" #include "tools/converter/legacy_optimizer/graph/weight_format_hardcode_pass.h" #include "tools/converter/legacy_optimizer/graph/weight_format_transform_pass.h" #include "tools/converter/legacy_optimizer/graph/format_trans_pass.h" -#include "tools/converter/legacy_optimizer/graph/eltwise_format_trans_pass.h" +#include "tools/converter/legacy_optimizer/graph/trans_format_insert_pass.h" #include "tools/converter/legacy_optimizer/graph/isolated_node_remove_pass.h" #include "tools/converter/legacy_optimizer/graph/unused_node_remove_pass.h" #include "tools/converter/legacy_optimizer/graph/topological_sort_pass.h" @@ -145,11 +147,14 @@ int GraphDefTransform::Transform(const converter::Flags &ctx) { formatTransPass->SetQuantType(ctx.quantType); formatTransPass->SetFmk(ctx.fmk); formatTransOptimizer.AddPass(formatTransPass); - formatTransOptimizer.AddPass(new EltwiseFormatTransPass()); + formatTransOptimizer.AddPass(new (std::nothrow) TopologicalSortPass()); + formatTransOptimizer.AddPass(new (std::nothrow) InferShapePass()); + formatTransOptimizer.AddPass(new (std::nothrow) TransOpRemovePass()); + formatTransOptimizer.AddPass(new (std::nothrow) TransOpInsertPass()); formatTransOptimizer.AddPass(new (std::nothrow) FormatTransFusionPass()); formatTransOptimizer.AddPass(new (std::nothrow) IsolatedNodeRemovePass()); status = formatTransOptimizer.Run(graphDefT); - if (status != RET_OK && status != RET_NO_CHANGE) { + if (status != RET_OK && status != RET_NO_CHANGE && status != RET_INFER_ERR) { MS_LOG(ERROR) << "Run formatTransOptimizer graphPasses Failed"; return status; } diff --git a/mindspore/lite/tools/converter/legacy_optimizer/graph/CMakeLists.txt b/mindspore/lite/tools/converter/legacy_optimizer/graph/CMakeLists.txt index eede1eb0d7a..919b03265c6 100755 --- a/mindspore/lite/tools/converter/legacy_optimizer/graph/CMakeLists.txt +++ b/mindspore/lite/tools/converter/legacy_optimizer/graph/CMakeLists.txt @@ -1,6 +1,6 @@ add_library(graph_pass_mid OBJECT ${CMAKE_CURRENT_SOURCE_DIR}/format_trans_pass.cc - ${CMAKE_CURRENT_SOURCE_DIR}/eltwise_format_trans_pass.cc + ${CMAKE_CURRENT_SOURCE_DIR}/trans_format_insert_pass.cc ${CMAKE_CURRENT_SOURCE_DIR}/dtype_trans_pass.cc ${CMAKE_CURRENT_SOURCE_DIR}/isolated_node_remove_pass.cc ${CMAKE_CURRENT_SOURCE_DIR}/model_input_format_preprocess_pass.cc @@ -9,4 +9,6 @@ add_library(graph_pass_mid OBJECT ${CMAKE_CURRENT_SOURCE_DIR}/topological_sort_pass.cc ${CMAKE_CURRENT_SOURCE_DIR}/unused_node_remove_pass.cc ${CMAKE_CURRENT_SOURCE_DIR}/batchnorm_convert_scale_pass.cc + ${CMAKE_CURRENT_SOURCE_DIR}/trans_format_remove_pass.cc + ${CMAKE_CURRENT_SOURCE_DIR}/infershape_pass.cc ) diff --git a/mindspore/lite/tools/converter/legacy_optimizer/graph/infershape_pass.cc b/mindspore/lite/tools/converter/legacy_optimizer/graph/infershape_pass.cc new file mode 100644 index 00000000000..90daca519c8 --- /dev/null +++ b/mindspore/lite/tools/converter/legacy_optimizer/graph/infershape_pass.cc @@ -0,0 +1,123 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tools/converter/legacy_optimizer/graph/infershape_pass.h" +#include +#include "utils/log_adapter.h" +#include "include/errorcode.h" +#include "src/ir/tensor.h" +#include "src/ops/primitive_c.h" + +using mindspore::lite::tensor::Tensor; +using mindspore::lite::PrimitiveC; +namespace mindspore { +namespace lite { +namespace { +std::vector ConvertTensorToLiteTensor(MetaGraphT *graph, const std::vector &tensor_indexs, + const schema::PrimitiveType node_type) { + std::vector lite_tensors; + for (size_t i = 0; i < tensor_indexs.size(); i++) { + auto &tensorT = graph->allTensors.at(tensor_indexs[i]); + auto tensor_shape = tensorT->dims; + auto lite_tensor = + new(std::nothrow) tensor::Tensor(TypeId(tensorT->dataType), tensor_shape, tensorT->format, tensorT->nodeType); + if (lite_tensor == nullptr) { + MS_LOG(ERROR) << "lite tensor is nullptr"; + return std::vector(); + } + // reshape op must get tensor data to infershape + if (node_type == schema::PrimitiveType_Reshape && i == 1 && tensorT->nodeType == NodeType_ValueNode) { + auto lite_tensor_size = tensorT->data.size() * sizeof(uint8_t); + // when tensorT as param input + if (lite_tensor_size == 0) { + delete lite_tensor; + return std::vector(); + } + auto tensor_data = new(std::nothrow) char[lite_tensor_size / sizeof(char)]; + if (tensor_data == nullptr) { + MS_LOG(ERROR) << "tensor_data is nullptr"; + delete lite_tensor; + return std::vector(); + } + auto ret = memcpy_s(tensor_data, lite_tensor_size, tensorT->data.data(), lite_tensor_size); + if (ret != EOK) { + delete lite_tensor; + delete[] tensor_data; + MS_LOG(ERROR) << "memcpy error: " << ret; + return std::vector(); + } + lite_tensor->SetData(tensor_data); + lite_tensors.emplace_back(lite_tensor); + continue; + } + lite_tensors.emplace_back(lite_tensor); + } + return lite_tensors; +} +} // namespace +STATUS InferShapePass::Run(MetaGraphT *graph) { + MS_ASSERT(graph != nullptr); + for (auto iter = graph->nodes.begin(); iter != graph->nodes.end(); iter++) { + auto &node = *iter; + auto input_tensors = ConvertTensorToLiteTensor(graph, node->inputIndex, node->primitive->value.type); + if (input_tensors.empty() || input_tensors.size() != node->inputIndex.size()) { + MS_LOG(ERROR) << "convert input lite tensor error"; + return RET_INFER_ERR; + } + auto output_tensors = ConvertTensorToLiteTensor(graph, node->outputIndex, node->primitive->value.type); + if (output_tensors.empty() || output_tensors.size() != node->outputIndex.size()) { + MS_LOG(ERROR) << "convert output lite tensor error"; + return RET_INFER_ERR; + } + std::unique_ptr primitiveT(new (std::nothrow) PrimitiveT(*node->primitive)); + if (primitiveT == nullptr) { + MS_LOG(ERROR) << "copy primitiveT error"; + return RET_ERROR; + } + auto primitiveC = std::shared_ptr(PrimitiveC::UnPackFromSchemaPrimitiveT(primitiveT.release())); + if (primitiveC == nullptr) { + MS_LOG(ERROR) << "unpack primitiveT error"; + return RET_ERROR; + } + auto ret = primitiveC->InferShape(input_tensors, output_tensors); + if (ret == RET_INFER_INVALID) { + MS_LOG(INFO) << "InferShape shouldn't be done before runtime, name: " << node->name + << ", type: " << schema::EnumNamePrimitiveType(node->primitive->value.type) + << "flag set to false."; + } else if (ret != RET_OK) { + MS_LOG(WARNING) << "InferShape failed, name: " << node->name + << ", type: " << schema::EnumNamePrimitiveType(node->primitive->value.type); + return RET_INFER_ERR; + } + // copy output shape to tensorT + for (size_t i = 0; i < output_tensors.size(); i++) { + auto output_dims = output_tensors[i]->shape(); + auto &output_tensor = graph->allTensors.at(node->outputIndex[i]); + output_tensor->dims.swap(output_dims); + output_tensor->format = output_tensors[i]->GetFormat(); + output_tensor->dataType = output_tensors[i]->data_type(); + } + for (auto input_tensor : input_tensors) { + delete input_tensor; + } + for (auto output_tensor : output_tensors) { + delete output_tensor; + } + } + return RET_OK; +} +} // namespace lite +} // namespace mindspore diff --git a/mindspore/lite/tools/converter/legacy_optimizer/graph/infershape_pass.h b/mindspore/lite/tools/converter/legacy_optimizer/graph/infershape_pass.h new file mode 100644 index 00000000000..fb5e1104d7c --- /dev/null +++ b/mindspore/lite/tools/converter/legacy_optimizer/graph/infershape_pass.h @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_PREDICT_INFERSHAPE_PASS_H +#define MINDSPORE_PREDICT_INFERSHAPE_PASS_H + +#include +#include +#include +#include +#include "tools/common/graph_util.h" +#include "tools/converter/optimizer.h" + +using mindspore::schema::TensorT; +namespace mindspore { +namespace lite { +class InferShapePass : public GraphPass { + public: + InferShapePass() = default; + + ~InferShapePass() = default; + + STATUS Run(MetaGraphT *graph) override; +}; +} // namespace lite +} // namespace mindspore +#endif // MINDSPORE_PREDICT_INFERSHAPE_PASS_H diff --git a/mindspore/lite/tools/converter/legacy_optimizer/graph/eltwise_format_trans_pass.cc b/mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_insert_pass.cc similarity index 90% rename from mindspore/lite/tools/converter/legacy_optimizer/graph/eltwise_format_trans_pass.cc rename to mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_insert_pass.cc index 17e7bffbb80..81f61dbe973 100644 --- a/mindspore/lite/tools/converter/legacy_optimizer/graph/eltwise_format_trans_pass.cc +++ b/mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_insert_pass.cc @@ -17,7 +17,7 @@ #include #include #include -#include "tools/converter/legacy_optimizer/graph/eltwise_format_trans_pass.h" +#include "tools/converter/legacy_optimizer/graph/trans_format_insert_pass.h" #include "tools/common/converter_op_utils.h" #include "tools/common/node_util.h" #include "utils/log_adapter.h" @@ -25,7 +25,7 @@ namespace mindspore { namespace lite { -bool EltwiseFormatTransPass::CanFusion(schema::MetaGraphT *graph, const std::unique_ptr &node) { +bool TransOpInsertPass::CanFusion(schema::MetaGraphT *graph, const std::unique_ptr &node) { auto input_node_indexes = GetInputNodeIdx(*graph, *node); pre_type_ = schema::PrimitiveType_NONE; size_t has_trans_count = 0; @@ -95,7 +95,7 @@ bool EltwiseFormatTransPass::CanFusion(schema::MetaGraphT *graph, const std::uni return can_fusion; } -STATUS EltwiseFormatTransPass::FindOutTransType() { +STATUS TransOpInsertPass::FindOutTransType() { pre_insert_trans_type_ = kNHWC2NCHW; post_insert_trans_type_ = kNHWC2NCHW; if (pre_type_ == PrimitiveType_NONE && post_type_ != PrimitiveType_NONE) { @@ -117,12 +117,12 @@ STATUS EltwiseFormatTransPass::FindOutTransType() { return RET_OK; } -STATUS EltwiseFormatTransPass::Run(schema::MetaGraphT *graph) { +STATUS TransOpInsertPass::Run(schema::MetaGraphT *graph) { MS_ASSERT(graph != nullptr); for (auto iter = graph->nodes.begin(); iter != graph->nodes.end(); iter++) { auto &node = *iter; auto type = node->primitive->value.type; - if (type != PrimitiveType_Eltwise && type != PrimitiveType_Activation) { + if (!IsContain(GetInsertOpList(), type)) { continue; } auto node_name = node->name; @@ -134,7 +134,14 @@ STATUS EltwiseFormatTransPass::Run(schema::MetaGraphT *graph) { MS_LOG(ERROR) << "FindOutTransType error"; return ret; } - + // 4 dims means infershape success,can delete + if (type == PrimitiveType_Concat) { + if (graph->allTensors.at(node->inputIndex[0])->dims.size() == 4) { + node->primitive->value.AsConcat()->axis = -1; + } else { + continue; + } + } STATUS status = RET_OK; auto input_tensor_size = (*iter)->inputIndex.size(); for (size_t i = 0; i < input_tensor_size; i++) { diff --git a/mindspore/lite/tools/converter/legacy_optimizer/graph/eltwise_format_trans_pass.h b/mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_insert_pass.h similarity index 90% rename from mindspore/lite/tools/converter/legacy_optimizer/graph/eltwise_format_trans_pass.h rename to mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_insert_pass.h index 1f62883d797..22a3798931f 100644 --- a/mindspore/lite/tools/converter/legacy_optimizer/graph/eltwise_format_trans_pass.h +++ b/mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_insert_pass.h @@ -24,11 +24,11 @@ namespace mindspore { namespace lite { -class EltwiseFormatTransPass : public FormatTransPass { +class TransOpInsertPass : public FormatTransPass { public: - EltwiseFormatTransPass() : FormatTransPass() {} + TransOpInsertPass() : FormatTransPass() {} - ~EltwiseFormatTransPass() override = default; + ~TransOpInsertPass() override = default; STATUS Run(schema::MetaGraphT *graph) override; diff --git a/mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_remove_pass.cc b/mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_remove_pass.cc new file mode 100644 index 00000000000..61c28f5a0a6 --- /dev/null +++ b/mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_remove_pass.cc @@ -0,0 +1,49 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tools/converter/legacy_optimizer/graph/trans_format_remove_pass.h" +#include +#include "utils/log_adapter.h" +#include "include/errorcode.h" +#include "tools/common/graph_util.h" +#include "src/ir/tensor.h" +#include "src/ops/primitive_c.h" + +using mindspore::lite::tensor::Tensor; +using mindspore::lite::PrimitiveC; +namespace mindspore { +namespace lite { +STATUS TransOpRemovePass::Run(MetaGraphT *graph) { + MS_ASSERT(graph != nullptr); + for (auto iter = graph->nodes.begin(); iter != graph->nodes.end(); iter++) { + auto &node = *iter; + auto type = node->primitive->value.type; + if (type == schema::PrimitiveType_Nchw2Nhwc || type == schema::PrimitiveType_Nhwc2Nchw) { + auto &input_tensor = graph->allTensors.at(node->inputIndex.at(0)); + // less than 4 dims can delete + if (!input_tensor->dims.empty() && input_tensor->dims.size() < 4) { + auto status = IsolateOneWayNode(graph, node.get(), true); + if (status != RET_OK) { + MS_LOG(ERROR) << "IsolateOneWayNode failed, node: " << node->name.c_str() << ", error: " << status; + return status; + } + } + } + } + return RET_OK; +} +} // namespace lite +} // namespace mindspore diff --git a/mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_remove_pass.h b/mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_remove_pass.h new file mode 100644 index 00000000000..8d5ce114842 --- /dev/null +++ b/mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_remove_pass.h @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_PREDICT_TRANS_FORMAT_REMOVE_PASS_H +#define MINDSPORE_PREDICT_TRANS_FORMAT_REMOVE_PASS_H + +#include +#include +#include +#include +#include "tools/common/graph_util.h" +#include "tools/converter/optimizer.h" + +using mindspore::schema::TensorT; +namespace mindspore { +namespace lite { +class TransOpRemovePass : public GraphPass { + public: + TransOpRemovePass() = default; + + ~TransOpRemovePass() = default; + + STATUS Run(MetaGraphT *graph) override; +}; +} // namespace lite +} // namespace mindspore +#endif // MINDSPORE_PREDICT_TRANS_FORMAT_REMOVE_PASS_H diff --git a/mindspore/lite/tools/converter/optimizer.cc b/mindspore/lite/tools/converter/optimizer.cc index d043138931c..4fd6bc239ff 100644 --- a/mindspore/lite/tools/converter/optimizer.cc +++ b/mindspore/lite/tools/converter/optimizer.cc @@ -52,7 +52,7 @@ STATUS Optimizer::Run(schema::MetaGraphT *graphDefT) { for (auto &opDef : graphDefT->nodes) { for (auto pass : this->nodePasses) { status = pass->Run(new GraphNode(graphDefT, opDef.get())); - if (status != RET_OK && status != RET_NO_CHANGE) { + if (status != RET_OK && status != RET_NO_CHANGE && status != RET_INFER_ERR) { MS_LOG(ERROR) << "Run NodePass failed"; return status; } else { @@ -65,7 +65,7 @@ STATUS Optimizer::Run(schema::MetaGraphT *graphDefT) { for (auto pass : this->graphPasses) { status = pass->Run(graphDefT); - if (status != RET_OK && status != RET_NO_CHANGE) { + if (status != RET_OK && status != RET_NO_CHANGE && status != RET_INFER_ERR) { MS_LOG(ERROR) << "Run GraphPass failed"; return status; } else {