From 48e9092d21e4f3b3ee8e12c0c90736bba47d85ff Mon Sep 17 00:00:00 2001 From: chengxianbin Date: Wed, 9 Nov 2022 11:09:50 +0800 Subject: [PATCH] multi-models shared workspace memory and converter of dpico decoupling with benchmark --- include/api/model_group.h | 68 +++++ .../kernel/aicpu/aicpu_ops/CMakeLists.txt | 1 + .../device/cpu/kernel/nnacl/CMakeLists.txt | 2 +- mindspore/lite/python/CMakeLists.txt | 6 +- mindspore/lite/src/CMakeLists.txt | 1 + .../src/litert/cxx_api/model/model_group.cc | 56 ++++ .../litert/cxx_api/model/model_group_impl.cc | 130 ++++++++ .../litert/cxx_api/model/model_group_impl.h | 51 ++++ .../src/litert/cxx_api/model/model_impl.cc | 32 ++ .../src/litert/cxx_api/model/model_impl.h | 2 + .../kernel/ascend/src/acl_mem_manager.cc | 86 ++++++ .../kernel/ascend/src/acl_mem_manager.h | 63 ++++ .../litert/kernel/ascend/src/custom_kernel.cc | 21 +- .../litert/kernel/ascend/src/custom_kernel.h | 1 + .../litert/kernel/ascend/src/model_infer.cc | 70 ++++- .../litert/kernel/ascend/src/model_infer.h | 7 +- .../litert/kernel/ascend/src/model_process.cc | 17 ++ .../litert/kernel/ascend/src/model_process.h | 3 + mindspore/lite/src/litert/lite_session.cc | 2 +- mindspore/lite/src/litert/lite_session.h | 2 + mindspore/lite/src/litert/model_manager.cc | 39 +++ mindspore/lite/src/litert/model_manager.h | 53 ++++ .../lite/test/config_level0/cropped_size.cfg | 2 +- .../lite/test/config_level1/cropped_size.cfg | 2 +- mindspore/lite/tools/converter/CMakeLists.txt | 1 + .../converter/adapter/dpico/CMakeLists.txt | 2 + .../adapter/dpico/src/calib_data_generator.cc | 129 ++------ .../adapter/dpico/src/calib_data_generator.h | 13 +- .../lite/tools/converter/adapter/utils.cc | 280 ++++++++++++++++++ .../lite/tools/converter/adapter/utils.h | 42 +++ 30 files changed, 1063 insertions(+), 121 deletions(-) create mode 100755 include/api/model_group.h create mode 100755 mindspore/lite/src/litert/cxx_api/model/model_group.cc create mode 100755 mindspore/lite/src/litert/cxx_api/model/model_group_impl.cc create mode 100644 mindspore/lite/src/litert/cxx_api/model/model_group_impl.h create mode 100644 mindspore/lite/src/litert/kernel/ascend/src/acl_mem_manager.cc create mode 100644 mindspore/lite/src/litert/kernel/ascend/src/acl_mem_manager.h create mode 100644 mindspore/lite/src/litert/model_manager.cc create mode 100644 mindspore/lite/src/litert/model_manager.h create mode 100644 mindspore/lite/tools/converter/adapter/utils.cc create mode 100644 mindspore/lite/tools/converter/adapter/utils.h diff --git a/include/api/model_group.h b/include/api/model_group.h new file mode 100755 index 00000000000..362ac5110fc --- /dev/null +++ b/include/api/model_group.h @@ -0,0 +1,68 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_INCLUDE_API_MODEL_GROUP_H +#define MINDSPORE_INCLUDE_API_MODEL_GROUP_H + +#include +#include +#include +#include +#include +#include "include/api/model.h" +#include "include/api/status.h" +#include "include/api/types.h" +#include "include/api/context.h" + +namespace mindspore { +class ModelGroupImpl; + +/// \brief The ModelGroup class is used to define a MindSpore model group, facilitating +/// multiple models to share workspace memory. +class MS_API ModelGroup { + public: + ModelGroup(); + ~ModelGroup() = default; + ModelGroup(const ModelGroup &) = delete; + ModelGroup &operator=(const ModelGroup &) = delete; + + /// \brief Add models that require shared workspace memory. + /// + /// \param[in] model_path_list Define the list of model path. + /// + /// \return Status. + Status AddModel(const std::vector &model_path_list); + + /// \brief Add models that require shared workspace memory. + /// + /// \param[in] model_buff_list Define the list of model buff. + /// + /// \return Status. + Status AddModel(const std::vector> &model_buff_list); + + /// \brief Calculate the max workspace of the added models. + /// + /// \param[in] model_type Define The type of model file. Options: ModelType::kMindIR_Lite, ModelType::kMindIR. Only + /// ModelType::kMindIR_Lite is valid for Lite. + /// \param[in] ms_context A context used to store options. + /// + /// \return Status. + Status CalMaxSizeOfWorkspace(ModelType model_type, const std::shared_ptr &ms_context); + + private: + std::shared_ptr impl_; +}; +} // namespace mindspore +#endif // MINDSPORE_INCLUDE_API_MODEL_GROUP_H diff --git a/mindspore/ccsrc/plugin/device/ascend/kernel/aicpu/aicpu_ops/CMakeLists.txt b/mindspore/ccsrc/plugin/device/ascend/kernel/aicpu/aicpu_ops/CMakeLists.txt index 22aef49619e..e9379bd721c 100644 --- a/mindspore/ccsrc/plugin/device/ascend/kernel/aicpu/aicpu_ops/CMakeLists.txt +++ b/mindspore/ccsrc/plugin/device/ascend/kernel/aicpu/aicpu_ops/CMakeLists.txt @@ -68,6 +68,7 @@ if(EXISTS ${CMAKE_C_COMPILER} AND EXISTS ${CMAKE_CXX_COMPILER}) -Wl,--no-whole-archive -Wl,-Bsymbolic -rdynamic + -s mindspore::protobuf_arm -pthread ) diff --git a/mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/CMakeLists.txt b/mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/CMakeLists.txt index 21cbd36d566..e9d74246128 100644 --- a/mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/CMakeLists.txt +++ b/mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/CMakeLists.txt @@ -255,7 +255,7 @@ if(ENABLE_CPU) endif() if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows") if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") - target_link_options(nnacl PRIVATE -Wl,-z,relro,-z,now,-z,noexecstack) + target_link_options(nnacl PRIVATE -Wl,-z,relro,-z,now,-z,noexecstack -fstack-protector-all) target_link_libraries(nnacl PRIVATE m) endif() if("${CMAKE_BUILD_TYPE}" STREQUAL "Release") diff --git a/mindspore/lite/python/CMakeLists.txt b/mindspore/lite/python/CMakeLists.txt index 08f475fdb9b..29cedd53e38 100644 --- a/mindspore/lite/python/CMakeLists.txt +++ b/mindspore/lite/python/CMakeLists.txt @@ -25,9 +25,11 @@ if(Python3_FOUND) if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE) add_dependencies(mindspore-extendrt_static fbs_inner_src) - target_link_libraries(_c_lite_wrapper PRIVATE -Wl,--whole-archive mindspore-extendrt -Wl,--no-whole-archive) + target_link_libraries(_c_lite_wrapper PRIVATE -Wl,--whole-archive mindspore-extendrt + -Wl,--no-whole-archive -Wl,-z,relro,-z,now,-z,noexecstack -fstack-protector-all -s) else() - target_link_libraries(_c_lite_wrapper PRIVATE -Wl,--whole-archive mindspore-lite_static -Wl,--no-whole-archive) + target_link_libraries(_c_lite_wrapper PRIVATE -Wl,--whole-archive mindspore-lite_static + -Wl,--no-whole-archive -Wl,-z,relro,-z,now,-z,noexecstack -fstack-protector-all -s) endif() if(MSLITE_ENABLE_CONVERTER OR MSLITE_ENABLE_RUNTIME_GLOG) target_link_libraries(_c_lite_wrapper PRIVATE -Wl,--no-as-needed mindspore::glog) diff --git a/mindspore/lite/src/CMakeLists.txt b/mindspore/lite/src/CMakeLists.txt index 1e2f1b22401..422dbdbd148 100644 --- a/mindspore/lite/src/CMakeLists.txt +++ b/mindspore/lite/src/CMakeLists.txt @@ -136,6 +136,7 @@ set(LITE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/litert/sub_graph_kernel.cc ${CMAKE_CURRENT_SOURCE_DIR}/litert/scheduler.cc ${CMAKE_CURRENT_SOURCE_DIR}/litert/lite_session.cc + ${CMAKE_CURRENT_SOURCE_DIR}/litert/model_manager.cc ${CMAKE_CURRENT_SOURCE_DIR}/errorcode.cc ${CMAKE_CURRENT_SOURCE_DIR}/litert/cpu_info.cc ${CMAKE_CURRENT_SOURCE_DIR}/litert/pack_weight_manager.cc diff --git a/mindspore/lite/src/litert/cxx_api/model/model_group.cc b/mindspore/lite/src/litert/cxx_api/model/model_group.cc new file mode 100755 index 00000000000..031e6dcc527 --- /dev/null +++ b/mindspore/lite/src/litert/cxx_api/model/model_group.cc @@ -0,0 +1,56 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "include/api/model_group.h" +#include +#include "include/api/types.h" +#include "include/api/context.h" +#include "include/api/dual_abi_helper.h" +#include "src/litert/cxx_api/model/model_group_impl.h" +#include "src/common/log_adapter.h" + +namespace mindspore { +ModelGroup::ModelGroup() { + impl_ = std::make_shared(); + if (impl_ == nullptr) { + MS_LOG(ERROR) << "New model group impl_ failed."; + } +} + +Status ModelGroup::AddModel(const std::vector &model_path_list) { + if (impl_ == nullptr) { + MS_LOG(ERROR) << "Model group implement is null."; + return kLiteUninitializedObj; + } + return impl_->AddModel(model_path_list); +} + +Status ModelGroup::AddModel(const std::vector> &model_buff_list) { + if (impl_ == nullptr) { + MS_LOG(ERROR) << "Model group implement is null."; + return kLiteUninitializedObj; + } + return impl_->AddModel(model_buff_list); +} + +Status ModelGroup::CalMaxSizeOfWorkspace(ModelType model_type, const std::shared_ptr &ms_context) { + if (impl_ == nullptr) { + MS_LOG(ERROR) << "Model group implement is null."; + return kLiteUninitializedObj; + } + return impl_->CalMaxSizeOfWorkspace(model_type, ms_context); +} +} // namespace mindspore diff --git a/mindspore/lite/src/litert/cxx_api/model/model_group_impl.cc b/mindspore/lite/src/litert/cxx_api/model/model_group_impl.cc new file mode 100755 index 00000000000..fc23bc3f2f3 --- /dev/null +++ b/mindspore/lite/src/litert/cxx_api/model/model_group_impl.cc @@ -0,0 +1,130 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/litert/cxx_api/model/model_group_impl.h" +#include +#include +#include +#include +#include +#include +#include "include/api/types.h" +#include "include/api/context.h" +#include "src/litert/cxx_api/converters.h" +#include "src/common/log_adapter.h" +#include "src/litert/lite_session.h" +#include "src/litert/model_manager.h" +#include "src/common/config_file.h" + +namespace mindspore { +using mindspore::lite::RET_OK; + +Status ModelGroupImpl::AddModel(const std::vector &model_path_list) { + if (model_path_list.empty()) { + MS_LOG(ERROR) << "Param model_path_list is empty."; + return kLiteParamInvalid; + } + for (auto &model_path : model_path_list) { + if (model_path.empty()) { + continue; + } + (void)model_path_list_.emplace_back(model_path); + } + + return kSuccess; +} + +Status ModelGroupImpl::AddModel(const std::vector> &model_buff_list) { + if (model_buff_list.empty()) { + MS_LOG(ERROR) << "Param model_buff_list is empty."; + return kLiteParamInvalid; + } + for (auto &model_buff : model_buff_list) { + if (model_buff.first == nullptr || model_buff.second == 0) { + continue; + } + (void)model_buff_list_.emplace_back(model_buff); + } + + return kSuccess; +} + +lite::LiteSession *ModelGroupImpl::CreateLiteSession(const std::shared_ptr &ms_context) { + auto session = new (std::nothrow) lite::LiteSession(); + if (session == nullptr) { + return nullptr; + } + + std::string sharing_workspace_section = "inner_common"; + std::string calc_workspace_key = "inner_calc_workspace_size"; + std::string calc_workspace_value = "true"; + std::map model_sharing{{calc_workspace_key, calc_workspace_value}}; + config_info_[sharing_workspace_section] = model_sharing; + session->SetConfigInfo(&config_info_); + session->SetPrepareSessionFlag(true); + auto ret = session->Init(ContextUtils::Convert(ms_context.get())); + if (ret != mindspore::lite::RET_OK) { + MS_LOG(ERROR) << "init session failed"; + delete session; + return nullptr; + } + return session; +} + +Status ModelGroupImpl::CalMaxSizeOfWorkspace(ModelType model_type, const std::shared_ptr &ms_context) { + for (auto &model_path : model_path_list_) { + auto *session = CreateLiteSession(ms_context); + if (session == nullptr) { + MS_LOG(ERROR) << "Calculate the maximum workspace size of the model " << model_path << " failed."; + ModelManager::GetInstance().ClearModel(); + return kLiteError; + } + auto ret = session->LoadModelAndCompileByPath(model_path, model_type); + if (ret != mindspore::lite::RET_OK) { + MS_LOG(ERROR) << "Calculate the maximum workspace size of the model " << model_path << " failed."; + delete session; + session = nullptr; + ModelManager::GetInstance().ClearModel(); + return kLiteError; + } + ModelManager::GetInstance().AddModel(model_path); + delete session; + session = nullptr; + } + + for (auto &model_buff : model_buff_list_) { + auto *session = CreateLiteSession(ms_context); + if (session == nullptr) { + MS_LOG(ERROR) << "Calculate the maximum workspace size of the model failed."; + ModelManager::GetInstance().ClearModel(); + return kLiteError; + } + auto ret = + session->LoadModelAndCompileByBuf(static_cast(model_buff.first), model_type, model_buff.second); + if (ret != mindspore::lite::RET_OK) { + MS_LOG(ERROR) << "Calculate the maximum workspace size of the model failed."; + delete session; + session = nullptr; + ModelManager::GetInstance().ClearModel(); + return kLiteError; + } + ModelManager::GetInstance().AddModel(model_buff); + delete session; + session = nullptr; + } + return kSuccess; +} +} // namespace mindspore diff --git a/mindspore/lite/src/litert/cxx_api/model/model_group_impl.h b/mindspore/lite/src/litert/cxx_api/model/model_group_impl.h new file mode 100644 index 00000000000..951a226d145 --- /dev/null +++ b/mindspore/lite/src/litert/cxx_api/model/model_group_impl.h @@ -0,0 +1,51 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_SRC_CXX_API_MODEL_MODEL_GROUP_IMPL_H_ +#define MINDSPORE_LITE_SRC_CXX_API_MODEL_MODEL_GROUP_IMPL_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "include/api/model_group.h" +#include "include/api/context.h" +#include "src/litert/lite_session.h" +#include "src/litert/inner_context.h" + +namespace mindspore { +class ModelGroupImpl { + public: + ModelGroupImpl() {} + ~ModelGroupImpl() = default; + + Status AddModel(const std::vector &model_path_list); + Status AddModel(const std::vector> &model_buff_list); + Status CalMaxSizeOfWorkspace(ModelType model_type, const std::shared_ptr &ms_context); + + private: + lite::LiteSession *CreateLiteSession(const std::shared_ptr &ms_context); + friend class ModelGroup; + std::vector model_path_list_; + std::vector> model_buff_list_; + std::map> config_info_; +}; +} // namespace mindspore + +#endif // MINDSPORE_LITE_SRC_CXX_API_MODEL_MODEL_GROUP_IMPL_H_ diff --git a/mindspore/lite/src/litert/cxx_api/model/model_impl.cc b/mindspore/lite/src/litert/cxx_api/model/model_impl.cc index d19b617aae8..b0c5050db09 100644 --- a/mindspore/lite/src/litert/cxx_api/model/model_impl.cc +++ b/mindspore/lite/src/litert/cxx_api/model/model_impl.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,7 @@ #include "src/litert/cxx_api/tensor_utils.h" #include "src/common/log_adapter.h" #include "src/litert/lite_session.h" +#include "src/litert/model_manager.h" #include "src/common/file_utils.h" #if defined(ENABLE_PRE_INFERENCE) && defined(__linux__) && !defined(Debug) #include "src/common/random_data_generator.h" @@ -42,10 +44,23 @@ namespace { const char *const kExecutionPlan = "execution_plan"; constexpr size_t kMaxSectionNum = 100; constexpr size_t kMaxConfigNumPerSection = 1000; +constexpr auto kSharingWorkspaceSection = "inner_common"; +constexpr auto kSharingWorkspaceKey = "inner_sharing_workspace"; +constexpr auto kSharingWorkspaceValue = "true"; } // namespace using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; +bool ModelImpl::IsEnableModelSharing(const std::string &model_path) { + const std::set &model_path_set = ModelManager::GetInstance().GetModelPath(); + return (model_path_set.find(model_path) != model_path_set.end()); +} + +bool ModelImpl::IsEnableModelSharing(const std::pair &model_buff) { + const std::set> &model_buff_set = ModelManager::GetInstance().GetModelBuff(); + return (model_buff_set.find(model_buff) != model_buff_set.end()); +} + CreateTrainSessionProto *CreateTrainSessionCallbackHolder(CreateTrainSessionProto *proto) { static CreateTrainSessionProto *proto_ = nullptr; if (proto != nullptr) { @@ -145,7 +160,16 @@ Status ModelImpl::Build(const void *model_data, size_t data_size, ModelType mode MS_LOG(ERROR) << "The platform exist don't support's instruction."; return kLiteNotSupport; } + context_ = ms_context; + bool model_sharing_flag = IsEnableModelSharing(std::make_pair(model_data, data_size)); + if (model_sharing_flag) { + auto ret = UpdateConfig(kSharingWorkspaceSection, std::make_pair(kSharingWorkspaceKey, kSharingWorkspaceValue)); + if (ret != kSuccess) { + MS_LOG(ERROR) << "UpdateConfig " << kSharingWorkspaceKey << " failed."; + return ret; + } + } auto session = std::shared_ptr(CreateLiteSession(ContextUtils::Convert(ms_context.get()))); if (session == nullptr) { MS_LOG(ERROR) << "Allocate session failed."; @@ -170,6 +194,14 @@ Status ModelImpl::Build(const std::string &model_path, ModelType model_type, return kLiteNotSupport; } + bool model_sharing_flag = IsEnableModelSharing(model_path); + if (model_sharing_flag) { + auto ret = UpdateConfig(kSharingWorkspaceSection, std::make_pair(kSharingWorkspaceKey, kSharingWorkspaceValue)); + if (ret != kSuccess) { + MS_LOG(ERROR) << "UpdateConfig " << kSharingWorkspaceKey << " failed."; + return ret; + } + } auto session = std::shared_ptr(CreateLiteSession(ContextUtils::Convert(ms_context.get()))); if (session == nullptr) { MS_LOG(ERROR) << "Allocate session failed."; diff --git a/mindspore/lite/src/litert/cxx_api/model/model_impl.h b/mindspore/lite/src/litert/cxx_api/model/model_impl.h index 6e7ca974520..3e1cb604b62 100644 --- a/mindspore/lite/src/litert/cxx_api/model/model_impl.h +++ b/mindspore/lite/src/litert/cxx_api/model/model_impl.h @@ -136,6 +136,8 @@ class ModelImpl { void SetContext(const std::shared_ptr &context) { context_ = context; } void SetConfig(const std::shared_ptr cfg) { cfg_ = cfg; } Status RunGraph(const MSKernelCallBack &before, const MSKernelCallBack &after); + bool IsEnableModelSharing(const std::string &model_path); + bool IsEnableModelSharing(const std::pair &model_buff); std::map execution_plan_; std::map> config_info_; }; diff --git a/mindspore/lite/src/litert/kernel/ascend/src/acl_mem_manager.cc b/mindspore/lite/src/litert/kernel/ascend/src/acl_mem_manager.cc new file mode 100644 index 00000000000..d687e9509b0 --- /dev/null +++ b/mindspore/lite/src/litert/kernel/ascend/src/acl_mem_manager.cc @@ -0,0 +1,86 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/litert/kernel/ascend/src/acl_mem_manager.h" +#include +#include +#include +#include +#include "acl/acl.h" +#include "src/common/log_adapter.h" + +namespace mindspore::kernel { +namespace acl { +void AclMemManager::UpdateWorkspace(size_t work_size, size_t weight_size) { + if (work_size > work_mem_info_.mem_size) { + work_mem_info_.mem_size = work_size; + MS_LOG(DEBUG) << "Update work_size = " << work_size << " successful."; + } + + if (weight_size > weight_mem_info_.mem_size) { + weight_mem_info_.mem_size = weight_size; + MS_LOG(DEBUG) << "Update weight_size = " << weight_size << " successful."; + } +} + +STATUS AclMemManager::GetModelWorkMem(AclModelMemInfo *acl_work_mem_info) { + std::unique_lock acl_mtx(acl_mem_alloc_mutex_); + if (work_mem_info_.mem_addr == nullptr) { + if (work_mem_info_.mem_size == 0) { + return lite::RET_ERROR; + } + auto acl_ret = aclrtMalloc(&work_mem_info_.mem_addr, work_mem_info_.mem_size, ACL_MEM_MALLOC_NORMAL_ONLY); + if (acl_ret != ACL_ERROR_NONE) { + MS_LOG(ERROR) << "Call aclrtMalloc failed, err_code = " << acl_ret; + return lite::RET_ERROR; + } + MS_LOG(DEBUG) << "Malloc max work size is " << work_mem_info_.mem_size; + } + *acl_work_mem_info = work_mem_info_; + return lite::RET_OK; +} + +STATUS AclMemManager::GetModelWeightMem(AclModelMemInfo *acl_weight_mem_info) { + std::unique_lock acl_mtx(acl_mem_alloc_mutex_); + if (weight_mem_info_.mem_addr == nullptr) { + if (weight_mem_info_.mem_size == 0) { + return lite::RET_ERROR; + } + auto acl_ret = aclrtMalloc(&weight_mem_info_.mem_addr, weight_mem_info_.mem_size, ACL_MEM_MALLOC_NORMAL_ONLY); + if (acl_ret != ACL_ERROR_NONE) { + MS_LOG(ERROR) << "Call aclrtMalloc failed, err_code = " << acl_ret; + return lite::RET_ERROR; + } + MS_LOG(DEBUG) << "Malloc max weight size is " << weight_mem_info_.mem_size; + } + *acl_weight_mem_info = weight_mem_info_; + return lite::RET_OK; +} + +AclMemManager::~AclMemManager() { + if (work_mem_info_.mem_addr != nullptr) { + (void)aclrtFree(work_mem_info_.mem_addr); + work_mem_info_.mem_addr = nullptr; + work_mem_info_.mem_size = 0; + } + if (weight_mem_info_.mem_addr != nullptr) { + (void)aclrtFree(weight_mem_info_.mem_addr); + weight_mem_info_.mem_addr = nullptr; + weight_mem_info_.mem_size = 0; + } +} +} // namespace acl +} // namespace mindspore::kernel diff --git a/mindspore/lite/src/litert/kernel/ascend/src/acl_mem_manager.h b/mindspore/lite/src/litert/kernel/ascend/src/acl_mem_manager.h new file mode 100644 index 00000000000..be747923775 --- /dev/null +++ b/mindspore/lite/src/litert/kernel/ascend/src/acl_mem_manager.h @@ -0,0 +1,63 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ASCEND_SRC_ACL_MEM_MANAGER_H_ +#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ASCEND_SRC_ACL_MEM_MANAGER_H_ + +#include +#include +#include +#include +#include +#include "include/errorcode.h" + +namespace mindspore::kernel { +namespace acl { +using mindspore::lite::STATUS; + +struct AclModelMemInfo { + void *mem_addr; + size_t mem_size; +}; + +class AclMemManager { + public: + AclMemManager() {} + ~AclMemManager(); + + AclMemManager(const AclMemManager &) = delete; + AclMemManager &operator=(const AclMemManager &) = delete; + + static AclMemManager &GetInstance() { + static AclMemManager instance; + return instance; + } + void UpdateWorkspace(size_t work_size, size_t weight_size); + STATUS GetModelWorkMem(AclModelMemInfo *acl_work_mem_info); + STATUS GetModelWeightMem(AclModelMemInfo *acl_weight_mem_info); + void Lock() { return acl_execute_mutex_.lock(); } + void Unlock() { return acl_execute_mutex_.unlock(); } + + private: + std::mutex acl_mem_alloc_mutex_; + std::mutex acl_execute_mutex_; + AclModelMemInfo work_mem_info_ = {nullptr, 0}; + AclModelMemInfo weight_mem_info_ = {nullptr, 0}; +}; +} // namespace acl +} // namespace mindspore::kernel + +#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ASCEND_SRC_ACL_MEM_MANAGER_H_ diff --git a/mindspore/lite/src/litert/kernel/ascend/src/custom_kernel.cc b/mindspore/lite/src/litert/kernel/ascend/src/custom_kernel.cc index 2a8df2d0e8a..195b82ba80f 100644 --- a/mindspore/lite/src/litert/kernel/ascend/src/custom_kernel.cc +++ b/mindspore/lite/src/litert/kernel/ascend/src/custom_kernel.cc @@ -16,6 +16,7 @@ #include "src/litert/kernel/ascend/src/custom_kernel.h" #include +#include #include "include/registry/register_kernel.h" #include "include/api/types.h" #include "include/api/data_type.h" @@ -33,12 +34,14 @@ constexpr auto kNHWCWidthIdx = 2; constexpr auto kNCHWHeightIdx = 2; constexpr auto kNCHWWidthIdx = 3; constexpr auto kImageSizeHwNum = 2; +constexpr auto kSharingWorkspaceSection = "inner_common"; } // namespace CustomAscendKernel::CustomAscendKernel(const std::vector &inputs, const std::vector &outputs, const schema::Primitive *primitive, const mindspore::Context *ctx) : Kernel(inputs, outputs, primitive, ctx), load_model_(false), + prepare_flag_(false), acl_options_({}), model_infer_(nullptr), InputDataIndex_(0) {} @@ -66,7 +69,7 @@ STATUS CustomAscendKernel::PrepareModelInfer() { MS_LOG(ERROR) << "Parse acl options failed."; return lite::RET_ERROR; } - model_infer_ = std::make_shared(om_data, acl_options_); + model_infer_ = std::make_shared(om_data, acl_options_, this->GetConfig(kSharingWorkspaceSection)); CHECK_NULL_RETURN(model_infer_); } int ret = model_infer_->Init(); @@ -79,6 +82,10 @@ STATUS CustomAscendKernel::PrepareModelInfer() { MS_LOG(ERROR) << "Load om data failed."; return lite::RET_ERROR; } + if (prepare_flag_) { + MS_LOG(INFO) << "Update workspace success."; + return lite::RET_OK; + } acl_options_.batch_size = model_infer_->GetDynamicBatch(); acl_options_.image_size = model_infer_->GetDynamicImage(); MS_LOG(INFO) << "Load om data success."; @@ -90,10 +97,18 @@ STATUS CustomAscendKernel::Prepare() { MS_LOG(INFO) << "Custom kernel has been prepared."; return lite::RET_OK; } + const std::string calc_workspace_size = "inner_calc_workspace_size"; + const std::map &config_comm = this->GetConfig(kSharingWorkspaceSection); + if (config_comm.find(calc_workspace_size) != config_comm.end()) { + prepare_flag_ = true; + } if (PrepareModelInfer() != lite::RET_OK) { MS_LOG(ERROR) << "Model infer prepare is not ok."; return lite::RET_ERROR; } + if (prepare_flag_) { + return lite::RET_OK; + } RecordInputDataIndex(); load_model_ = true; @@ -225,8 +240,8 @@ STATUS CustomAscendKernel::GetRealImageSize(std::vector *in STATUS CustomAscendKernel::Execute() { if (!load_model_) { - MS_LOG(WARNING) << "Custom kernel has not been prepared."; - return lite::RET_OK; + MS_LOG(ERROR) << "Custom kernel has not been prepared."; + return lite::RET_ERROR; } std::vector inputs(inputs_.begin(), inputs_.end() - 1); if (ProcDynamicInput(&inputs) != lite::RET_OK) { diff --git a/mindspore/lite/src/litert/kernel/ascend/src/custom_kernel.h b/mindspore/lite/src/litert/kernel/ascend/src/custom_kernel.h index b9fd528ca99..55e80876830 100644 --- a/mindspore/lite/src/litert/kernel/ascend/src/custom_kernel.h +++ b/mindspore/lite/src/litert/kernel/ascend/src/custom_kernel.h @@ -49,6 +49,7 @@ class CustomAscendKernel : public kernel::Kernel { STATUS GetRealImageSize(std::vector *inputs, int32_t *image_size, int32_t num); bool load_model_; + bool prepare_flag_; AclModelOptions acl_options_; std::shared_ptr model_infer_; size_t InputDataIndex_; diff --git a/mindspore/lite/src/litert/kernel/ascend/src/model_infer.cc b/mindspore/lite/src/litert/kernel/ascend/src/model_infer.cc index 1f2a940ac6e..bcada1897d1 100644 --- a/mindspore/lite/src/litert/kernel/ascend/src/model_infer.cc +++ b/mindspore/lite/src/litert/kernel/ascend/src/model_infer.cc @@ -17,10 +17,17 @@ #include "src/litert/kernel/ascend/src/model_infer.h" #include "common/log_adapter.h" #include "acl/acl.h" +#include "src/litert/kernel/ascend/src/acl_mem_manager.h" namespace mindspore::kernel { namespace acl { -ModelInfer::ModelInfer(const Buffer &om_data, const AclModelOptions &options) +namespace { +constexpr auto kModelSharingPrepareKey = "multi_model_sharing_mem_prepare"; +constexpr auto kModelSharingKey = "multi_model_sharing_mem"; +} // namespace + +ModelInfer::ModelInfer(const Buffer &om_data, const AclModelOptions &options, + const std::map &config_info) : init_flag_(false), load_flag_(false), device_type_("AscendCL"), @@ -28,6 +35,7 @@ ModelInfer::ModelInfer(const Buffer &om_data, const AclModelOptions &options) om_data_(om_data), options_(options), model_process_(options), + config_info_(config_info), acl_env_(nullptr) {} STATUS ModelInfer::Init() { @@ -108,6 +116,20 @@ STATUS ModelInfer::Finalize() { return lite::RET_OK; } +bool ModelInfer::IsEnableMultiModelSharingMemPrepare() { + if (config_info_.find(kModelSharingPrepareKey) != config_info_.end()) { + return true; + } + return false; +} + +bool ModelInfer::IsEnableMultiModelSharingMem() { + if (config_info_.find(kModelSharingKey) != config_info_.end()) { + return true; + } + return false; +} + STATUS ModelInfer::Load() { if (!load_flag_) { int ret = LoadAclModel(om_data_); @@ -131,15 +153,51 @@ STATUS ModelInfer::LoadAclModel(const Buffer &om_data) { MS_LOG(INFO) << "Start load acl model."; // acl load model uint32_t acl_model_id; - auto acl_ret = aclmdlLoadFromMem(om_data.Data(), om_data.DataSize(), &acl_model_id); - if (acl_ret != ACL_ERROR_NONE) { - MS_LOG(ERROR) << "Call aclmdlLoadFromMem failed, ret = " << acl_ret; - return lite::RET_ERROR; + size_t work_size = 0; + size_t weight_size = 0; + + if (IsEnableMultiModelSharingMemPrepare()) { + auto acl_ret = aclmdlQuerySizeFromMem(om_data.Data(), om_data.DataSize(), &work_size, &weight_size); + if (acl_ret != ACL_ERROR_NONE) { + MS_LOG(ERROR) << "Call aclmdlQuerySizeFromMem failed, ret = " << acl_ret; + return lite::RET_ERROR; + } + AclMemManager::GetInstance().UpdateWorkspace(work_size, weight_size); + return lite::RET_OK; + } else if (IsEnableMultiModelSharingMem()) { + AclModelMemInfo acl_work_mem_info; + AclModelMemInfo acl_weight_mem_info; + auto ret = AclMemManager::GetInstance().GetModelWorkMem(&acl_work_mem_info); + if (ret != lite::RET_OK) { + MS_LOG(ERROR) << "Get work mem failed."; + return ret; + } + ret = AclMemManager::GetInstance().GetModelWeightMem(&acl_weight_mem_info); + if (ret != lite::RET_OK) { + MS_LOG(ERROR) << "Get weight mem failed."; + return ret; + } + MS_LOG(DEBUG) << "Sharing work size = " << acl_work_mem_info.mem_size; + MS_LOG(DEBUG) << "Sharing weight size = " << acl_weight_mem_info.mem_size; + auto acl_ret = + aclmdlLoadFromMemWithMem(om_data.Data(), om_data.DataSize(), &acl_model_id, acl_work_mem_info.mem_addr, + acl_work_mem_info.mem_size, acl_weight_mem_info.mem_addr, acl_weight_mem_info.mem_size); + if (acl_ret != ACL_ERROR_NONE) { + MS_LOG(ERROR) << "Call aclmdlLoadFromMemWithMem failed, ret = " << acl_ret; + return lite::RET_ERROR; + } + model_process_.SetSharingWorkspaceFlag(true); + } else { + auto acl_ret = aclmdlLoadFromMem(om_data.Data(), om_data.DataSize(), &acl_model_id); + if (acl_ret != ACL_ERROR_NONE) { + MS_LOG(ERROR) << "Call aclmdlLoadFromMem failed, ret = " << acl_ret; + return lite::RET_ERROR; + } } // acl init model resource model_process_.set_model_id(acl_model_id); - int ret = model_process_.PreInitModelResource(); + auto ret = model_process_.PreInitModelResource(); if (ret != lite::RET_OK) { (void)aclmdlUnload(acl_model_id); MS_LOG(ERROR) << "Pre init model resource failed."; diff --git a/mindspore/lite/src/litert/kernel/ascend/src/model_infer.h b/mindspore/lite/src/litert/kernel/ascend/src/model_infer.h index eecb8f17615..47fd43a553b 100644 --- a/mindspore/lite/src/litert/kernel/ascend/src/model_infer.h +++ b/mindspore/lite/src/litert/kernel/ascend/src/model_infer.h @@ -18,6 +18,7 @@ #define MINDSPORE_LITE_SRC_EXTENDRT_KERNEL_ASCEND_SRC_MODEL_INFER_H_ #include +#include #include #include #include @@ -34,7 +35,8 @@ using mindspore::lite::STATUS; class ModelInfer { public: - ModelInfer(const Buffer &om_data, const AclModelOptions &options); + ModelInfer(const Buffer &om_data, const AclModelOptions &options, + const std::map &config_info); ~ModelInfer() = default; STATUS Init(); @@ -48,6 +50,8 @@ class ModelInfer { private: STATUS LoadAclModel(const Buffer &om_data); + bool IsEnableMultiModelSharingMemPrepare(); + bool IsEnableMultiModelSharingMem(); bool init_flag_; bool load_flag_; @@ -56,6 +60,7 @@ class ModelInfer { Buffer om_data_; AclModelOptions options_; ModelProcess model_process_; + std::map config_info_; std::shared_ptr acl_env_; }; } // namespace acl diff --git a/mindspore/lite/src/litert/kernel/ascend/src/model_process.cc b/mindspore/lite/src/litert/kernel/ascend/src/model_process.cc index a36fc177506..d64ce2a8aaf 100644 --- a/mindspore/lite/src/litert/kernel/ascend/src/model_process.cc +++ b/mindspore/lite/src/litert/kernel/ascend/src/model_process.cc @@ -22,6 +22,7 @@ #include "common/log_adapter.h" #include "src/common/utils.h" #include "src/common/log_util.h" +#include "src/litert/kernel/ascend/src/acl_mem_manager.h" namespace mindspore::kernel { namespace acl { @@ -600,7 +601,15 @@ STATUS ModelProcess::PredictFromHost(const std::vector &inp struct timeval start_time; struct timeval end_time; (void)gettimeofday(&start_time, nullptr); + if (is_sharing_workspace_) { + MS_LOG(DEBUG) << "Need to lock before aclmdlExecute."; + AclMemManager::GetInstance().Lock(); + } acl_ret = aclmdlExecute(model_id_, inputs_, outputs_); + if (is_sharing_workspace_) { + MS_LOG(DEBUG) << "Need to unlock after aclmdlExecute."; + AclMemManager::GetInstance().Unlock(); + } (void)gettimeofday(&end_time, nullptr); constexpr uint64_t kUSecondInSecond = 1000000; uint64_t cost = @@ -608,7 +617,15 @@ STATUS ModelProcess::PredictFromHost(const std::vector &inp (kUSecondInSecond * static_cast(start_time.tv_sec) + static_cast(start_time.tv_usec)); MS_LOG(INFO) << "Model execute in " << cost << " us"; } else { + if (is_sharing_workspace_) { + MS_LOG(DEBUG) << "Need to lock before aclmdlExecute."; + AclMemManager::GetInstance().Lock(); + } acl_ret = aclmdlExecute(model_id_, inputs_, outputs_); + if (is_sharing_workspace_) { + MS_LOG(DEBUG) << "Need to unlock after aclmdlExecute."; + AclMemManager::GetInstance().Unlock(); + } } DestroyInputsDataset(); diff --git a/mindspore/lite/src/litert/kernel/ascend/src/model_process.h b/mindspore/lite/src/litert/kernel/ascend/src/model_process.h index 9cb9ed6f6b4..8f434e2ef3f 100644 --- a/mindspore/lite/src/litert/kernel/ascend/src/model_process.h +++ b/mindspore/lite/src/litert/kernel/ascend/src/model_process.h @@ -46,6 +46,7 @@ class ModelProcess { explicit ModelProcess(const AclModelOptions &options) : options_(options), model_id_(0xffffffff), + is_sharing_workspace_(false), is_run_on_device_(false), model_desc_(nullptr), inputs_(nullptr), @@ -63,6 +64,7 @@ class ModelProcess { void set_model_id(uint32_t model_id) { model_id_ = model_id; } uint32_t model_id() const { return model_id_; } + void SetSharingWorkspaceFlag(bool is_sharing_workspace) { is_sharing_workspace_ = is_sharing_workspace; } std::set GetDynamicBatch(); std::set> GetDynamicImage(); @@ -92,6 +94,7 @@ class ModelProcess { AclModelOptions options_; uint32_t model_id_; + bool is_sharing_workspace_; // if run one device(AICPU), there is no need to alloc device memory and copy inputs to(/outputs from) device bool is_run_on_device_; aclmdlDesc *model_desc_; diff --git a/mindspore/lite/src/litert/lite_session.cc b/mindspore/lite/src/litert/lite_session.cc index 2c8a94f68e6..74626e0bd86 100644 --- a/mindspore/lite/src/litert/lite_session.cc +++ b/mindspore/lite/src/litert/lite_session.cc @@ -569,7 +569,7 @@ int LiteSession::CompileGraph(Model *model) { return ret; } - if (is_train_session_) { + if (is_train_session_ || is_prepare_session_) { is_running_.store(false); return RET_OK; } diff --git a/mindspore/lite/src/litert/lite_session.h b/mindspore/lite/src/litert/lite_session.h index 81ff6cc31f6..b6c40d8395e 100644 --- a/mindspore/lite/src/litert/lite_session.h +++ b/mindspore/lite/src/litert/lite_session.h @@ -80,6 +80,7 @@ class LiteSession { void SetConfigInfo(const std::map> *config_info) { config_info_ = config_info; } + void SetPrepareSessionFlag(bool is_prepare_session) { is_prepare_session_ = is_prepare_session; } const std::vector &GetTensors() const { return this->tensors_; } virtual int Train() { return mindspore::lite::RET_ERROR; } @@ -206,6 +207,7 @@ class LiteSession { Model *model_ = nullptr; std::atomic is_running_ = {false}; bool is_train_session_ = false; + bool is_prepare_session_ = false; friend class TransferSession; #if GPU_OPENCL opencl::OpenCLRuntimeInnerWrapper *opencl_runtime_wrapper_{nullptr}; diff --git a/mindspore/lite/src/litert/model_manager.cc b/mindspore/lite/src/litert/model_manager.cc new file mode 100644 index 00000000000..cb475778412 --- /dev/null +++ b/mindspore/lite/src/litert/model_manager.cc @@ -0,0 +1,39 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/litert/model_manager.h" +#include +#include +#include +#include + +namespace mindspore { +void ModelManager::AddModel(const std::string model_path) { (void)model_path_set_.insert(model_path); } + +void ModelManager::AddModel(const std::pair model_buff) { + (void)model_buff_set_.insert(model_buff); +} + +void ModelManager::ClearModel() { + model_path_set_.clear(); + model_buff_set_.clear(); +} + +ModelManager::~ModelManager() { + model_path_set_.clear(); + model_buff_set_.clear(); +} +} // namespace mindspore diff --git a/mindspore/lite/src/litert/model_manager.h b/mindspore/lite/src/litert/model_manager.h new file mode 100644 index 00000000000..4760f987738 --- /dev/null +++ b/mindspore/lite/src/litert/model_manager.h @@ -0,0 +1,53 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_SRC_MODEL_MANAGER_H_ +#define MINDSPORE_LITE_SRC_MODEL_MANAGER_H_ + +#include +#include +#include +#include +#include +#include "src/litert/lite_session.h" + +namespace mindspore { +class ModelManager { + public: + ModelManager() {} + ~ModelManager(); + + ModelManager(const ModelManager &) = delete; + ModelManager &operator=(const ModelManager &) = delete; + + static ModelManager &GetInstance() { + static ModelManager instance; + return instance; + } + + void AddModel(const std::string model_path); + void AddModel(const std::pair model_buff); + const std::set &GetModelPath() const { return model_path_set_; } + const std::set> &GetModelBuff() const { return model_buff_set_; } + void ClearModel(); + + private: + std::set model_path_set_; + std::set> model_buff_set_; +}; +} // namespace mindspore + +#endif // MINDSPORE_LITE_SRC_MODEL_MANAGER_H_ diff --git a/mindspore/lite/test/config_level0/cropped_size.cfg b/mindspore/lite/test/config_level0/cropped_size.cfg index befa5293668..7f9ed9c0d97 100644 --- a/mindspore/lite/test/config_level0/cropped_size.cfg +++ b/mindspore/lite/test/config_level0/cropped_size.cfg @@ -1,2 +1,2 @@ Note: This is the mindspore Lite inference framework size threshold. Offline review is required before modify this value!!! -1106192 +1116516 diff --git a/mindspore/lite/test/config_level1/cropped_size.cfg b/mindspore/lite/test/config_level1/cropped_size.cfg index 8e9efff7cbc..a77e02ec9e6 100644 --- a/mindspore/lite/test/config_level1/cropped_size.cfg +++ b/mindspore/lite/test/config_level1/cropped_size.cfg @@ -1,2 +1,2 @@ Note: This is the mindspore Lite inference framework size threshold. Modifying this threshold requires meeting review. -1106192 +1116516 diff --git a/mindspore/lite/tools/converter/CMakeLists.txt b/mindspore/lite/tools/converter/CMakeLists.txt index 2ffa2d0cce6..36f99933ca2 100644 --- a/mindspore/lite/tools/converter/CMakeLists.txt +++ b/mindspore/lite/tools/converter/CMakeLists.txt @@ -157,6 +157,7 @@ set(LITE_SRC ${API_SRC} ${SRC_DIR}/litert/lite_session.cc ${SRC_DIR}/litert/executor.cc ${SRC_DIR}/litert/lite_model.cc + ${SRC_DIR}/litert/model_manager.cc ${SRC_DIR}/errorcode.cc ${SRC_DIR}/litert/weight_decoder.cc ${SRC_DIR}/litert/pack_weight_manager.cc diff --git a/mindspore/lite/tools/converter/adapter/dpico/CMakeLists.txt b/mindspore/lite/tools/converter/adapter/dpico/CMakeLists.txt index 7cd08c62c95..5cb20941ae9 100644 --- a/mindspore/lite/tools/converter/adapter/dpico/CMakeLists.txt +++ b/mindspore/lite/tools/converter/adapter/dpico/CMakeLists.txt @@ -27,6 +27,7 @@ file(GLOB_RECURSE DPICO_SRC ${CMAKE_CURRENT_SOURCE_DIR}/checker/*.cc ${CMAKE_CURRENT_SOURCE_DIR}/infer/*.cc ${CMAKE_CURRENT_SOURCE_DIR}/mapper/*.cc + ${TOOLS_DIR}/converter/adapter/utils.cc ) include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}) @@ -34,6 +35,7 @@ include_directories(BEFORE ${OPENCV_PATH}include/opencv4) include_directories(BEFORE ${PROTOBUF_PATH}/include) include_directories(${PICO_MAPPER_PATH}/include) include_directories(${PROTO_OUTPUT_PATH}) +include_directories(${TOOLS_DIR}/converter) link_directories(${pico_mapper_SOURCE_DIR}/lib) set_property(SOURCE ${DPICO_SRC} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_LITE) diff --git a/mindspore/lite/tools/converter/adapter/dpico/src/calib_data_generator.cc b/mindspore/lite/tools/converter/adapter/dpico/src/calib_data_generator.cc index b1f00b64df2..5c2754b92ed 100644 --- a/mindspore/lite/tools/converter/adapter/dpico/src/calib_data_generator.cc +++ b/mindspore/lite/tools/converter/adapter/dpico/src/calib_data_generator.cc @@ -20,12 +20,15 @@ #include #include #include +#include #include "ops/tuple_get_item.h" #include "common/anf_util.h" #include "common/string_util.h" #include "common/file_util.h" #include "src/mapper_config_parser.h" #include "src/data_preprocessor.h" +#include "adapter/utils.h" + using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; namespace mindspore { @@ -33,44 +36,17 @@ namespace dpico { namespace { constexpr size_t kMaxSize = 1024; } // namespace -int CalibDataGenerator::GenerateDumpConfig(const std::string &dump_cfg_path, - const std::vector &dump_op_infos) { - auto output_path = MapperConfigParser::GetInstance()->GetOutputPath(); - auto kernels_str = std::accumulate(dump_op_infos.begin(), dump_op_infos.end(), std::string("["), - [](const std::string &cur_str, const DumpOpInfo &dump_op_info) { - return cur_str + "\"" + dump_op_info.dump_op_name + "\","; - }); - kernels_str.replace(kernels_str.size() - 1, 1, "]"); - std::ofstream dump_ofs; - dump_ofs.open(dump_cfg_path, std::ios::out); - if (!dump_ofs.is_open()) { - MS_LOG(ERROR) << "file open failed." << dump_cfg_path; - return RET_ERROR; - } - dump_ofs << "{" << std::endl; - dump_ofs << "\"common_dump_settings\": {" << std::endl; - dump_ofs << "\"dump_mode\": 1," << std::endl; - dump_ofs << R"("path": ")" << output_path << "\"," << std::endl; - dump_ofs << R"("net_name": "dump_data",)" << std::endl; - dump_ofs << "\"input_output\": " << dump_level_ << "," << std::endl; - dump_ofs << "\"kernels\":" << kernels_str << std::endl; - dump_ofs << "}" << std::endl; // end of common_dump_setting - dump_ofs << "}" << std::endl; // end of dump oss - dump_ofs.close(); - return RET_OK; -} - -std::string CalibDataGenerator::GetInputShapesStr(const api::AnfNodePtrList &graph_inputs) { - std::string input_shapes_str; +std::vector> CalibDataGenerator::GetInputShapes(const api::AnfNodePtrList &graph_inputs) { + std::vector> input_shapes; for (const auto &input : graph_inputs) { ShapeVector shape_vector; if (GetShapeVectorFromParameter(input, &shape_vector) != RET_OK) { MS_LOG(ERROR) << "get graph input shape failed. " << input->fullname_with_scope(); - return ""; + return std::vector>(); } if (shape_vector.empty()) { MS_LOG(ERROR) << input->fullname_with_scope() << " input shape is empty."; - return ""; + return std::vector>(); } if (shape_vector.size() == kDims4) { // transform nchw to nhwc shape_vector = std::vector{shape_vector[kNCHW_N], shape_vector[kNCHW_H], shape_vector[kNCHW_W], @@ -84,20 +60,13 @@ std::string CalibDataGenerator::GetInputShapesStr(const api::AnfNodePtrList &gra } else { MS_LOG(ERROR) << input->fullname_with_scope() << "'s input shape[" << i << "] is " << dim << ", which is unsupported by dpico."; - return ""; + return std::vector>(); } } - input_shapes_str += std::to_string(dim); - if (i != shape_vector.size() - 1) { - input_shapes_str += ','; - } } - input_shapes_str += ':'; + input_shapes.emplace_back(shape_vector); } - if (!input_shapes_str.empty() && input_shapes_str.at(input_shapes_str.size() - 1) == ':') { - input_shapes_str.pop_back(); - } - return input_shapes_str; + return input_shapes; } std::vector CalibDataGenerator::GetInDataFileList(const api::AnfNodePtrList &graph_inputs) { @@ -125,57 +94,20 @@ std::vector CalibDataGenerator::GetInDataFileList(const api::AnfNod return {}; } } - in_data_files_list.emplace_back(in_data_files); + (void)in_data_files_list.emplace_back(in_data_files); } return in_data_files_list; } -int CalibDataGenerator::DumpKernelsData(const std::string &dump_cfg_path, +int CalibDataGenerator::DumpKernelsData(const std::string ¤t_path, const std::vector &in_data_file_list, - const std::string &input_shapes_str) { - if (setenv("MINDSPORE_DUMP_CONFIG", dump_cfg_path.c_str(), 1)) { - MS_LOG(ERROR) << "set env for dump config failed."; - return RET_ERROR; - } - std::string benchmark_path = "../../benchmark/benchmark"; - bool use_default_benchmark = true; - auto config_info = converter::ConverterContext::GetConfigInfo("dpico"); - if (!config_info.empty()) { - if (config_info.find("benchmark_path") != config_info.end()) { - benchmark_path = config_info.at("benchmark_path"); - use_default_benchmark = false; - } - } - if (use_default_benchmark) { - MS_LOG(WARNING) << R"(there is no "benchmark_path" in the converter config file, - will use the default value: "../../benchmark/benchmark")"; - } - if (AccessFile(benchmark_path, F_OK) != 0) { - MS_LOG(ERROR) << "File not exist: " << benchmark_path; - return RET_ERROR; - } - std::string current_path; - auto benchmark_file = RealPath(benchmark_path.c_str()); - if (benchmark_file.empty()) { - MS_LOG(ERROR) << "Get realpath failed, benchmark path is " << benchmark_path; - return RET_ERROR; - } - std::string model_file = current_path + "model.ms"; - char buf[kMaxSize] = {0}; - for (const auto &in_data_file : in_data_file_list) { - std::string result; - std::string cmd = benchmark_file + " --modelFile=" + model_file + " --inDataFile=" + in_data_file + - " --inputShapes=" + input_shapes_str + " --loopCount=1 --warmUpLoopCount=0"; - MS_LOG(INFO) << "begin to run benchmark: " << cmd; - auto fp = popen(cmd.c_str(), "r"); - if (fp != nullptr) { - while (fgets(buf, kMaxSize, fp) != nullptr) { - result += buf; - } - } - pclose(fp); - if (result.find("Success") == std::string::npos) { - MS_LOG(ERROR) << "Run benchmark failed. " << result; + const std::vector kernel_names, + const std::vector> &input_shapes) { + std::string model_file = "model.ms"; + for (auto in_data_file : in_data_file_list) { + auto ret = lite::converter::InnerPredict(model_file, in_data_file, kernel_names, current_path, input_shapes); + if (ret != RET_OK) { + MS_LOG(ERROR) << "Execute InnerPredict failed."; return RET_ERROR; } } @@ -237,7 +169,7 @@ int CalibDataGenerator::TransBinsToTxt(const std::vector &dump_op_in MS_LOG(ERROR) << "file open failed. " << dump_op_txt_path; return RET_ERROR; } - ofs.precision(kNumPrecision); + (void)ofs.precision(kNumPrecision); bool is_input = dump_op_info.input_index >= 0; auto del_special_character = ReplaceSpecifiedChar(dump_op_info.dump_op_name, '/', '.'); auto pattern = is_input @@ -310,7 +242,7 @@ int CalibDataGenerator::Run(const api::AnfNodePtrList &graph_inputs, const api:: if (has_visited.find(node) != has_visited.end()) { continue; } - has_visited.insert(node); + (void)has_visited.insert(node); DumpOpInfo dump_op_info = {node->fullname_with_scope(), node->fullname_with_scope(), -1, -1}; if (control_flow_inputs_.find(node) != control_flow_inputs_.end()) { dump_op_info.dump_op_name = control_flow_inputs_[node].first->fullname_with_scope(); @@ -328,7 +260,7 @@ int CalibDataGenerator::Run(const api::AnfNodePtrList &graph_inputs, const api:: } } if (image_lists.find(dump_op_info.dump_op_name) == image_lists.end()) { - dump_op_infos.emplace_back(dump_op_info); + (void)dump_op_infos.emplace_back(dump_op_info); } } if (dump_op_infos.empty()) { @@ -336,25 +268,23 @@ int CalibDataGenerator::Run(const api::AnfNodePtrList &graph_inputs, const api:: return RET_ERROR; } - std::string dump_cfg_path = MapperConfigParser::GetInstance()->GetOutputPath() + "generated_dump.cfg"; - if (GenerateDumpConfig(dump_cfg_path, dump_op_infos) != RET_OK) { - MS_LOG(ERROR) << "generate dump config failed."; - return RET_ERROR; - } - auto in_data_file_list = GetInDataFileList(graph_inputs); if (in_data_file_list.empty()) { MS_LOG(ERROR) << "get in data file for benchmark failed."; return RET_ERROR; } - auto input_shapes_str = GetInputShapesStr(graph_inputs); - if (input_shapes_str.empty()) { + auto input_shapes = GetInputShapes(graph_inputs); + if (input_shapes.empty()) { MS_LOG(ERROR) << "get input shapes for benchmark failed."; return RET_ERROR; } - if (DumpKernelsData(dump_cfg_path, in_data_file_list, input_shapes_str) != RET_OK) { + std::vector kernel_names; + (void)std::transform(dump_op_infos.begin(), dump_op_infos.end(), std::back_inserter(kernel_names), + [](DumpOpInfo const &op_info) { return op_info.dump_op_name; }); + auto output_path = MapperConfigParser::GetInstance()->GetOutputPath(); + if (DumpKernelsData(output_path, in_data_file_list, kernel_names, input_shapes) != RET_OK) { MS_LOG(ERROR) << "dump kernels data failed."; return RET_ERROR; } @@ -363,6 +293,7 @@ int CalibDataGenerator::Run(const api::AnfNodePtrList &graph_inputs, const api:: MS_LOG(ERROR) << "transform dumped files to txt failed."; return RET_ERROR; } + return RET_OK; } } // namespace dpico diff --git a/mindspore/lite/tools/converter/adapter/dpico/src/calib_data_generator.h b/mindspore/lite/tools/converter/adapter/dpico/src/calib_data_generator.h index 0dcaed8a388..988ff912a55 100644 --- a/mindspore/lite/tools/converter/adapter/dpico/src/calib_data_generator.h +++ b/mindspore/lite/tools/converter/adapter/dpico/src/calib_data_generator.h @@ -57,10 +57,11 @@ class CalibDataGenerator { private: int GenerateDumpConfig(const std::string &dump_cfg_path, const std::vector &dump_infos); - std::string GetInputShapesStr(const api::AnfNodePtrList &graph_inputs); + std::vector> GetInputShapes(const api::AnfNodePtrList &graph_inputs); std::vector GetInDataFileList(const api::AnfNodePtrList &graph_inputs); int DumpKernelsData(const std::string &dump_cfg_path, const std::vector &in_data_file_list, - const std::string &input_shapes_str); + const std::vector kernel_names, + const std::vector> &input_shapes); STATUS ParseAttrFromFilename(struct OpAttr *op_attr, const std::string &file_name, bool is_input); int TransBinsToTxt(const std::vector &dump_infos); @@ -75,7 +76,7 @@ class CalibDataGenerator { size_t shape_size = 1; for (size_t i = 0; i < op_attr.shape.size(); i++) { if (op_attr.shape.at(i) < 0) { - MS_LOG(ERROR) << "dim val should be greater than 0"; + MS_LOG(ERROR) << "dim val should be equal or greater than 0"; return RET_ERROR; } if (SIZE_MUL_OVERFLOW(shape_size, static_cast(op_attr.shape.at(i)))) { @@ -84,7 +85,7 @@ class CalibDataGenerator { } shape_size *= static_cast(op_attr.shape.at(i)); } - ifs.seekg(0, std::ios::end); + (void)ifs.seekg(0, std::ios::end); size_t file_size = static_cast(ifs.tellg()); if (file_size != shape_size * sizeof(T)) { MS_LOG(ERROR) << "file size " << file_size << " is not equal to shape size " << shape_size; @@ -95,8 +96,8 @@ class CalibDataGenerator { MS_LOG(ERROR) << "new T failed."; return RET_ERROR; } - ifs.seekg(0, std::ios::beg); - ifs.read(reinterpret_cast(raw_datas.get()), shape_size * sizeof(T)); + (void)ifs.seekg(0, std::ios::beg); + (void)ifs.read(reinterpret_cast(raw_datas.get()), shape_size * sizeof(T)); ifs.close(); if (op_attr.format == "NHWC" && op_attr.shape.size() == kDims4) { auto dst_datas = std::make_unique(shape_size); diff --git a/mindspore/lite/tools/converter/adapter/utils.cc b/mindspore/lite/tools/converter/adapter/utils.cc new file mode 100644 index 00000000000..48e86a93b80 --- /dev/null +++ b/mindspore/lite/tools/converter/adapter/utils.cc @@ -0,0 +1,280 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "adapter/utils.h" +#include +#ifndef _WIN32 +#include +#include +#else +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "include/api/model.h" +#include "include/api/context.h" +#include "src/common/file_utils.h" + +#ifdef _WIN32 +#define ACCESS _access +#define RMDIR _rmdir +#else +#define ACCESS access +#define RMDIR rmdir +#endif +namespace mindspore { +namespace lite { +namespace converter { +constexpr int kThreadNum = 2; +constexpr bool kEnableParallel = false; +constexpr int kThreadAffinity = 1; +constexpr int kInterOpParallelNum = 1; +constexpr bool kEnableFP16 = false; +const std::unordered_map kTypeIdMap{ + {kNumberTypeFloat16, "Float16"}, {kNumberTypeFloat, "Float32"}, {kNumberTypeFloat32, "Float32"}, + {kNumberTypeInt8, "Int8"}, {kNumberTypeInt16, "Int16"}, {kNumberTypeInt, "Int32"}, + {kNumberTypeInt32, "Int32"}, {kNumberTypeUInt8, "UInt8"}, {kNumberTypeUInt16, "UInt16"}, + {kNumberTypeUInt, "UInt32"}, {kNumberTypeUInt32, "UInt32"}, {kObjectTypeString, "String"}, + {kNumberTypeBool, "Bool"}, {kObjectTypeTensorType, "Tensor"}}; + +const std::unordered_map kTensorFormatMap{ + {mindspore::NCHW, "NCHW"}, {mindspore::NHWC, "NHWC"}, {mindspore::NHWC4, "NHWC4"}, {mindspore::HWKC, "HWKC"}, + {mindspore::HWCK, "HWCK"}, {mindspore::KCHW, "KCHW"}, {mindspore::CKHW, "CKHW"}, {mindspore::KHWC, "KHWC"}, + {mindspore::CHWK, "CHWK"}, {mindspore::HW, "HW"}, {mindspore::HW4, "HW4"}, {mindspore::NC, "NC"}, + {mindspore::NC4, "NC4"}, {mindspore::NC4HW4, "NC4HW4"}, {mindspore::NCDHW, "NCDHW"}}; + +bool RemoveDir(const std::string &path) { + std::string str_path = path; +#ifdef _WIN32 + struct _finddata_t fb; + if (str_path.at(str_path.length() - 1) != '\\' || str_path.at(str_path.length() - 1) != '/') str_path.append("\\"); + std::string find_path = str_path + "*"; + intptr_t handle = _findfirst(find_path.c_str(), &fb); + if (handle != -1L) { + std::string tmp_path; + do { + if (strcmp(fb.name, "..") != 0 && strcmp(fb.name, ".") != 0) { + tmp_path.clear(); + tmp_path = str_path + std::string(fb.name); + if (fb.attrib == _A_SUBDIR) { + (void)RemoveDir(tmp_path.c_str()); + } else { + remove(tmp_path.c_str()); + } + } + } while (_findnext(handle, &fb) == 0); + _findclose(handle); + } + return RMDIR(str_path.c_str()) == 0 ? true : false; +#else + if (str_path.at(str_path.length() - 1) != '\\' || str_path.at(str_path.length() - 1) != '/') { + (void)str_path.append("/"); + } + DIR *d = opendir(str_path.c_str()); + if (d != nullptr) { + struct dirent *dt = readdir(d); + while (dt) { + if (strcmp(dt->d_name, "..") != 0 && strcmp(dt->d_name, ".") != 0) { + struct stat st {}; + std::string file_name = str_path + std::string(dt->d_name); + (void)stat(file_name.c_str(), &st); + if (S_ISDIR(st.st_mode)) { + (void)RemoveDir(file_name); + } else { + (void)remove(file_name.c_str()); + } + } + dt = readdir(d); + } + (void)closedir(d); + } + return rmdir(str_path.c_str()) == 0; +#endif +} + +int ReadInputFile(const std::vector &in_data_files, std::vector *input_tensors) { + for (size_t i = 0; i < in_data_files.size(); i++) { + auto &cur_tensor = (*input_tensors).at(i); + MS_ASSERT(cur_tensor != nullptr); + size_t size; + char *bin_buf = lite::ReadFile(in_data_files[i].c_str(), &size); + if (bin_buf == nullptr) { + MS_LOG(ERROR) << "ReadFile return nullptr"; + return RET_ERROR; + } + if (static_cast(cur_tensor.DataType()) == kObjectTypeString) { + std::string str(bin_buf, size); + MSTensor *input = MSTensor::StringsToTensor(cur_tensor.Name(), {str}); + if (input == nullptr) { + MS_LOG(ERROR) << "StringsToTensor failed"; + delete[] bin_buf; + return RET_ERROR; + } + cur_tensor = *input; + } else { + auto tensor_data_size = cur_tensor.DataSize(); + if (size != tensor_data_size) { + MS_LOG(ERROR) << "Input binary file size error, required: " << tensor_data_size << ", in fact: " << size; + delete[] bin_buf; + return RET_ERROR; + } + auto input_data = cur_tensor.MutableData(); + if (input_data == nullptr) { + MS_LOG(ERROR) << "input_data is nullptr."; + delete[] bin_buf; + return RET_ERROR; + } + auto ret = memcpy_s(input_data, tensor_data_size, bin_buf, size); + if (ret != EOK) { + MS_LOG(ERROR) << "Execute memcpy_s failed."; + delete[] bin_buf; + return RET_ERROR; + } + } + delete[] bin_buf; + } + + return RET_OK; +} +std::string GenerateOutputFileName(const mindspore::MSTensor *tensor, const std::string &op_name, + const std::string &file_type, const size_t &idx) { + std::string file_name = op_name; + auto pos = file_name.find_first_of('/'); + while (pos != std::string::npos) { + file_name.replace(pos, 1, "."); + pos = file_name.find_first_of('/'); + } + file_name += "_" + file_type + "_" + std::to_string(idx) + "_shape_"; + for (const auto &dim : tensor->Shape()) { + file_name += std::to_string(dim) + "_"; + } + if (kTypeIdMap.find(static_cast(tensor->DataType())) != kTypeIdMap.end()) { + file_name += kTypeIdMap.at(static_cast(tensor->DataType())); + } + auto tensor_format = tensor->format(); + if (kTensorFormatMap.find(tensor_format) != kTensorFormatMap.end()) { + file_name += "_" + kTensorFormatMap.at(tensor_format) + ".bin"; + } + + file_name += +".bin"; + return file_name; +} + +int InnerPredict(const std::string &model_name, const std::string &in_data_file, + const std::vector &output_names, const std::string &dump_directory, + const std::vector> &input_shapes) { + std::string dump_data_path = dump_directory + "dump_data/"; + if (ACCESS(dump_data_path.c_str(), 0) == 0 && !RemoveDir(dump_data_path)) { + MS_LOG(ERROR) << "The dump data path is exist and rmdir failed: " + << " " << dump_data_path; + return RET_ERROR; + } + if (lite::CreateOutputDir(&dump_data_path) != RET_OK) { + MS_LOG(ERROR) << "create data output directory failed."; + return RET_ERROR; + } + + MSKernelCallBack ms_after_call_back = [output_names, dump_data_path]( + const std::vector &after_inputs, + const std::vector &after_outputs, + const MSCallBackParam &call_param) { + if (std::find(output_names.begin(), output_names.end(), call_param.node_name) != output_names.end()) { + for (size_t i = 0; i < after_outputs.size(); i++) { + auto ms_tensor = after_outputs.at(i); + auto file_name = GenerateOutputFileName(&ms_tensor, call_param.node_name, "output", i); + auto abs_file_path = dump_data_path + "/" + file_name; + if (lite::WriteToBin(abs_file_path, ms_tensor.MutableData(), ms_tensor.DataSize()) != RET_OK) { + MS_LOG(ERROR) << "write tensor data to file failed."; + return false; + } + } + } + return true; + }; + + mindspore::Model ms_model; + mindspore::ModelType model_type = mindspore::ModelType::kMindIR_Lite; + auto context = std::make_shared(); + if (context == nullptr) { + MS_LOG(ERROR) << "New context failed while running " << model_name.c_str(); + return RET_ERROR; + } + context->SetThreadNum(kThreadNum); + context->SetEnableParallel(kEnableParallel); + context->SetThreadAffinity(kThreadAffinity); + context->SetInterOpParallelNum(kInterOpParallelNum); + auto &device_list = context->MutableDeviceInfo(); + + std::shared_ptr device_info = std::make_shared(); + device_info->SetEnableFP16(kEnableFP16); + device_list.push_back(device_info); + auto ret = ms_model.Build(model_name, model_type, context); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Execute model " << model_name.c_str() << " build failed."; + return RET_ERROR; + } + + if (!input_shapes.empty()) { + if (input_shapes.size() != ms_model.GetInputs().size()) { + MS_LOG(ERROR) << "Size of inputs_shape_list should be equal to size of inputs"; + return RET_ERROR; + } + + ret = ms_model.Resize(ms_model.GetInputs(), input_shapes); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Input tensor resize failed."; + return RET_ERROR; + } + } + + auto inputs = ms_model.GetInputs(); + if (inputs.empty()) { + MS_LOG(ERROR) << "Inputs is empty."; + return RET_ERROR; + } + std::regex re{"[\\s,]+"}; + std::vector in_data_list = std::vector{ + std::sregex_token_iterator(in_data_file.begin(), in_data_file.end(), re, -1), std::sregex_token_iterator()}; + if (in_data_list.size() != inputs.size()) { + MS_LOG(ERROR) << "Size of inputs should be equal to size of input inDataPath"; + return RET_ERROR; + } + auto status = ReadInputFile(in_data_list, &inputs); + if (status != RET_OK) { + MS_LOG(ERROR) << "ReadInputFile error, " << status; + return status; + } + + std::vector outputs; + ret = ms_model.Predict(inputs, &outputs, nullptr, ms_after_call_back); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Predict error "; + return RET_ERROR; + } + + return RET_OK; +} +} // namespace converter +} // namespace lite +} // namespace mindspore diff --git a/mindspore/lite/tools/converter/adapter/utils.h b/mindspore/lite/tools/converter/adapter/utils.h new file mode 100644 index 00000000000..de7a9922d1b --- /dev/null +++ b/mindspore/lite/tools/converter/adapter/utils.h @@ -0,0 +1,42 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_TOOLS_CONVERTER_ADAPTER_UTILS_H +#define MINDSPORE_LITE_TOOLS_CONVERTER_ADAPTER_UTILS_H +#include +#include +#include +#include +#include +#include "securec/include/securec.h" +#include "mindapi/base/base.h" +#include "include/errorcode.h" + +using mindspore::lite::RET_ERROR; +using mindspore::lite::RET_OK; +using mindspore::lite::STATUS; + +namespace mindspore { +namespace lite { +namespace converter { +bool RemoveDir(const std::string &path); +int InnerPredict(const std::string &model_name, const std::string &in_data_file, + const std::vector &output_names, const std::string &dump_directory, + const std::vector> &input_shapes = std::vector>()); +} // namespace converter +} // namespace lite +} // namespace mindspore +#endif // MINDSPORE_LITE_TOOLS_CONVERTER_NNIE_NNIE_SEGDATA_GENERATE_H