Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

multi-models shared workspace memory and converter of dpico decoupling 

with benchmark
This commit is contained in:
chengxianbin 2022-11-09 11:09:50 +08:00
parent ed6ad60271
commit 48e9092d21
30 changed files with 1063 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
include/api/model_group.h Executable file
View File

@ -0,0 +1,68 @@
/**
* Copyright 2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_INCLUDE_API_MODEL_GROUP_H
#define MINDSPORE_INCLUDE_API_MODEL_GROUP_H
#include <string>
#include <vector>
#include <map>
#include <memory>
#include <utility>
#include "include/api/model.h"
#include "include/api/status.h"
#include "include/api/types.h"
#include "include/api/context.h"
namespace mindspore {
class ModelGroupImpl;
/// \brief The ModelGroup class is used to define a MindSpore model group, facilitating
/// multiple models to share workspace memory.
class MS_API ModelGroup {
public:
ModelGroup();
~ModelGroup() = default;
ModelGroup(const ModelGroup &) = delete;
ModelGroup &operator=(const ModelGroup &) = delete;
/// \brief Add models that require shared workspace memory.
///
/// \param[in] model_path_list Define the list of model path.
///
/// \return Status.
Status AddModel(const std::vector<std::string> &model_path_list);
/// \brief Add models that require shared workspace memory.
///
/// \param[in] model_buff_list Define the list of model buff.
///
/// \return Status.
Status AddModel(const std::vector<std::pair<const void *, size_t>> &model_buff_list);
/// \brief Calculate the max workspace of the added models.
///
/// \param[in] model_type Define The type of model file. Options: ModelType::kMindIR_Lite, ModelType::kMindIR. Only
/// ModelType::kMindIR_Lite is valid for Lite.
/// \param[in] ms_context A context used to store options.
///
/// \return Status.
Status CalMaxSizeOfWorkspace(ModelType model_type, const std::shared_ptr<Context> &ms_context);
private:
std::shared_ptr<ModelGroupImpl> impl_;
};
} // namespace mindspore
#endif // MINDSPORE_INCLUDE_API_MODEL_GROUP_H

1
mindspore/ccsrc/plugin/device/ascend/kernel/aicpu/aicpu_ops/CMakeLists.txt
View File

@ -68,6 +68,7 @@ if(EXISTS ${CMAKE_C_COMPILER} AND EXISTS ${CMAKE_CXX_COMPILER})
-Wl,--no-whole-archive
-Wl,-Bsymbolic
-rdynamic
-s
mindspore::protobuf_arm
-pthread
)

2
mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/CMakeLists.txt
View File

@ -255,7 +255,7 @@ if(ENABLE_CPU)
endif()
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_options(nnacl PRIVATE -Wl,-z,relro,-z,now,-z,noexecstack)
target_link_options(nnacl PRIVATE -Wl,-z,relro,-z,now,-z,noexecstack -fstack-protector-all)
target_link_libraries(nnacl PRIVATE m)
endif()
if("${CMAKE_BUILD_TYPE}" STREQUAL "Release")

6
mindspore/lite/python/CMakeLists.txt
View File

@ -25,9 +25,11 @@ if(Python3_FOUND)
if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE)
add_dependencies(mindspore-extendrt_static fbs_inner_src)
target_link_libraries(_c_lite_wrapper PRIVATE -Wl,--whole-archive mindspore-extendrt -Wl,--no-whole-archive)
target_link_libraries(_c_lite_wrapper PRIVATE -Wl,--whole-archive mindspore-extendrt
-Wl,--no-whole-archive -Wl,-z,relro,-z,now,-z,noexecstack -fstack-protector-all -s)
else()
target_link_libraries(_c_lite_wrapper PRIVATE -Wl,--whole-archive mindspore-lite_static -Wl,--no-whole-archive)
target_link_libraries(_c_lite_wrapper PRIVATE -Wl,--whole-archive mindspore-lite_static
-Wl,--no-whole-archive -Wl,-z,relro,-z,now,-z,noexecstack -fstack-protector-all -s)
endif()
if(MSLITE_ENABLE_CONVERTER OR MSLITE_ENABLE_RUNTIME_GLOG)
target_link_libraries(_c_lite_wrapper PRIVATE -Wl,--no-as-needed mindspore::glog)

1
mindspore/lite/src/CMakeLists.txt
View File

@ -136,6 +136,7 @@ set(LITE_SRC
${CMAKE_CURRENT_SOURCE_DIR}/litert/sub_graph_kernel.cc
${CMAKE_CURRENT_SOURCE_DIR}/litert/scheduler.cc
${CMAKE_CURRENT_SOURCE_DIR}/litert/lite_session.cc
${CMAKE_CURRENT_SOURCE_DIR}/litert/model_manager.cc
${CMAKE_CURRENT_SOURCE_DIR}/errorcode.cc
${CMAKE_CURRENT_SOURCE_DIR}/litert/cpu_info.cc
${CMAKE_CURRENT_SOURCE_DIR}/litert/pack_weight_manager.cc

56
mindspore/lite/src/litert/cxx_api/model/model_group.cc Executable file
View File

@ -0,0 +1,56 @@
/**
* Copyright 2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "include/api/model_group.h"
#include <mutex>
#include "include/api/types.h"
#include "include/api/context.h"
#include "include/api/dual_abi_helper.h"
#include "src/litert/cxx_api/model/model_group_impl.h"
#include "src/common/log_adapter.h"
namespace mindspore {
ModelGroup::ModelGroup() {
impl_ = std::make_shared<ModelGroupImpl>();
if (impl_ == nullptr) {
MS_LOG(ERROR) << "New model group impl_ failed.";
}
}
Status ModelGroup::AddModel(const std::vector<std::string> &model_path_list) {
if (impl_ == nullptr) {
MS_LOG(ERROR) << "Model group implement is null.";
return kLiteUninitializedObj;
}
return impl_->AddModel(model_path_list);
}
Status ModelGroup::AddModel(const std::vector<std::pair<const void *, size_t>> &model_buff_list) {
if (impl_ == nullptr) {
MS_LOG(ERROR) << "Model group implement is null.";
return kLiteUninitializedObj;
}
return impl_->AddModel(model_buff_list);
}
Status ModelGroup::CalMaxSizeOfWorkspace(ModelType model_type, const std::shared_ptr<Context> &ms_context) {
if (impl_ == nullptr) {
MS_LOG(ERROR) << "Model group implement is null.";
return kLiteUninitializedObj;
}
return impl_->CalMaxSizeOfWorkspace(model_type, ms_context);
}
} // namespace mindspore

130
mindspore/lite/src/litert/cxx_api/model/model_group_impl.cc Executable file
View File

@ -0,0 +1,130 @@
/**
* Copyright 2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "src/litert/cxx_api/model/model_group_impl.h"
#include <memory>
#include <algorithm>
#include <map>
#include <string>
#include <utility>
#include <vector>
#include "include/api/types.h"
#include "include/api/context.h"
#include "src/litert/cxx_api/converters.h"
#include "src/common/log_adapter.h"
#include "src/litert/lite_session.h"
#include "src/litert/model_manager.h"
#include "src/common/config_file.h"
namespace mindspore {
using mindspore::lite::RET_OK;
Status ModelGroupImpl::AddModel(const std::vector<std::string> &model_path_list) {
if (model_path_list.empty()) {
MS_LOG(ERROR) << "Param model_path_list is empty.";
return kLiteParamInvalid;
}
for (auto &model_path : model_path_list) {
if (model_path.empty()) {
continue;
}
(void)model_path_list_.emplace_back(model_path);
}
return kSuccess;
}
Status ModelGroupImpl::AddModel(const std::vector<std::pair<const void *, size_t>> &model_buff_list) {
if (model_buff_list.empty()) {
MS_LOG(ERROR) << "Param model_buff_list is empty.";
return kLiteParamInvalid;
}
for (auto &model_buff : model_buff_list) {
if (model_buff.first == nullptr || model_buff.second == 0) {
continue;
}
(void)model_buff_list_.emplace_back(model_buff);
}
return kSuccess;
}
lite::LiteSession *ModelGroupImpl::CreateLiteSession(const std::shared_ptr<Context> &ms_context) {
auto session = new (std::nothrow) lite::LiteSession();
if (session == nullptr) {
return nullptr;
}
std::string sharing_workspace_section = "inner_common";
std::string calc_workspace_key = "inner_calc_workspace_size";
std::string calc_workspace_value = "true";
std::map<std::string, std::string> model_sharing{{calc_workspace_key, calc_workspace_value}};
config_info_[sharing_workspace_section] = model_sharing;
session->SetConfigInfo(&config_info_);
session->SetPrepareSessionFlag(true);
auto ret = session->Init(ContextUtils::Convert(ms_context.get()));
if (ret != mindspore::lite::RET_OK) {
MS_LOG(ERROR) << "init session failed";
delete session;
return nullptr;
}
return session;
}
Status ModelGroupImpl::CalMaxSizeOfWorkspace(ModelType model_type, const std::shared_ptr<Context> &ms_context) {
for (auto &model_path : model_path_list_) {
auto *session = CreateLiteSession(ms_context);
if (session == nullptr) {
MS_LOG(ERROR) << "Calculate the maximum workspace size of the model " << model_path << " failed.";
ModelManager::GetInstance().ClearModel();
return kLiteError;
}
auto ret = session->LoadModelAndCompileByPath(model_path, model_type);
if (ret != mindspore::lite::RET_OK) {
MS_LOG(ERROR) << "Calculate the maximum workspace size of the model " << model_path << " failed.";
delete session;
session = nullptr;
ModelManager::GetInstance().ClearModel();
return kLiteError;
}
ModelManager::GetInstance().AddModel(model_path);
delete session;
session = nullptr;
}
for (auto &model_buff : model_buff_list_) {
auto *session = CreateLiteSession(ms_context);
if (session == nullptr) {
MS_LOG(ERROR) << "Calculate the maximum workspace size of the model failed.";
ModelManager::GetInstance().ClearModel();
return kLiteError;
}
auto ret =
session->LoadModelAndCompileByBuf(static_cast<const char *>(model_buff.first), model_type, model_buff.second);
if (ret != mindspore::lite::RET_OK) {
MS_LOG(ERROR) << "Calculate the maximum workspace size of the model failed.";
delete session;
session = nullptr;
ModelManager::GetInstance().ClearModel();
return kLiteError;
}
ModelManager::GetInstance().AddModel(model_buff);
delete session;
session = nullptr;
}
return kSuccess;
}
} // namespace mindspore

51
mindspore/lite/src/litert/cxx_api/model/model_group_impl.h Normal file
View File

@ -0,0 +1,51 @@
/**
* Copyright 2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_SRC_CXX_API_MODEL_MODEL_GROUP_IMPL_H_
#define MINDSPORE_LITE_SRC_CXX_API_MODEL_MODEL_GROUP_IMPL_H_
#include <functional>
#include <map>
#include <string>
#include <vector>
#include <memory>
#include <utility>
#include <unordered_map>
#include "include/api/model_group.h"
#include "include/api/context.h"
#include "src/litert/lite_session.h"
#include "src/litert/inner_context.h"
namespace mindspore {
class ModelGroupImpl {
public:
ModelGroupImpl() {}
~ModelGroupImpl() = default;
Status AddModel(const std::vector<std::string> &model_path_list);
Status AddModel(const std::vector<std::pair<const void *, size_t>> &model_buff_list);
Status CalMaxSizeOfWorkspace(ModelType model_type, const std::shared_ptr<Context> &ms_context);
private:
lite::LiteSession *CreateLiteSession(const std::shared_ptr<Context> &ms_context);
friend class ModelGroup;
std::vector<std::string> model_path_list_;
std::vector<std::pair<const void *, size_t>> model_buff_list_;
std::map<std::string, std::map<std::string, std::string>> config_info_;
};
} // namespace mindspore
#endif // MINDSPORE_LITE_SRC_CXX_API_MODEL_MODEL_GROUP_IMPL_H_

32
mindspore/lite/src/litert/cxx_api/model/model_impl.cc
View File

@ -18,6 +18,7 @@
#include <memory>
#include <algorithm>
#include <map>
#include <set>
#include <string>
#include <utility>
#include <vector>
@ -30,6 +31,7 @@
#include "src/litert/cxx_api/tensor_utils.h"
#include "src/common/log_adapter.h"
#include "src/litert/lite_session.h"
#include "src/litert/model_manager.h"
#include "src/common/file_utils.h"
#if defined(ENABLE_PRE_INFERENCE) && defined(__linux__) && !defined(Debug)
#include "src/common/random_data_generator.h"
@ -42,10 +44,23 @@ namespace {
const char *const kExecutionPlan = "execution_plan";
constexpr size_t kMaxSectionNum = 100;
constexpr size_t kMaxConfigNumPerSection = 1000;
constexpr auto kSharingWorkspaceSection = "inner_common";
constexpr auto kSharingWorkspaceKey = "inner_sharing_workspace";
constexpr auto kSharingWorkspaceValue = "true";
} // namespace
using mindspore::lite::RET_ERROR;
using mindspore::lite::RET_OK;
bool ModelImpl::IsEnableModelSharing(const std::string &model_path) {
const std::set<std::string> &model_path_set = ModelManager::GetInstance().GetModelPath();
return (model_path_set.find(model_path) != model_path_set.end());
}
bool ModelImpl::IsEnableModelSharing(const std::pair<const void *, size_t> &model_buff) {
const std::set<std::pair<const void *, size_t>> &model_buff_set = ModelManager::GetInstance().GetModelBuff();
return (model_buff_set.find(model_buff) != model_buff_set.end());
}
CreateTrainSessionProto *CreateTrainSessionCallbackHolder(CreateTrainSessionProto *proto) {
static CreateTrainSessionProto *proto_ = nullptr;
if (proto != nullptr) {
@ -145,7 +160,16 @@ Status ModelImpl::Build(const void *model_data, size_t data_size, ModelType mode
MS_LOG(ERROR) << "The platform exist don't support's instruction.";
return kLiteNotSupport;
}
context_ = ms_context;
bool model_sharing_flag = IsEnableModelSharing(std::make_pair(model_data, data_size));
if (model_sharing_flag) {
auto ret = UpdateConfig(kSharingWorkspaceSection, std::make_pair(kSharingWorkspaceKey, kSharingWorkspaceValue));
if (ret != kSuccess) {
MS_LOG(ERROR) << "UpdateConfig " << kSharingWorkspaceKey << " failed.";
return ret;
}
}
auto session = std::shared_ptr<lite::LiteSession>(CreateLiteSession(ContextUtils::Convert(ms_context.get())));
if (session == nullptr) {
MS_LOG(ERROR) << "Allocate session failed.";
@ -170,6 +194,14 @@ Status ModelImpl::Build(const std::string &model_path, ModelType model_type,
return kLiteNotSupport;
}
bool model_sharing_flag = IsEnableModelSharing(model_path);
if (model_sharing_flag) {
auto ret = UpdateConfig(kSharingWorkspaceSection, std::make_pair(kSharingWorkspaceKey, kSharingWorkspaceValue));
if (ret != kSuccess) {
MS_LOG(ERROR) << "UpdateConfig " << kSharingWorkspaceKey << " failed.";
return ret;
}
}
auto session = std::shared_ptr<lite::LiteSession>(CreateLiteSession(ContextUtils::Convert(ms_context.get())));
if (session == nullptr) {
MS_LOG(ERROR) << "Allocate session failed.";

2
mindspore/lite/src/litert/cxx_api/model/model_impl.h
View File

@ -136,6 +136,8 @@ class ModelImpl {
void SetContext(const std::shared_ptr<Context> &context) { context_ = context; }
void SetConfig(const std::shared_ptr<TrainCfg> cfg) { cfg_ = cfg; }
Status RunGraph(const MSKernelCallBack &before, const MSKernelCallBack &after);
bool IsEnableModelSharing(const std::string &model_path);
bool IsEnableModelSharing(const std::pair<const void *, size_t> &model_buff);
std::map<std::string, TypeId> execution_plan_;
std::map<std::string, std::map<std::string, std::string>> config_info_;
};

86
mindspore/lite/src/litert/kernel/ascend/src/acl_mem_manager.cc Normal file
View File

@ -0,0 +1,86 @@
/**
* Copyright 2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "src/litert/kernel/ascend/src/acl_mem_manager.h"
#include <memory>
#include <algorithm>
#include <map>
#include <string>
#include "acl/acl.h"
#include "src/common/log_adapter.h"
namespace mindspore::kernel {
namespace acl {
void AclMemManager::UpdateWorkspace(size_t work_size, size_t weight_size) {
if (work_size > work_mem_info_.mem_size) {
work_mem_info_.mem_size = work_size;
MS_LOG(DEBUG) << "Update work_size = " << work_size << " successful.";
}
if (weight_size > weight_mem_info_.mem_size) {
weight_mem_info_.mem_size = weight_size;
MS_LOG(DEBUG) << "Update weight_size = " << weight_size << " successful.";
}
}
STATUS AclMemManager::GetModelWorkMem(AclModelMemInfo *acl_work_mem_info) {
std::unique_lock<std::mutex> acl_mtx(acl_mem_alloc_mutex_);
if (work_mem_info_.mem_addr == nullptr) {
if (work_mem_info_.mem_size == 0) {
return lite::RET_ERROR;
}
auto acl_ret = aclrtMalloc(&work_mem_info_.mem_addr, work_mem_info_.mem_size, ACL_MEM_MALLOC_NORMAL_ONLY);
if (acl_ret != ACL_ERROR_NONE) {
MS_LOG(ERROR) << "Call aclrtMalloc failed, err_code = " << acl_ret;
return lite::RET_ERROR;
}
MS_LOG(DEBUG) << "Malloc max work size is " << work_mem_info_.mem_size;
}
*acl_work_mem_info = work_mem_info_;
return lite::RET_OK;
}
STATUS AclMemManager::GetModelWeightMem(AclModelMemInfo *acl_weight_mem_info) {
std::unique_lock<std::mutex> acl_mtx(acl_mem_alloc_mutex_);
if (weight_mem_info_.mem_addr == nullptr) {
if (weight_mem_info_.mem_size == 0) {
return lite::RET_ERROR;
}
auto acl_ret = aclrtMalloc(&weight_mem_info_.mem_addr, weight_mem_info_.mem_size, ACL_MEM_MALLOC_NORMAL_ONLY);
if (acl_ret != ACL_ERROR_NONE) {
MS_LOG(ERROR) << "Call aclrtMalloc failed, err_code = " << acl_ret;
return lite::RET_ERROR;
}
MS_LOG(DEBUG) << "Malloc max weight size is " << weight_mem_info_.mem_size;
}
*acl_weight_mem_info = weight_mem_info_;
return lite::RET_OK;
}
AclMemManager::~AclMemManager() {
if (work_mem_info_.mem_addr != nullptr) {
(void)aclrtFree(work_mem_info_.mem_addr);
work_mem_info_.mem_addr = nullptr;
work_mem_info_.mem_size = 0;
}
if (weight_mem_info_.mem_addr != nullptr) {
(void)aclrtFree(weight_mem_info_.mem_addr);
weight_mem_info_.mem_addr = nullptr;
weight_mem_info_.mem_size = 0;
}
}
} // namespace acl
} // namespace mindspore::kernel

63
mindspore/lite/src/litert/kernel/ascend/src/acl_mem_manager.h Normal file
View File

@ -0,0 +1,63 @@
/**
* Copyright 2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ASCEND_SRC_ACL_MEM_MANAGER_H_
#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ASCEND_SRC_ACL_MEM_MANAGER_H_
#include <functional>
#include <map>
#include <mutex>
#include <string>
#include <memory>
#include "include/errorcode.h"
namespace mindspore::kernel {
namespace acl {
using mindspore::lite::STATUS;
struct AclModelMemInfo {
void *mem_addr;
size_t mem_size;
};
class AclMemManager {
public:
AclMemManager() {}
~AclMemManager();
AclMemManager(const AclMemManager &) = delete;
AclMemManager &operator=(const AclMemManager &) = delete;
static AclMemManager &GetInstance() {
static AclMemManager instance;
return instance;
}
void UpdateWorkspace(size_t work_size, size_t weight_size);
STATUS GetModelWorkMem(AclModelMemInfo *acl_work_mem_info);
STATUS GetModelWeightMem(AclModelMemInfo *acl_weight_mem_info);
void Lock() { return acl_execute_mutex_.lock(); }
void Unlock() { return acl_execute_mutex_.unlock(); }
private:
std::mutex acl_mem_alloc_mutex_;
std::mutex acl_execute_mutex_;
AclModelMemInfo work_mem_info_ = {nullptr, 0};
AclModelMemInfo weight_mem_info_ = {nullptr, 0};
};
} // namespace acl
} // namespace mindspore::kernel
#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ASCEND_SRC_ACL_MEM_MANAGER_H_

21
mindspore/lite/src/litert/kernel/ascend/src/custom_kernel.cc
View File

@ -16,6 +16,7 @@
#include "src/litert/kernel/ascend/src/custom_kernel.h"
#include <utility>
#include <map>
#include "include/registry/register_kernel.h"
#include "include/api/types.h"
#include "include/api/data_type.h"
@ -33,12 +34,14 @@ constexpr auto kNHWCWidthIdx = 2;
constexpr auto kNCHWHeightIdx = 2;
constexpr auto kNCHWWidthIdx = 3;
constexpr auto kImageSizeHwNum = 2;
constexpr auto kSharingWorkspaceSection = "inner_common";
} // namespace
CustomAscendKernel::CustomAscendKernel(const std::vector<mindspore::MSTensor> &inputs,
const std::vector<mindspore::MSTensor> &outputs,
const schema::Primitive *primitive, const mindspore::Context *ctx)
: Kernel(inputs, outputs, primitive, ctx),
load_model_(false),
prepare_flag_(false),
acl_options_({}),
model_infer_(nullptr),
InputDataIndex_(0) {}
@ -66,7 +69,7 @@ STATUS CustomAscendKernel::PrepareModelInfer() {
MS_LOG(ERROR) << "Parse acl options failed.";
return lite::RET_ERROR;
}
model_infer_ = std::make_shared<ModelInfer>(om_data, acl_options_);
model_infer_ = std::make_shared<ModelInfer>(om_data, acl_options_, this->GetConfig(kSharingWorkspaceSection));
CHECK_NULL_RETURN(model_infer_);
}
int ret = model_infer_->Init();
@ -79,6 +82,10 @@ STATUS CustomAscendKernel::PrepareModelInfer() {
MS_LOG(ERROR) << "Load om data failed.";
return lite::RET_ERROR;
}
if (prepare_flag_) {
MS_LOG(INFO) << "Update workspace success.";
return lite::RET_OK;
}
acl_options_.batch_size = model_infer_->GetDynamicBatch();
acl_options_.image_size = model_infer_->GetDynamicImage();
MS_LOG(INFO) << "Load om data success.";
@ -90,10 +97,18 @@ STATUS CustomAscendKernel::Prepare() {
MS_LOG(INFO) << "Custom kernel has been prepared.";
return lite::RET_OK;
}
const std::string calc_workspace_size = "inner_calc_workspace_size";
const std::map<std::string, std::string> &config_comm = this->GetConfig(kSharingWorkspaceSection);
if (config_comm.find(calc_workspace_size) != config_comm.end()) {
prepare_flag_ = true;
}
if (PrepareModelInfer() != lite::RET_OK) {
MS_LOG(ERROR) << "Model infer prepare is not ok.";
return lite::RET_ERROR;
}
if (prepare_flag_) {
return lite::RET_OK;
}
RecordInputDataIndex();
load_model_ = true;
@ -225,8 +240,8 @@ STATUS CustomAscendKernel::GetRealImageSize(std::vector<mindspore::MSTensor> *in
STATUS CustomAscendKernel::Execute() {
if (!load_model_) {
MS_LOG(WARNING) << "Custom kernel has not been prepared.";
return lite::RET_OK;
MS_LOG(ERROR) << "Custom kernel has not been prepared.";
return lite::RET_ERROR;
}
std::vector<mindspore::MSTensor> inputs(inputs_.begin(), inputs_.end() - 1);
if (ProcDynamicInput(&inputs) != lite::RET_OK) {

1
mindspore/lite/src/litert/kernel/ascend/src/custom_kernel.h
View File

@ -49,6 +49,7 @@ class CustomAscendKernel : public kernel::Kernel {
STATUS GetRealImageSize(std::vector<mindspore::MSTensor> *inputs, int32_t *image_size, int32_t num);
bool load_model_;
bool prepare_flag_;
AclModelOptions acl_options_;
std::shared_ptr<ModelInfer> model_infer_;
size_t InputDataIndex_;

70
mindspore/lite/src/litert/kernel/ascend/src/model_infer.cc
View File

@ -17,10 +17,17 @@
#include "src/litert/kernel/ascend/src/model_infer.h"
#include "common/log_adapter.h"
#include "acl/acl.h"
#include "src/litert/kernel/ascend/src/acl_mem_manager.h"
namespace mindspore::kernel {
namespace acl {
ModelInfer::ModelInfer(const Buffer &om_data, const AclModelOptions &options)
namespace {
constexpr auto kModelSharingPrepareKey = "multi_model_sharing_mem_prepare";
constexpr auto kModelSharingKey = "multi_model_sharing_mem";
} // namespace
ModelInfer::ModelInfer(const Buffer &om_data, const AclModelOptions &options,
const std::map<std::string, std::string> &config_info)
: init_flag_(false),
load_flag_(false),
device_type_("AscendCL"),
@ -28,6 +35,7 @@ ModelInfer::ModelInfer(const Buffer &om_data, const AclModelOptions &options)
om_data_(om_data),
options_(options),
model_process_(options),
config_info_(config_info),
acl_env_(nullptr) {}
STATUS ModelInfer::Init() {
@ -108,6 +116,20 @@ STATUS ModelInfer::Finalize() {
return lite::RET_OK;
}
bool ModelInfer::IsEnableMultiModelSharingMemPrepare() {
if (config_info_.find(kModelSharingPrepareKey) != config_info_.end()) {
return true;
}
return false;
}
bool ModelInfer::IsEnableMultiModelSharingMem() {
if (config_info_.find(kModelSharingKey) != config_info_.end()) {
return true;
}
return false;
}
STATUS ModelInfer::Load() {
if (!load_flag_) {
int ret = LoadAclModel(om_data_);
@ -131,15 +153,51 @@ STATUS ModelInfer::LoadAclModel(const Buffer &om_data) {
MS_LOG(INFO) << "Start load acl model.";
// acl load model
uint32_t acl_model_id;
auto acl_ret = aclmdlLoadFromMem(om_data.Data(), om_data.DataSize(), &acl_model_id);
if (acl_ret != ACL_ERROR_NONE) {
MS_LOG(ERROR) << "Call aclmdlLoadFromMem failed, ret = " << acl_ret;
return lite::RET_ERROR;
size_t work_size = 0;
size_t weight_size = 0;
if (IsEnableMultiModelSharingMemPrepare()) {
auto acl_ret = aclmdlQuerySizeFromMem(om_data.Data(), om_data.DataSize(), &work_size, &weight_size);
if (acl_ret != ACL_ERROR_NONE) {
MS_LOG(ERROR) << "Call aclmdlQuerySizeFromMem failed, ret = " << acl_ret;
return lite::RET_ERROR;
}
AclMemManager::GetInstance().UpdateWorkspace(work_size, weight_size);
return lite::RET_OK;
} else if (IsEnableMultiModelSharingMem()) {
AclModelMemInfo acl_work_mem_info;
AclModelMemInfo acl_weight_mem_info;
auto ret = AclMemManager::GetInstance().GetModelWorkMem(&acl_work_mem_info);
if (ret != lite::RET_OK) {
MS_LOG(ERROR) << "Get work mem failed.";
return ret;
}
ret = AclMemManager::GetInstance().GetModelWeightMem(&acl_weight_mem_info);
if (ret != lite::RET_OK) {
MS_LOG(ERROR) << "Get weight mem failed.";
return ret;
}
MS_LOG(DEBUG) << "Sharing work size = " << acl_work_mem_info.mem_size;
MS_LOG(DEBUG) << "Sharing weight size = " << acl_weight_mem_info.mem_size;
auto acl_ret =
aclmdlLoadFromMemWithMem(om_data.Data(), om_data.DataSize(), &acl_model_id, acl_work_mem_info.mem_addr,
acl_work_mem_info.mem_size, acl_weight_mem_info.mem_addr, acl_weight_mem_info.mem_size);
if (acl_ret != ACL_ERROR_NONE) {
MS_LOG(ERROR) << "Call aclmdlLoadFromMemWithMem failed, ret = " << acl_ret;
return lite::RET_ERROR;
}
model_process_.SetSharingWorkspaceFlag(true);
} else {
auto acl_ret = aclmdlLoadFromMem(om_data.Data(), om_data.DataSize(), &acl_model_id);
if (acl_ret != ACL_ERROR_NONE) {
MS_LOG(ERROR) << "Call aclmdlLoadFromMem failed, ret = " << acl_ret;
return lite::RET_ERROR;
}
}
// acl init model resource
model_process_.set_model_id(acl_model_id);
int ret = model_process_.PreInitModelResource();
auto ret = model_process_.PreInitModelResource();
if (ret != lite::RET_OK) {
(void)aclmdlUnload(acl_model_id);
MS_LOG(ERROR) << "Pre init model resource failed.";

7
mindspore/lite/src/litert/kernel/ascend/src/model_infer.h
View File

@ -18,6 +18,7 @@
#define MINDSPORE_LITE_SRC_EXTENDRT_KERNEL_ASCEND_SRC_MODEL_INFER_H_
#include <vector>
#include <map>
#include <memory>
#include <set>
#include <utility>
@ -34,7 +35,8 @@ using mindspore::lite::STATUS;
class ModelInfer {
public:
ModelInfer(const Buffer &om_data, const AclModelOptions &options);
ModelInfer(const Buffer &om_data, const AclModelOptions &options,
const std::map<std::string, std::string> &config_info);
~ModelInfer() = default;
STATUS Init();
@ -48,6 +50,8 @@ class ModelInfer {
private:
STATUS LoadAclModel(const Buffer &om_data);
bool IsEnableMultiModelSharingMemPrepare();
bool IsEnableMultiModelSharingMem();
bool init_flag_;
bool load_flag_;
@ -56,6 +60,7 @@ class ModelInfer {
Buffer om_data_;
AclModelOptions options_;
ModelProcess model_process_;
std::map<std::string, std::string> config_info_;
std::shared_ptr<AclEnvGuard> acl_env_;
};
} // namespace acl

17
mindspore/lite/src/litert/kernel/ascend/src/model_process.cc
View File

@ -22,6 +22,7 @@
#include "common/log_adapter.h"
#include "src/common/utils.h"
#include "src/common/log_util.h"
#include "src/litert/kernel/ascend/src/acl_mem_manager.h"
namespace mindspore::kernel {
namespace acl {
@ -600,7 +601,15 @@ STATUS ModelProcess::PredictFromHost(const std::vector<mindspore::MSTensor> &inp
struct timeval start_time;
struct timeval end_time;
(void)gettimeofday(&start_time, nullptr);
if (is_sharing_workspace_) {
MS_LOG(DEBUG) << "Need to lock before aclmdlExecute.";
AclMemManager::GetInstance().Lock();
}
acl_ret = aclmdlExecute(model_id_, inputs_, outputs_);
if (is_sharing_workspace_) {
MS_LOG(DEBUG) << "Need to unlock after aclmdlExecute.";
AclMemManager::GetInstance().Unlock();
}
(void)gettimeofday(&end_time, nullptr);
constexpr uint64_t kUSecondInSecond = 1000000;
uint64_t cost =
@ -608,7 +617,15 @@ STATUS ModelProcess::PredictFromHost(const std::vector<mindspore::MSTensor> &inp
(kUSecondInSecond * static_cast<uint64_t>(start_time.tv_sec) + static_cast<uint64_t>(start_time.tv_usec));
MS_LOG(INFO) << "Model execute in " << cost << " us";
} else {
if (is_sharing_workspace_) {
MS_LOG(DEBUG) << "Need to lock before aclmdlExecute.";
AclMemManager::GetInstance().Lock();
}
acl_ret = aclmdlExecute(model_id_, inputs_, outputs_);
if (is_sharing_workspace_) {
MS_LOG(DEBUG) << "Need to unlock after aclmdlExecute.";
AclMemManager::GetInstance().Unlock();
}
}
DestroyInputsDataset();

3
mindspore/lite/src/litert/kernel/ascend/src/model_process.h
View File

@ -46,6 +46,7 @@ class ModelProcess {
explicit ModelProcess(const AclModelOptions &options)
: options_(options),
model_id_(0xffffffff),
is_sharing_workspace_(false),
is_run_on_device_(false),
model_desc_(nullptr),
inputs_(nullptr),
@ -63,6 +64,7 @@ class ModelProcess {
void set_model_id(uint32_t model_id) { model_id_ = model_id; }
uint32_t model_id() const { return model_id_; }
void SetSharingWorkspaceFlag(bool is_sharing_workspace) { is_sharing_workspace_ = is_sharing_workspace; }
std::set<uint64_t> GetDynamicBatch();
std::set<std::pair<uint64_t, uint64_t>> GetDynamicImage();
@ -92,6 +94,7 @@ class ModelProcess {
AclModelOptions options_;
uint32_t model_id_;
bool is_sharing_workspace_;
// if run one device(AICPU), there is no need to alloc device memory and copy inputs to(/outputs from) device
bool is_run_on_device_;
aclmdlDesc *model_desc_;

2
mindspore/lite/src/litert/lite_session.cc
View File

@ -569,7 +569,7 @@ int LiteSession::CompileGraph(Model *model) {
return ret;
}
if (is_train_session_) {
if (is_train_session_ || is_prepare_session_) {
is_running_.store(false);
return RET_OK;
}

2
mindspore/lite/src/litert/lite_session.h
View File

@ -80,6 +80,7 @@ class LiteSession {
void SetConfigInfo(const std::map<std::string, std::map<std::string, std::string>> *config_info) {
config_info_ = config_info;
}
void SetPrepareSessionFlag(bool is_prepare_session) { is_prepare_session_ = is_prepare_session; }
const std::vector<Tensor *> &GetTensors() const { return this->tensors_; }
virtual int Train() { return mindspore::lite::RET_ERROR; }
@ -206,6 +207,7 @@ class LiteSession {
Model *model_ = nullptr;
std::atomic<bool> is_running_ = {false};
bool is_train_session_ = false;
bool is_prepare_session_ = false;
friend class TransferSession;
#if GPU_OPENCL
opencl::OpenCLRuntimeInnerWrapper *opencl_runtime_wrapper_{nullptr};

39
mindspore/lite/src/litert/model_manager.cc Normal file
View File

@ -0,0 +1,39 @@
/**
* Copyright 2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "src/litert/model_manager.h"
#include <memory>
#include <algorithm>
#include <map>
#include <string>
namespace mindspore {
void ModelManager::AddModel(const std::string model_path) { (void)model_path_set_.insert(model_path); }
void ModelManager::AddModel(const std::pair<const void *, size_t> model_buff) {
(void)model_buff_set_.insert(model_buff);
}
void ModelManager::ClearModel() {
model_path_set_.clear();
model_buff_set_.clear();
}
ModelManager::~ModelManager() {
model_path_set_.clear();
model_buff_set_.clear();
}
} // namespace mindspore

53
mindspore/lite/src/litert/model_manager.h Normal file
View File

@ -0,0 +1,53 @@
/**
* Copyright 2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_SRC_MODEL_MANAGER_H_
#define MINDSPORE_LITE_SRC_MODEL_MANAGER_H_
#include <functional>
#include <set>
#include <string>
#include <memory>
#include <utility>
#include "src/litert/lite_session.h"
namespace mindspore {
class ModelManager {
public:
ModelManager() {}
~ModelManager();
ModelManager(const ModelManager &) = delete;
ModelManager &operator=(const ModelManager &) = delete;
static ModelManager &GetInstance() {
static ModelManager instance;
return instance;
}
void AddModel(const std::string model_path);
void AddModel(const std::pair<const void *, size_t> model_buff);
const std::set<std::string> &GetModelPath() const { return model_path_set_; }
const std::set<std::pair<const void *, size_t>> &GetModelBuff() const { return model_buff_set_; }
void ClearModel();
private:
std::set<std::string> model_path_set_;
std::set<std::pair<const void *, size_t>> model_buff_set_;
};
} // namespace mindspore
#endif // MINDSPORE_LITE_SRC_MODEL_MANAGER_H_

2
mindspore/lite/test/config_level0/cropped_size.cfg
View File

@ -1,2 +1,2 @@
Note: This is the mindspore Lite inference framework size threshold. Offline review is required before modify this value!!!
1106192
1116516

2
mindspore/lite/test/config_level1/cropped_size.cfg
View File

@ -1,2 +1,2 @@
Note: This is the mindspore Lite inference framework size threshold. Modifying this threshold requires meeting review.
1106192
1116516

1
mindspore/lite/tools/converter/CMakeLists.txt
View File

@ -157,6 +157,7 @@ set(LITE_SRC ${API_SRC}
${SRC_DIR}/litert/lite_session.cc
${SRC_DIR}/litert/executor.cc
${SRC_DIR}/litert/lite_model.cc
${SRC_DIR}/litert/model_manager.cc
${SRC_DIR}/errorcode.cc
${SRC_DIR}/litert/weight_decoder.cc
${SRC_DIR}/litert/pack_weight_manager.cc

2
mindspore/lite/tools/converter/adapter/dpico/CMakeLists.txt
View File

@ -27,6 +27,7 @@ file(GLOB_RECURSE DPICO_SRC
${CMAKE_CURRENT_SOURCE_DIR}/checker/*.cc
${CMAKE_CURRENT_SOURCE_DIR}/infer/*.cc
${CMAKE_CURRENT_SOURCE_DIR}/mapper/*.cc
${TOOLS_DIR}/converter/adapter/utils.cc
)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR})
@ -34,6 +35,7 @@ include_directories(BEFORE ${OPENCV_PATH}include/opencv4)
include_directories(BEFORE ${PROTOBUF_PATH}/include)
include_directories(${PICO_MAPPER_PATH}/include)
include_directories(${PROTO_OUTPUT_PATH})
include_directories(${TOOLS_DIR}/converter)
link_directories(${pico_mapper_SOURCE_DIR}/lib)
set_property(SOURCE ${DPICO_SRC} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_LITE)

129
mindspore/lite/tools/converter/adapter/dpico/src/calib_data_generator.cc
View File

@ -20,12 +20,15 @@
#include <set>
#include <string>
#include <numeric>
#include <algorithm>
#include "ops/tuple_get_item.h"
#include "common/anf_util.h"
#include "common/string_util.h"
#include "common/file_util.h"
#include "src/mapper_config_parser.h"
#include "src/data_preprocessor.h"
#include "adapter/utils.h"
using mindspore::lite::RET_ERROR;
using mindspore::lite::RET_OK;
namespace mindspore {
@ -33,44 +36,17 @@ namespace dpico {
namespace {
constexpr size_t kMaxSize = 1024;
} // namespace
int CalibDataGenerator::GenerateDumpConfig(const std::string &dump_cfg_path,
const std::vector<DumpOpInfo> &dump_op_infos) {
auto output_path = MapperConfigParser::GetInstance()->GetOutputPath();
auto kernels_str = std::accumulate(dump_op_infos.begin(), dump_op_infos.end(), std::string("["),
[](const std::string &cur_str, const DumpOpInfo &dump_op_info) {
return cur_str + "\"" + dump_op_info.dump_op_name + "\",";
});
kernels_str.replace(kernels_str.size() - 1, 1, "]");
std::ofstream dump_ofs;
dump_ofs.open(dump_cfg_path, std::ios::out);
if (!dump_ofs.is_open()) {
MS_LOG(ERROR) << "file open failed." << dump_cfg_path;
return RET_ERROR;
}
dump_ofs << "{" << std::endl;
dump_ofs << "\"common_dump_settings\": {" << std::endl;
dump_ofs << "\"dump_mode\": 1," << std::endl;
dump_ofs << R"("path": ")" << output_path << "\"," << std::endl;
dump_ofs << R"("net_name": "dump_data",)" << std::endl;
dump_ofs << "\"input_output\": " << dump_level_ << "," << std::endl;
dump_ofs << "\"kernels\":" << kernels_str << std::endl;
dump_ofs << "}" << std::endl; // end of common_dump_setting
dump_ofs << "}" << std::endl; // end of dump oss
dump_ofs.close();
return RET_OK;
}
std::string CalibDataGenerator::GetInputShapesStr(const api::AnfNodePtrList &graph_inputs) {
std::string input_shapes_str;
std::vector<std::vector<int64_t>> CalibDataGenerator::GetInputShapes(const api::AnfNodePtrList &graph_inputs) {
std::vector<std::vector<int64_t>> input_shapes;
for (const auto &input : graph_inputs) {
ShapeVector shape_vector;
if (GetShapeVectorFromParameter(input, &shape_vector) != RET_OK) {
MS_LOG(ERROR) << "get graph input shape failed. " << input->fullname_with_scope();
return "";
return std::vector<std::vector<int64_t>>();
}
if (shape_vector.empty()) {
MS_LOG(ERROR) << input->fullname_with_scope() << " input shape is empty.";
return "";
return std::vector<std::vector<int64_t>>();
}
if (shape_vector.size() == kDims4) { // transform nchw to nhwc
shape_vector = std::vector<int64_t>{shape_vector[kNCHW_N], shape_vector[kNCHW_H], shape_vector[kNCHW_W],
@ -84,20 +60,13 @@ std::string CalibDataGenerator::GetInputShapesStr(const api::AnfNodePtrList &gra
} else {
MS_LOG(ERROR) << input->fullname_with_scope() << "'s input shape[" << i << "] is " << dim
<< ", which is unsupported by dpico.";
return "";
return std::vector<std::vector<int64_t>>();
}
}
input_shapes_str += std::to_string(dim);
if (i != shape_vector.size() - 1) {
input_shapes_str += ',';
}
}
input_shapes_str += ':';
input_shapes.emplace_back(shape_vector);
}
if (!input_shapes_str.empty() && input_shapes_str.at(input_shapes_str.size() - 1) == ':') {
input_shapes_str.pop_back();
}
return input_shapes_str;
return input_shapes;
}
std::vector<std::string> CalibDataGenerator::GetInDataFileList(const api::AnfNodePtrList &graph_inputs) {
@ -125,57 +94,20 @@ std::vector<std::string> CalibDataGenerator::GetInDataFileList(const api::AnfNod
return {};
}
}
in_data_files_list.emplace_back(in_data_files);
(void)in_data_files_list.emplace_back(in_data_files);
}
return in_data_files_list;
}
int CalibDataGenerator::DumpKernelsData(const std::string &dump_cfg_path,
int CalibDataGenerator::DumpKernelsData(const std::string &current_path,
const std::vector<std::string> &in_data_file_list,
const std::string &input_shapes_str) {
if (setenv("MINDSPORE_DUMP_CONFIG", dump_cfg_path.c_str(), 1)) {
MS_LOG(ERROR) << "set env for dump config failed.";
return RET_ERROR;
}
std::string benchmark_path = "../../benchmark/benchmark";
bool use_default_benchmark = true;
auto config_info = converter::ConverterContext::GetConfigInfo("dpico");
if (!config_info.empty()) {
if (config_info.find("benchmark_path") != config_info.end()) {
benchmark_path = config_info.at("benchmark_path");
use_default_benchmark = false;
}
}
if (use_default_benchmark) {
MS_LOG(WARNING) << R"(there is no "benchmark_path" in the converter config file,
will use the default value: "../../benchmark/benchmark")";
}
if (AccessFile(benchmark_path, F_OK) != 0) {
MS_LOG(ERROR) << "File not exist: " << benchmark_path;
return RET_ERROR;
}
std::string current_path;
auto benchmark_file = RealPath(benchmark_path.c_str());
if (benchmark_file.empty()) {
MS_LOG(ERROR) << "Get realpath failed, benchmark path is " << benchmark_path;
return RET_ERROR;
}
std::string model_file = current_path + "model.ms";
char buf[kMaxSize] = {0};
for (const auto &in_data_file : in_data_file_list) {
std::string result;
std::string cmd = benchmark_file + " --modelFile=" + model_file + " --inDataFile=" + in_data_file +
" --inputShapes=" + input_shapes_str + " --loopCount=1 --warmUpLoopCount=0";
MS_LOG(INFO) << "begin to run benchmark: " << cmd;
auto fp = popen(cmd.c_str(), "r");
if (fp != nullptr) {
while (fgets(buf, kMaxSize, fp) != nullptr) {
result += buf;
}
}
pclose(fp);
if (result.find("Success") == std::string::npos) {
MS_LOG(ERROR) << "Run benchmark failed. " << result;
const std::vector<std::string> kernel_names,
const std::vector<std::vector<int64_t>> &input_shapes) {
std::string model_file = "model.ms";
for (auto in_data_file : in_data_file_list) {
auto ret = lite::converter::InnerPredict(model_file, in_data_file, kernel_names, current_path, input_shapes);
if (ret != RET_OK) {
MS_LOG(ERROR) << "Execute InnerPredict failed.";
return RET_ERROR;
}
}
@ -237,7 +169,7 @@ int CalibDataGenerator::TransBinsToTxt(const std::vector<DumpOpInfo> &dump_op_in
MS_LOG(ERROR) << "file open failed. " << dump_op_txt_path;
return RET_ERROR;
}
ofs.precision(kNumPrecision);
(void)ofs.precision(kNumPrecision);
bool is_input = dump_op_info.input_index >= 0;
auto del_special_character = ReplaceSpecifiedChar(dump_op_info.dump_op_name, '/', '.');
auto pattern = is_input
@ -310,7 +242,7 @@ int CalibDataGenerator::Run(const api::AnfNodePtrList &graph_inputs, const api::
if (has_visited.find(node) != has_visited.end()) {
continue;
}
has_visited.insert(node);
(void)has_visited.insert(node);
DumpOpInfo dump_op_info = {node->fullname_with_scope(), node->fullname_with_scope(), -1, -1};
if (control_flow_inputs_.find(node) != control_flow_inputs_.end()) {
dump_op_info.dump_op_name = control_flow_inputs_[node].first->fullname_with_scope();
@ -328,7 +260,7 @@ int CalibDataGenerator::Run(const api::AnfNodePtrList &graph_inputs, const api::
}
}
if (image_lists.find(dump_op_info.dump_op_name) == image_lists.end()) {
dump_op_infos.emplace_back(dump_op_info);
(void)dump_op_infos.emplace_back(dump_op_info);
}
}
if (dump_op_infos.empty()) {
@ -336,25 +268,23 @@ int CalibDataGenerator::Run(const api::AnfNodePtrList &graph_inputs, const api::
return RET_ERROR;
}
std::string dump_cfg_path = MapperConfigParser::GetInstance()->GetOutputPath() + "generated_dump.cfg";
if (GenerateDumpConfig(dump_cfg_path, dump_op_infos) != RET_OK) {
MS_LOG(ERROR) << "generate dump config failed.";
return RET_ERROR;
}
auto in_data_file_list = GetInDataFileList(graph_inputs);
if (in_data_file_list.empty()) {
MS_LOG(ERROR) << "get in data file for benchmark failed.";
return RET_ERROR;
}
auto input_shapes_str = GetInputShapesStr(graph_inputs);
if (input_shapes_str.empty()) {
auto input_shapes = GetInputShapes(graph_inputs);
if (input_shapes.empty()) {
MS_LOG(ERROR) << "get input shapes for benchmark failed.";
return RET_ERROR;
}
if (DumpKernelsData(dump_cfg_path, in_data_file_list, input_shapes_str) != RET_OK) {
std::vector<std::string> kernel_names;
(void)std::transform(dump_op_infos.begin(), dump_op_infos.end(), std::back_inserter(kernel_names),
[](DumpOpInfo const &op_info) { return op_info.dump_op_name; });
auto output_path = MapperConfigParser::GetInstance()->GetOutputPath();
if (DumpKernelsData(output_path, in_data_file_list, kernel_names, input_shapes) != RET_OK) {
MS_LOG(ERROR) << "dump kernels data failed.";
return RET_ERROR;
}
@ -363,6 +293,7 @@ int CalibDataGenerator::Run(const api::AnfNodePtrList &graph_inputs, const api::
MS_LOG(ERROR) << "transform dumped files to txt failed.";
return RET_ERROR;
}
return RET_OK;
}
} // namespace dpico

13
mindspore/lite/tools/converter/adapter/dpico/src/calib_data_generator.h
View File

@ -57,10 +57,11 @@ class CalibDataGenerator {
private:
int GenerateDumpConfig(const std::string &dump_cfg_path, const std::vector<DumpOpInfo> &dump_infos);
std::string GetInputShapesStr(const api::AnfNodePtrList &graph_inputs);
std::vector<std::vector<int64_t>> GetInputShapes(const api::AnfNodePtrList &graph_inputs);
std::vector<std::string> GetInDataFileList(const api::AnfNodePtrList &graph_inputs);
int DumpKernelsData(const std::string &dump_cfg_path, const std::vector<std::string> &in_data_file_list,
const std::string &input_shapes_str);
const std::vector<std::string> kernel_names,
const std::vector<std::vector<int64_t>> &input_shapes);
STATUS ParseAttrFromFilename(struct OpAttr *op_attr, const std::string &file_name, bool is_input);
int TransBinsToTxt(const std::vector<DumpOpInfo> &dump_infos);
@ -75,7 +76,7 @@ class CalibDataGenerator {
size_t shape_size = 1;
for (size_t i = 0; i < op_attr.shape.size(); i++) {
if (op_attr.shape.at(i) < 0) {
MS_LOG(ERROR) << "dim val should be greater than 0";
MS_LOG(ERROR) << "dim val should be equal or greater than 0";
return RET_ERROR;
}
if (SIZE_MUL_OVERFLOW(shape_size, static_cast<size_t>(op_attr.shape.at(i)))) {
@ -84,7 +85,7 @@ class CalibDataGenerator {
}
shape_size *= static_cast<size_t>(op_attr.shape.at(i));
}
ifs.seekg(0, std::ios::end);
(void)ifs.seekg(0, std::ios::end);
size_t file_size = static_cast<size_t>(ifs.tellg());
if (file_size != shape_size * sizeof(T)) {
MS_LOG(ERROR) << "file size " << file_size << " is not equal to shape size " << shape_size;
@ -95,8 +96,8 @@ class CalibDataGenerator {
MS_LOG(ERROR) << "new T failed.";
return RET_ERROR;
}
ifs.seekg(0, std::ios::beg);
ifs.read(reinterpret_cast<char *>(raw_datas.get()), shape_size * sizeof(T));
(void)ifs.seekg(0, std::ios::beg);
(void)ifs.read(reinterpret_cast<char *>(raw_datas.get()), shape_size * sizeof(T));
ifs.close();
if (op_attr.format == "NHWC" && op_attr.shape.size() == kDims4) {
auto dst_datas = std::make_unique<T[]>(shape_size);

280
mindspore/lite/tools/converter/adapter/utils.cc Normal file
View File

@ -0,0 +1,280 @@
/**
* Copyright 2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "adapter/utils.h"
#include <cstdio>
#ifndef _WIN32
#include <dlfcn.h>
#include <unistd.h>
#else
#include <direct.h>
#include <io.h>
#endif
#include <dirent.h>
#include <sys/stat.h>
#include <algorithm>
#include <functional>
#include <unordered_map>
#include <memory>
#include <numeric>
#include <string>
#include <vector>
#include <regex>
#include "include/api/model.h"
#include "include/api/context.h"
#include "src/common/file_utils.h"
#ifdef _WIN32
#define ACCESS _access
#define RMDIR _rmdir
#else
#define ACCESS access
#define RMDIR rmdir
#endif
namespace mindspore {
namespace lite {
namespace converter {
constexpr int kThreadNum = 2;
constexpr bool kEnableParallel = false;
constexpr int kThreadAffinity = 1;
constexpr int kInterOpParallelNum = 1;
constexpr bool kEnableFP16 = false;
const std::unordered_map<int, std::string> kTypeIdMap{
{kNumberTypeFloat16, "Float16"}, {kNumberTypeFloat, "Float32"}, {kNumberTypeFloat32, "Float32"},
{kNumberTypeInt8, "Int8"}, {kNumberTypeInt16, "Int16"}, {kNumberTypeInt, "Int32"},
{kNumberTypeInt32, "Int32"}, {kNumberTypeUInt8, "UInt8"}, {kNumberTypeUInt16, "UInt16"},
{kNumberTypeUInt, "UInt32"}, {kNumberTypeUInt32, "UInt32"}, {kObjectTypeString, "String"},
{kNumberTypeBool, "Bool"}, {kObjectTypeTensorType, "Tensor"}};
const std::unordered_map<mindspore::Format, std::string> kTensorFormatMap{
{mindspore::NCHW, "NCHW"}, {mindspore::NHWC, "NHWC"}, {mindspore::NHWC4, "NHWC4"}, {mindspore::HWKC, "HWKC"},
{mindspore::HWCK, "HWCK"}, {mindspore::KCHW, "KCHW"}, {mindspore::CKHW, "CKHW"}, {mindspore::KHWC, "KHWC"},
{mindspore::CHWK, "CHWK"}, {mindspore::HW, "HW"}, {mindspore::HW4, "HW4"}, {mindspore::NC, "NC"},
{mindspore::NC4, "NC4"}, {mindspore::NC4HW4, "NC4HW4"}, {mindspore::NCDHW, "NCDHW"}};
bool RemoveDir(const std::string &path) {
std::string str_path = path;
#ifdef _WIN32
struct _finddata_t fb;
if (str_path.at(str_path.length() - 1) != '\\' || str_path.at(str_path.length() - 1) != '/') str_path.append("\\");
std::string find_path = str_path + "*";
intptr_t handle = _findfirst(find_path.c_str(), &fb);
if (handle != -1L) {
std::string tmp_path;
do {
if (strcmp(fb.name, "..") != 0 && strcmp(fb.name, ".") != 0) {
tmp_path.clear();
tmp_path = str_path + std::string(fb.name);
if (fb.attrib == _A_SUBDIR) {
(void)RemoveDir(tmp_path.c_str());
} else {
remove(tmp_path.c_str());
}
}
} while (_findnext(handle, &fb) == 0);
_findclose(handle);
}
return RMDIR(str_path.c_str()) == 0 ? true : false;
#else
if (str_path.at(str_path.length() - 1) != '\\' || str_path.at(str_path.length() - 1) != '/') {
(void)str_path.append("/");
}
DIR *d = opendir(str_path.c_str());
if (d != nullptr) {
struct dirent *dt = readdir(d);
while (dt) {
if (strcmp(dt->d_name, "..") != 0 && strcmp(dt->d_name, ".") != 0) {
struct stat st {};
std::string file_name = str_path + std::string(dt->d_name);
(void)stat(file_name.c_str(), &st);
if (S_ISDIR(st.st_mode)) {
(void)RemoveDir(file_name);
} else {
(void)remove(file_name.c_str());
}
}
dt = readdir(d);
}
(void)closedir(d);
}
return rmdir(str_path.c_str()) == 0;
#endif
}
int ReadInputFile(const std::vector<std::string> &in_data_files, std::vector<MSTensor> *input_tensors) {
for (size_t i = 0; i < in_data_files.size(); i++) {
auto &cur_tensor = (*input_tensors).at(i);
MS_ASSERT(cur_tensor != nullptr);
size_t size;
char *bin_buf = lite::ReadFile(in_data_files[i].c_str(), &size);
if (bin_buf == nullptr) {
MS_LOG(ERROR) << "ReadFile return nullptr";
return RET_ERROR;
}
if (static_cast<int>(cur_tensor.DataType()) == kObjectTypeString) {
std::string str(bin_buf, size);
MSTensor *input = MSTensor::StringsToTensor(cur_tensor.Name(), {str});
if (input == nullptr) {
MS_LOG(ERROR) << "StringsToTensor failed";
delete[] bin_buf;
return RET_ERROR;
}
cur_tensor = *input;
} else {
auto tensor_data_size = cur_tensor.DataSize();
if (size != tensor_data_size) {
MS_LOG(ERROR) << "Input binary file size error, required: " << tensor_data_size << ", in fact: " << size;
delete[] bin_buf;
return RET_ERROR;
}
auto input_data = cur_tensor.MutableData();
if (input_data == nullptr) {
MS_LOG(ERROR) << "input_data is nullptr.";
delete[] bin_buf;
return RET_ERROR;
}
auto ret = memcpy_s(input_data, tensor_data_size, bin_buf, size);
if (ret != EOK) {
MS_LOG(ERROR) << "Execute memcpy_s failed.";
delete[] bin_buf;
return RET_ERROR;
}
}
delete[] bin_buf;
}
return RET_OK;
}
std::string GenerateOutputFileName(const mindspore::MSTensor *tensor, const std::string &op_name,
const std::string &file_type, const size_t &idx) {
std::string file_name = op_name;
auto pos = file_name.find_first_of('/');
while (pos != std::string::npos) {
file_name.replace(pos, 1, ".");
pos = file_name.find_first_of('/');
}
file_name += "_" + file_type + "_" + std::to_string(idx) + "_shape_";
for (const auto &dim : tensor->Shape()) {
file_name += std::to_string(dim) + "_";
}
if (kTypeIdMap.find(static_cast<int>(tensor->DataType())) != kTypeIdMap.end()) {
file_name += kTypeIdMap.at(static_cast<int>(tensor->DataType()));
}
auto tensor_format = tensor->format();
if (kTensorFormatMap.find(tensor_format) != kTensorFormatMap.end()) {
file_name += "_" + kTensorFormatMap.at(tensor_format) + ".bin";
}
file_name += +".bin";
return file_name;
}
int InnerPredict(const std::string &model_name, const std::string &in_data_file,
const std::vector<std::string> &output_names, const std::string &dump_directory,
const std::vector<std::vector<int64_t>> &input_shapes) {
std::string dump_data_path = dump_directory + "dump_data/";
if (ACCESS(dump_data_path.c_str(), 0) == 0 && !RemoveDir(dump_data_path)) {
MS_LOG(ERROR) << "The dump data path is exist and rmdir failed: "
<< " " << dump_data_path;
return RET_ERROR;
}
if (lite::CreateOutputDir(&dump_data_path) != RET_OK) {
MS_LOG(ERROR) << "create data output directory failed.";
return RET_ERROR;
}
MSKernelCallBack ms_after_call_back = [output_names, dump_data_path](
const std::vector<mindspore::MSTensor> &after_inputs,
const std::vector<mindspore::MSTensor> &after_outputs,
const MSCallBackParam &call_param) {
if (std::find(output_names.begin(), output_names.end(), call_param.node_name) != output_names.end()) {
for (size_t i = 0; i < after_outputs.size(); i++) {
auto ms_tensor = after_outputs.at(i);
auto file_name = GenerateOutputFileName(&ms_tensor, call_param.node_name, "output", i);
auto abs_file_path = dump_data_path + "/" + file_name;
if (lite::WriteToBin(abs_file_path, ms_tensor.MutableData(), ms_tensor.DataSize()) != RET_OK) {
MS_LOG(ERROR) << "write tensor data to file failed.";
return false;
}
}
}
return true;
};
mindspore::Model ms_model;
mindspore::ModelType model_type = mindspore::ModelType::kMindIR_Lite;
auto context = std::make_shared<mindspore::Context>();
if (context == nullptr) {
MS_LOG(ERROR) << "New context failed while running " << model_name.c_str();
return RET_ERROR;
}
context->SetThreadNum(kThreadNum);
context->SetEnableParallel(kEnableParallel);
context->SetThreadAffinity(kThreadAffinity);
context->SetInterOpParallelNum(kInterOpParallelNum);
auto &device_list = context->MutableDeviceInfo();
std::shared_ptr<CPUDeviceInfo> device_info = std::make_shared<CPUDeviceInfo>();
device_info->SetEnableFP16(kEnableFP16);
device_list.push_back(device_info);
auto ret = ms_model.Build(model_name, model_type, context);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Execute model " << model_name.c_str() << " build failed.";
return RET_ERROR;
}
if (!input_shapes.empty()) {
if (input_shapes.size() != ms_model.GetInputs().size()) {
MS_LOG(ERROR) << "Size of inputs_shape_list should be equal to size of inputs";
return RET_ERROR;
}
ret = ms_model.Resize(ms_model.GetInputs(), input_shapes);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Input tensor resize failed.";
return RET_ERROR;
}
}
auto inputs = ms_model.GetInputs();
if (inputs.empty()) {
MS_LOG(ERROR) << "Inputs is empty.";
return RET_ERROR;
}
std::regex re{"[\\s,]+"};
std::vector<std::string> in_data_list = std::vector<std::string>{
std::sregex_token_iterator(in_data_file.begin(), in_data_file.end(), re, -1), std::sregex_token_iterator()};
if (in_data_list.size() != inputs.size()) {
MS_LOG(ERROR) << "Size of inputs should be equal to size of input inDataPath";
return RET_ERROR;
}
auto status = ReadInputFile(in_data_list, &inputs);
if (status != RET_OK) {
MS_LOG(ERROR) << "ReadInputFile error, " << status;
return status;
}
std::vector<MSTensor> outputs;
ret = ms_model.Predict(inputs, &outputs, nullptr, ms_after_call_back);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Predict error ";
return RET_ERROR;
}
return RET_OK;
}
} // namespace converter
} // namespace lite
} // namespace mindspore

42
mindspore/lite/tools/converter/adapter/utils.h Normal file
View File

@ -0,0 +1,42 @@
/**
* Copyright 2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_TOOLS_CONVERTER_ADAPTER_UTILS_H
#define MINDSPORE_LITE_TOOLS_CONVERTER_ADAPTER_UTILS_H
#include <map>
#include <string>
#include <vector>
#include <fstream>
#include <iomanip>
#include "securec/include/securec.h"
#include "mindapi/base/base.h"
#include "include/errorcode.h"
using mindspore::lite::RET_ERROR;
using mindspore::lite::RET_OK;
using mindspore::lite::STATUS;
namespace mindspore {
namespace lite {
namespace converter {
bool RemoveDir(const std::string &path);
int InnerPredict(const std::string &model_name, const std::string &in_data_file,
const std::vector<std::string> &output_names, const std::string &dump_directory,
const std::vector<std::vector<int64_t>> &input_shapes = std::vector<std::vector<int64_t>>());
} // namespace converter
} // namespace lite
} // namespace mindspore
#endif // MINDSPORE_LITE_TOOLS_CONVERTER_NNIE_NNIE_SEGDATA_GENERATE_H