Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

mindspore lite: add lite cpu infer session for lite runtime

This commit is contained in:
liu lili 2022-07-26 08:58:46 +08:00
parent 740776ac05
commit 30b2e7f22e
30 changed files with 1313 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
mindspore/lite/src/CMakeLists.txt
View File

@ -138,52 +138,10 @@ set(LITE_SRC
${CMAKE_CURRENT_SOURCE_DIR}/control_flow/control_subgraph_creator.cc
)
# set(MODEL_LOADER_FRAMEWORK_SRC
# ${MODEL_LOADER_FRAMEWORK_SRC}
# ${CMAKE_CURRENT_SOURCE_DIR}/extendrt/mindir_loader/model_loader.cc
# )
# if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE)
# add_compile_definitions(ENABLE_CLOUD_FUSION_INFERENCE)
# remove_definitions(-DBUILD_LITE_INFERENCE)
# # include(${LITE_DIR}/cmake/ccsrc_module.cmake)
# # string(REPLACE "-Werror" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
# # string(REPLACE "-Werror" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
# # set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-error=return-type")
# # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=return-type")
# set(MINDIR_MODEL_SRC
# ${MINDIR_MODEL_SRC}
# ${CMAKE_CURRENT_SOURCE_DIR}/extendrt/mindir_loader/mindir_model/mindir_model.cc
# ${CMAKE_CURRENT_SOURCE_DIR}/extendrt/mindir_loader/mindir_model/mindir_model_util.cc
# ${CMAKE_CURRENT_SOURCE_DIR}/extendrt/mindir_loader/mindir_model/mindir_model_convertor.cc
# ${CMAKE_CURRENT_SOURCE_DIR}/extendrt/mindir_loader/mindir_model/mindir_model_loader.cc
# ${CMAKE_CURRENT_SOURCE_DIR}/extendrt/mindir_loader/mindir_model/kernel_mod_util.cc
# )
# set(MINDIR_KERNEL_SRC
# ${MINDIR_KERNEL_SRC}
# ${CMAKE_CURRENT_SOURCE_DIR}/extendrt/mindir_loader/mindir_model/inner_kernel.cc)
# set(LITE_SRC
# ${LITE_SRC}
# ${CMAKE_CURRENT_SOURCE_DIR}/extendrt/mindir_loader/mindir_model/less_test_kernel_mod.cc
# ${CMAKE_CURRENT_SOURCE_DIR}/extendrt/mindir_loader/mindir_model/kernel_mod_mock.cc)
# set(MSLITE_EXTEND_RUNTIME_SRC ${MSLITE_EXTEND_RUNTIME_SRC}
# ${MINDIR_MODEL_SRC}
# ${MINDIR_KERNEL_SRC}
# ${CMAKE_CURRENT_SOURCE_DIR}/extendrt/mindir_loader/mindir_model/less_test_kernel_mod.cc
# ${CMAKE_CURRENT_SOURCE_DIR}/extendrt/mindir_loader/mindir_model/kernel_mod_mock.cc)
add_subdirectory(extendrt)
# else()
# set(MODEL_LOADER_FRAMEWORK_SRC
# ${MODEL_LOADER_FRAMEWORK_SRC}
# ${CMAKE_CURRENT_SOURCE_DIR}/extendrt/mindir_loader/model_loader.cc
# )
# endif()
set(MODEL_LOADER_FRAMEWORK_SRC
${MODEL_LOADER_FRAMEWORK_SRC}
${CMAKE_CURRENT_SOURCE_DIR}/extendrt/mindir_loader/model_loader.cc
)
set(LITE_SRC
${LITE_SRC}
@ -501,8 +459,8 @@ target_link_libraries(mindspore-lite cpu_kernel_mid nnacl_mid cpu_ops_mid)
target_link_libraries(mindspore-lite_static cpu_kernel_mid nnacl_mid cpu_ops_mid)
# endif()
target_link_libraries(mindspore-lite mindspore-extendrt)
target_link_libraries(mindspore-lite_static mindspore-extendrt)
# target_link_libraries(mindspore-lite mindspore-extendrt)
# target_link_libraries(mindspore-lite_static mindspore-extendrt)
if(SUPPORT_TRAIN)
target_link_libraries(mindspore-lite train_cpu_kernel_mid)
@ -683,3 +641,5 @@ if(MSLITE_ENABLE_ACL AND (NOT MSLITE_ENABLE_CLOUD_FUSION_INFERENCE))
target_link_libraries(mindspore-lite ascend_kernel_mid)
target_link_libraries(mindspore-lite_static ascend_kernel_mid)
endif()
add_subdirectory(extendrt)

17
mindspore/lite/src/extendrt/CMakeLists.txt
View File

@ -46,6 +46,14 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE)
${CMAKE_CURRENT_SOURCE_DIR}/delegate/graph_executor/tensorrt/graph_executor.cc
)
if(NOT MSLITE_ENABLE_ACL)
set(MSLITE_EXTEND_RUNTIME_SRC ${MSLITE_EXTEND_RUNTIME_SRC}
${MINDIR_KERNEL_SRC}
${MINDIR_MODEL_SRC}
${CMAKE_CURRENT_SOURCE_DIR}/session/lite_infer_session.cc
)
endif()
set(FBS_FILES
${CCSRC_DIR}/../schema/cipher.fbs
${CCSRC_DIR}/../schema/fl_job.fbs
@ -67,7 +75,6 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE)
add_library(mindspore-lite-proto OBJECT ${MSLITE_PROTO_SRC})
set(ANF_ALG_SRC ${ANF_ALG_SRC}
${CCSRC_DIR}/utils/anfalgo.cc
${CCSRC_DIR}/utils/parallel_context.cc
@ -121,11 +128,15 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE)
add_dependencies(mindspore-extendrt mindspore-kernel-graph)
add_subdirectory(cxx_api)
if(NOT MSLITE_ENABLE_ACL)
add_subdirectory(mock/lite_runtime)
add_dependencies(mindspore-extendrt mindspore-infer-lite)
target_link_libraries(mindspore-extendrt mindspore-infer-lite)
endif()
add_subdirectory(${CCSRC_DIR}/backend/common/pass common_pass)
add_subdirectory(${CCSRC_DIR}/backend/common/optimizer mindspore_ccsrc_backend_cmmon_optimizer)
target_link_libraries(mindspore-extendrt mindspore_infer_shared_lib_obj)
target_link_libraries(mindspore-extendrt mindspore-infer-anfalgo
mindspore-kernel-graph _mindspore_backend_common_optimizer_obj
_mindspore_backend_common_pass_obj)

47
mindspore/lite/src/extendrt/cxx_api/context.cc
View File

@ -62,21 +62,6 @@ constexpr auto KModelOptionAscend310FusionSwitchCfgPath = "mindspore.option.asce
constexpr auto kModelOptionAscend310DynamicBatchSize = "mindspore.option.ascend310.dynamic_batch_size";
constexpr auto kModelOptionAscend310BufferOptimize = "mindspore.option.ascend310.buffer_optimize";
// class Allocator {};
// struct Context::Data {
// std::vector<std::shared_ptr<DeviceInfoContext>> device_info_list;
// int32_t thread_num;
// bool enable_parallel_ = false;
// std::vector<int32_t> affinity_core_list_;
// int affinity_mode_ = 2;
// std::shared_ptr<Delegate> delegate = nullptr;
// };
// struct DeviceInfoContext::Data {
// std::map<std::string, std::any> params;
// };
Context::Context() : data_(std::make_shared<Data>()) {}
template <class T, typename U = std::remove_cv_t<std::remove_reference_t<T>>>
@ -229,6 +214,22 @@ std::vector<std::shared_ptr<DeviceInfoContext>> &Context::MutableDeviceInfo() {
DeviceInfoContext::DeviceInfoContext() : data_(std::make_shared<Data>()) {}
void DeviceInfoContext::SetProvider(const std::vector<char> &provider) {
if (data_ == nullptr) {
MS_LOG(ERROR) << "Invalid context.";
return;
}
data_->params[kModelOptionProvider] = CharToString(provider);
}
void DeviceInfoContext::SetProviderDevice(const std::vector<char> &device) {
if (data_ == nullptr) {
MS_LOG(ERROR) << "Invalid context.";
return;
}
data_->params[kModelOptionProviderDevice] = CharToString(device);
}
std::vector<char> DeviceInfoContext::GetProviderChar() const {
if (data_ == nullptr) {
MS_LOG(ERROR) << "Invalid context.";
@ -238,14 +239,6 @@ std::vector<char> DeviceInfoContext::GetProviderChar() const {
return StringToChar(ref);
}
void DeviceInfoContext::SetProvider(const std::vector<char> &provider) {
if (data_ == nullptr) {
MS_LOG(ERROR) << "Invalid context.";
return;
}
data_->params[kModelOptionProvider] = CharToString(provider);
}
std::vector<char> DeviceInfoContext::GetProviderDeviceChar() const {
if (data_ == nullptr) {
MS_LOG(ERROR) << "Invalid context.";
@ -255,14 +248,6 @@ std::vector<char> DeviceInfoContext::GetProviderDeviceChar() const {
return StringToChar(ref);
}
void DeviceInfoContext::SetProviderDevice(const std::vector<char> &device) {
if (data_ == nullptr) {
MS_LOG(ERROR) << "Invalid context.";
return;
}
data_->params[kModelOptionProviderDevice] = CharToString(device);
}
void DeviceInfoContext::SetAllocator(const std::shared_ptr<Allocator> &allocator) {
if (data_ == nullptr) {
MS_LOG(ERROR) << "Invalid context.";

26
mindspore/lite/src/extendrt/cxx_api/file_utils.h Normal file
View File

@ -0,0 +1,26 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_SRC_EXTENDRT_CXX_API_FILE_UTILS_H_
#define MINDSPORE_LITE_SRC_EXTENDRT_CXX_API_FILE_UTILS_H_
#include <string>
#include "include/api/types.h"
namespace mindspore {
Buffer ReadFile(const std::string &file);
} // namespace mindspore
#endif // MINDSPORE_LITE_SRC_EXTENDRT_CXX_API_FILE_UTILS_H_

21
mindspore/lite/src/extendrt/cxx_api/model/model_impl.cc
View File

@ -18,6 +18,7 @@
#include "extendrt/cxx_api/model/model_impl.h"
#include "extendrt/cxx_api/dlutils.h"
#include "extendrt/cxx_api/file_utils.h"
#include "extendrt/utils/tensor_utils.h"
namespace mindspore {
@ -29,20 +30,19 @@ Status ModelImpl::Build(const void *model_data, size_t data_size, ModelType mode
return ret;
}
session_ = InferSession::CreateSession(model_context);
if (session_ == nullptr) {
return kLiteNullptr;
}
session_->Init(model_context);
return session_->CompileGraph(graph_->graph_data_->GetFuncGraph());
return session_->CompileGraph(graph_->graph_data_->GetFuncGraph(), model_data, data_size);
}
Status ModelImpl::Build(const std::string &model_path, ModelType model_type,
const std::shared_ptr<Context> &model_context) {
graph_ = std::make_shared<Graph>();
auto ret = Serialization::Load(model_path, model_type, graph_.get());
if (ret != kSuccess) {
return ret;
}
session_ = InferSession::CreateSession(model_context);
session_->Init(model_context);
return session_->CompileGraph(graph_->graph_data_->GetFuncGraph());
auto buffer = ReadFile(model_path);
return this->Build(buffer.Data(), buffer.DataSize(), model_type, model_context);
}
Status ModelImpl::Resize(const std::vector<MSTensor> &inputs, const std::vector<std::vector<int64_t>> &dims) {
return kSuccess;
}
@ -126,9 +126,6 @@ Status ModelImpl::Predict(const std::vector<MSTensor> &inputs, std::vector<MSTen
}
auto ms_outputs = TensorUtils::TensorPtrToMSTensor(graph_outputs, session_->GetOutputNames());
(void)std::copy(ms_outputs.begin(), ms_outputs.end(), std::back_inserter(*outputs));
// for (auto ms_output : ms_outputs) {
// outputs->push_back(ms_output);
// }
return kSuccess;
}

3
mindspore/lite/src/extendrt/cxx_api/serialization.cc
View File

@ -23,6 +23,7 @@
#include "extendrt/cxx_api/dlutils.h"
#endif
#include "utils/crypto.h"
#include "extendrt/cxx_api/file_utils.h"
namespace mindspore {
static Status RealPath(const std::string &file, std::string *realpath_str) {
@ -40,7 +41,7 @@ static Status RealPath(const std::string &file, std::string *realpath_str) {
return kSuccess;
}
static Buffer ReadFile(const std::string &file) {
Buffer ReadFile(const std::string &file) {
Buffer buffer;
if (file.empty()) {
MS_LOG(ERROR) << "Pointer file is nullptr";

9
mindspore/lite/src/extendrt/infer_session.cc
View File

@ -33,13 +33,14 @@ static const std::vector<PrimitivePtr> ms_infer_cut_list = {prim::kPrimReturn,
prim::kPrimSwitch, prim::kPrimMakeTuple,
prim::kPrimBpropCut, prim::kPrimSwitchLayer};
static bool is_infer_single_op = true;
static bool is_use_lite_session = false;
class DefaultInferSession : public InferSession {
public:
DefaultInferSession() = default;
virtual ~DefaultInferSession() = default;
Status Init(const std::shared_ptr<Context> context) override;
Status CompileGraph(FuncGraphPtr graph) override;
Status CompileGraph(FuncGraphPtr graph, const void *data = nullptr, size_t size = 0) override;
Status RunGraph() override;
Status RunGraph(const std::vector<tensor::TensorPtr> &inputs, std::vector<tensor::TensorPtr> *outputs) override;
Status Resize(const std::vector<tensor::TensorPtr> &inputs, const std::vector<std::vector<int64_t>> &dims) override;
@ -63,7 +64,7 @@ Status DefaultInferSession::Init(const std::shared_ptr<Context> context) {
partition_ = std::make_shared<compile::GraphPartition>(ms_infer_cut_list, "ms");
return kSuccess;
}
Status DefaultInferSession::CompileGraph(FuncGraphPtr graph) {
Status DefaultInferSession::CompileGraph(FuncGraphPtr graph, const void *data, size_t size) {
MS_LOG(INFO) << "DefaultInferSession::CompileGraph";
return kSuccess;
}
@ -117,6 +118,10 @@ SessionConfig InferSession::SelectSessionArg(const std::shared_ptr<Context> cont
config.type_ = kSingleOpSession;
return config;
}
if (is_use_lite_session) {
config.type_ = kLiteInferSession;
return config;
}
config.type_ = kDefaultSession;
return config;
}

2
mindspore/lite/src/extendrt/infer_session.h
View File

@ -35,7 +35,7 @@ class InferSession : public std::enable_shared_from_this<InferSession> {
static std::shared_ptr<InferSession> CreateSession(const std::shared_ptr<Context> context);
static SessionConfig SelectSessionArg(const std::shared_ptr<Context> context);
virtual Status Init(const std::shared_ptr<Context> context) = 0;
virtual Status CompileGraph(FuncGraphPtr graph) = 0;
virtual Status CompileGraph(FuncGraphPtr graph, const void *data = nullptr, size_t size = 0) = 0;
virtual Status RunGraph() = 0;
virtual Status RunGraph(const std::vector<tensor::TensorPtr> &inputs, std::vector<tensor::TensorPtr> *outputs) = 0;
virtual Status Resize(const std::vector<tensor::TensorPtr> &inputs,

35
mindspore/lite/src/extendrt/mindir_loader/mindir_model/mindir_model.cc
View File

@ -25,6 +25,9 @@
#include "extendrt/mindir_loader/mindir_model/kernel_mod_util.h"
#include "src/runtime/kernel_exec.h"
#include "extendrt/mindir_loader/mindir_model/inner_kernel.h"
#include "extendrt/mock/lite_runtime/populate/base_operator_populate_register.h"
#include "src/runtime/kernel_registry.h"
namespace mindspore::infer::mindir {
#define IS_LITTLE_ENDIAN (uint8_t)1U
@ -77,6 +80,9 @@ mindspore::kernel::KernelExec *MindirModel::FindBackendKernel(const std::vector<
const std::vector<mindspore::lite::Tensor *> &out_tensors,
const LiteGraph::Node *node, lite::Context *context,
TypeId prefer_data_type) {
if (select_lite_kernel_) {
return FindLiteKernel(in_tensors, out_tensors, node, context, prefer_data_type);
}
std::shared_ptr<kernel::InnerKernel> inner_kernel =
mindspore::kernel::KernelModUtil::GetInnerKernel(in_tensors, out_tensors, node, context);
kernel::KernelExec *kernel_exec = new kernel::KernelExec(inner_kernel);
@ -86,6 +92,35 @@ mindspore::kernel::KernelExec *MindirModel::FindBackendKernel(const std::vector<
return kernel_exec;
}
mindspore::kernel::KernelExec *MindirModel::FindLiteKernel(const std::vector<mindspore::lite::Tensor *> &in_tensors,
const std::vector<mindspore::lite::Tensor *> &out_tensors,
const LiteGraph::Node *node, lite::Context *context,
TypeId prefer_data_type) {
mindspore::kernel::KernelExec *kernel_exec = nullptr;
auto op_type_str = node->op_type_;
auto op_type = BaseOperatorPopulateRegistry::GetInstance()->TypeStrToType(op_type_str);
auto parame_gen = BaseOperatorPopulateRegistry::GetInstance()->GetParameterCreator(op_type);
if (parame_gen == nullptr) {
MS_LOG(ERROR) << "parameter generator is nullptr.";
return nullptr;
}
OpParameter *op_parameter = parame_gen(node->base_operator_.get());
kernel::KernelKey desc{kernel::KERNEL_ARCH::kCPU, kNumberTypeInt32, NHWC, op_type, "", kernel::kBuiltin};
auto inner_context = static_cast<mindspore::lite::InnerContext *>(context);
// mindspore::lite::InnerContext *inner_context = new (std::nothrow) mindspore::lite::InnerContext(context);
if (inner_context == nullptr) {
MS_LOG(ERROR) << "new inner context failed";
return nullptr;
}
auto ret = lite::KernelRegistry::GetInstance()->GetKernelExec(in_tensors, out_tensors, inner_context, nullptr, desc,
op_parameter, &kernel_exec, node->primitive_);
if (ret != lite::RET_OK || kernel_exec == nullptr) {
MS_LOG(ERROR) << "find lite kernel failed with code " << ret;
return nullptr;
}
return kernel_exec;
}
mindspore::lite::Tensor *MindirModel::ConvertTensor(TensorProtoWrap mindir_tensor_wrap) {
auto mindir_tensor = mindir_tensor_wrap.tensor_proto();
auto data_type = MindirModelUtil::ProtoTypeToTypeId(mindir_tensor.data_type());

5
mindspore/lite/src/extendrt/mindir_loader/mindir_model/mindir_model.h
View File

@ -63,6 +63,10 @@ class MindirModel : public AbstractBaseModel {
mindspore::lite::Tensor *ConvertTensor(TensorProtoWrap mindir_tensor);
int LoadTensorData(mindspore::lite::Tensor *lite_tensor, const mind_ir::TensorProto &mindir_tensor);
int CheckTensorValid(lite::Tensor *dst_tensor);
mindspore::kernel::KernelExec *FindLiteKernel(const std::vector<mindspore::lite::Tensor *> &in_tensors,
const std::vector<mindspore::lite::Tensor *> &out_tensors,
const LiteGraph::Node *node, lite::Context *context,
TypeId prefer_data_type);
public:
std::vector<TensorProtoWrap> all_mindir_tensors_;
@ -70,6 +74,7 @@ class MindirModel : public AbstractBaseModel {
private:
std::string model_path_;
bool select_lite_kernel_ = true;
};
} // namespace mindspore::infer::mindir

363
mindspore/lite/src/extendrt/mock/lite_runtime/CMakeLists.txt Normal file
View File

@ -0,0 +1,363 @@
if(NOT MSLITE_ENABLE_RUNTIME_CONVERT)
set(API_SRC ${API_SRC} ${CORE_DIR}/utils/status.cc)
set(LITE_SRC ${LITE_SRC} ${LITE_DIR}/src/common/config_file.cc)
endif()
set(API_SRC "")
set(LITE_SRC "")
set(MOCK_SRC
${MOCK_SRC}
${LITE_DIR}/src/extendrt/mock/lite_runtime/converters.cc
${LITE_DIR}/src/extendrt/mock/lite_runtime/populate/base_operator_populate_register.cc
${LITE_DIR}/src/extendrt/mock/lite_runtime/populate/arithmetic_populate.cc
)
set(LITE_SRC
${LITE_SRC}
${API_SRC}
${MOCK_SRC}
${LITE_DIR}/src/common/context_util.cc
${LITE_DIR}/src/common/file_utils.cc
${LITE_DIR}/src/common/utils.cc
${LITE_DIR}/src/common/graph_util.cc
${LITE_DIR}/src/common/log.cc
${LITE_DIR}/src/common/prim_util.cc
${LITE_DIR}/src/common/tensor_util.cc
${LITE_DIR}/src/runtime/allocator.cc
${LITE_DIR}/src/runtime/inner_allocator.cc
${LITE_DIR}/src/runtime/runtime_allocator.cc
${LITE_DIR}/src/runtime/infer_manager.cc
${LITE_DIR}/src/runtime/runtime_shape_fusion_pass.cc
${LITE_DIR}/src/runtime/runtime_pass.cc
${LITE_DIR}/src/runtime/pass/runtime_ncx_pass.cc
${LITE_DIR}/src/runtime/schema_tensor_wrapper.cc
${LITE_DIR}/src/tensor.cc
${LITE_DIR}/src/tensorlist.cc
${LITE_DIR}/src/runtime/tensor_category.cc
${LITE_DIR}/src/runtime/executor.cc
${LITE_DIR}/src/runtime/inner_context.cc
${LITE_DIR}/src/runtime/lite_model.cc
${LITE_DIR}/src/runtime/kernel_registry.cc
${LITE_DIR}/src/runtime/lite_kernel.cc
${LITE_DIR}/src/runtime/kernel_exec.cc
${LITE_DIR}/src/runtime/kernel_exec_util.cc
${LITE_DIR}/src/runtime/sub_graph_kernel.cc
${LITE_DIR}/src/runtime/scheduler.cc
${LITE_DIR}/src/runtime/lite_session.cc
${LITE_DIR}/src/errorcode.cc
${LITE_DIR}/src/runtime/cpu_info.cc
${LITE_DIR}/src/runtime/pack_weight_manager.cc
${LITE_DIR}/src/control_flow/control_flow_scheduler.cc
${LITE_DIR}/src/control_flow/control_subgraph_creator.cc
)
set(LITE_SRC
${LITE_SRC}
${MODEL_LOADER_FRAMEWORK_SRC}
)
if(MSLITE_ENABLE_MODEL_ENCRYPTION)
set(LITE_SRC
${LITE_SRC}
${LITE_DIR}/src/common/decrypt.cc
${LITE_DIR}/src/common/dynamic_library_loader.cc
)
endif()
if(MSLITE_ENABLE_BFC_MEMORY)
set(LITE_SRC
${LITE_SRC}
${LITE_DIR}/src/runtime/dynamic_mem_allocator.cc
${LITE_DIR}/src/runtime/dynamic_mem_manager.cc
${LITE_DIR}/src/runtime/numa_adapter.cc
)
endif()
if(MSLITE_ENABLE_SHARING_MODEL_WEIGHT)
set(LITE_SRC
${LITE_SRC}
${LITE_DIR}/src/runtime/pack_weight.cc
)
endif()
if(MSLITE_ENABLE_DYNAMIC_THREAD_DISTRIBUTE)
set(LITE_SRC
${LITE_SRC}
${LITE_DIR}/src/runtime/thread_cost_model.cc
)
endif()
if(MSLITE_ENABLE_CONTROLFLOW)
file(GLOB CONTROL_FLOW_KERNEL_SRC
${LITE_DIR}/src/control_flow/kernel/*.cc
)
set(LITE_SRC ${LITE_SRC} ${CONTROL_FLOW_KERNEL_SRC})
endif()
if(MSLITE_ENABLE_EXPERIMENTAL_KERNEL)
file(GLOB EXPERIMENT_SRC
${LITE_DIR}/src/../experimental/src/exec_env_utils.cc
${LITE_DIR}/src/../experimental/kernel/*.cc)
set(LITE_SRC ${LITE_SRC} ${EXPERIMENT_SRC})
endif()
if(MSLITE_ENABLE_RUNTIME_GLOG)
if(NOT MSLITE_ENABLE_RUNTIME_CONVERT AND NOT MSLITE_ENABLE_KERNEL_EXECUTOR)
set(LITE_SRC ${LITE_SRC}
${CORE_DIR}/utils/log_adapter.cc)
endif()
endif()
if(MSLITE_ENABLE_RUNTIME_CONVERT)
file(GLOB RUNTIME_CONVERT_SRC
${LITE_DIR}/src/common/ops/ops_def.cc
${LITE_DIR}/src/common/ops/ops_utils.cc
${LITE_DIR}/src/common/ops/anf_utils.cc
${LITE_DIR}/src/runtime/runtime_convert.cc)
set(LITE_SRC ${LITE_SRC} ${RUNTIME_CONVERT_SRC})
endif()
if(MSLITE_ENABLE_STRING_KERNEL)
set(LITE_SRC
${LITE_SRC}
${LITE_DIR}/src/common/string_util.cc
)
endif()
if(ENABLE_RUNTIME_NCX_PASS)
file(GLOB RUNTIME_NCX_PASS_SRC
${LITE_DIR}/src/runtime/pass/decrease_transpose_algo.cc
${LITE_DIR}/src/runtime/pass/delete_isolated_kernel.cc
${LITE_DIR}/src/runtime/pass/infershape_pass.cc
${LITE_DIR}/src/runtime/pass/pass_utils.cc
${LITE_DIR}/src/runtime/pass/runtime_optimizer.cc
${LITE_DIR}/src/runtime/pass/to_nchw_format.cc
${LITE_DIR}/src/runtime/pass/transpose_strategy.cc
)
set(LITE_SRC ${LITE_SRC} ${RUNTIME_NCX_PASS_SRC})
endif()
if(MSLITE_ENABLE_WEIGHT_DECODE)
set(LITE_SRC ${LITE_SRC}
${LITE_DIR}/src/runtime/huffman_decode.cc)
if(NOT MSLITE_ENABLE_RUNTIME_CONVERT)
set(LITE_SRC ${LITE_SRC}
${TOOLS_DIR}/converter/quantizer/fse_decoder.cc
${TOOLS_DIR}/converter/quantizer/fse_bit_stream.cc)
endif()
endif()
if(MSLITE_ENABLE_AUTO_PARALLEL)
set(LITE_SRC
${LITE_SRC}
${LITE_DIR}/src/runtime/sub_graph_split.cc
)
endif()
if(MSLITE_ENABLE_CUSTOM_KERNEL)
file(GLOB KERNEL_REG_SRC ${LITE_DIR}/src/registry/*.cc)
endif()
set(LITE_SRC
${LITE_SRC}
${KERNEL_REG_SRC}
${LITE_DIR}/src/runtime/weight_decoder.cc
)
if(MSLITE_GPU_BACKEND STREQUAL opencl)
file(GLOB_RECURSE OPENCL_RUNTIME_SRC
${LITE_DIR}/src/runtime/kernel/gpu/opencl/*.cc
)
set(LITE_SRC
${LITE_SRC}
${OPENCL_RUNTIME_SRC}
)
endif()
if(MSLITE_GPU_BACKEND STREQUAL cuda)
file(GLOB CUDA_RUNTIME_SRC
${LITE_DIR}/src/runtime/gpu/*.cc
${LITE_DIR}/src/runtime/gpu/cuda/*.cc
)
set(LITE_SRC
${LITE_SRC}
${CUDA_RUNTIME_SRC}
)
endif()
if(MSLITE_MINDDATA_IMPLEMENT STREQUAL "full")
set(TRAIN_SRC_WITH_MD
${LITE_DIR}/src/train/train_loop.cc
)
endif()
file(GLOB CXX_API_EXPRESSION
${LITE_DIR}/src/runtime/cxx_api/expression/*.cc
)
file(GLOB EXPRESSION_OPS
${LITE_DIR}/src/expression/ops/*.cc
)
set(EXPRESSION_SRC
${CXX_API_EXPRESSION}
${LITE_DIR}/src/expression/export.cc
${LITE_DIR}/src/expression/expr.cc
${LITE_DIR}/src/expression/import.cc
${LITE_DIR}/src/expression/net.cc
${LITE_DIR}/src/expression/node.cc
${LITE_DIR}/src/expression/ops.cc
${LITE_DIR}/src/expression/ops_utils.cc
${LITE_DIR}/src/expression/param.cc
${LITE_DIR}/src/expression/sequential.cc
${EXPRESSION_OPS}
)
set(TRAIN_SRC
${API_TRAIN_SRC}
${TRAIN_SRC_WITH_MD}
${EXPRESSION_SRC}
${LITE_DIR}/src/common/quant_utils.cc
${LITE_DIR}/src/runtime/cxx_api/metrics/accuracy.cc
${LITE_DIR}/src/runtime/cxx_api/train/model_build.cc
${LITE_DIR}/src/runtime/cxx_api/train/model_build_impl.cc
${LITE_DIR}/src/runtime/cxx_api/train/converters.cc
${LITE_DIR}/src/runtime/cxx_api/train/train_support.cc
${LITE_DIR}/src/train/train_populate_parameter.cc
${LITE_DIR}/src/train/train_session.cc
${LITE_DIR}/src/train/graph_fusion.cc
${LITE_DIR}/src/train/graph_dropout.cc
${LITE_DIR}/src/train/transfer_session.cc
${LITE_DIR}/src/train/train_utils.cc
${LITE_DIR}/src/train/loss_monitor.cc
${LITE_DIR}/src/train/lr_scheduler.cc
${LITE_DIR}/src/train/accuracy_metrics.cc
${LITE_DIR}/src/train/accuracy_monitor.cc
${LITE_DIR}/src/train/classification_train_accuracy_monitor.cc
${LITE_DIR}/src/train/train_export.cc
${LITE_DIR}/src/train/opt_allocator.cc
${LITE_DIR}/src/common/storage.cc
${TOOLS_DIR}/converter/optimizer.cc
${TOOLS_DIR}/converter/legacy_optimizer/fusion/fusion_pass.cc
${TOOLS_DIR}/converter/legacy_optimizer/fusion/fusion_pattern.cc
${TOOLS_DIR}/common/meta_graph_utils.cc
${TOOLS_DIR}/converter/legacy_optimizer/fusion/matmul_biasadd_fusion_pass.cc
${TOOLS_DIR}/converter/legacy_optimizer/graph/dropout_node_remove_pass.cc
${TOOLS_DIR}/converter/legacy_optimizer/graph/isolated_node_remove_pass.cc
${TOOLS_DIR}/converter/legacy_optimizer/graph/subgraph_node_pass.cc
)
if(MSLITE_ENABLE_MINDRT)
set(LITE_SRC ${LITE_SRC}
${LITE_DIR}/src/runtime/lite_mindrt.cc
${LITE_DIR}/src/runtime/mindrt_executor.cc
${LITE_DIR}/src/control_flow/control_actor_creator.cc
)
if(MSLITE_ENABLE_CONTROLFLOW)
file(GLOB CONTROL_FLOW_ACTOR_SRC
${LITE_DIR}/src/control_flow/actor/*.cc
)
set(LITE_SRC ${LITE_SRC} ${CONTROL_FLOW_ACTOR_SRC})
endif()
else()
set(LITE_SRC ${LITE_SRC}
${CORE_DIR}/mindrt/src/thread/core_affinity.cc
${CORE_DIR}/mindrt/src/thread/threadpool.cc
)
endif()
if(MSLITE_ENABLE_GRAPH_KERNEL)
file(GLOB_RECURSE GRAPH_KERNEL_SRC
${TOOLS_DIR}/graph_kernel/common/*.cc
${TOOLS_DIR}/graph_kernel/runtime/*.cc
)
set(LITE_SRC ${LITE_SRC} ${GRAPH_KERNEL_SRC})
endif()
if(NOT MSLITE_ENABLE_COREML)
set(LITE_SRC ${LITE_SRC}
${LITE_DIR}/src/runtime/delegate/coreml/stub/coreml_delegate_stub.cc)
endif()
add_library(infer_lite_src_mid OBJECT ${LITE_SRC})
add_dependencies(infer_lite_src_mid fbs_src)
add_library(mindspore-infer-lite SHARED $<TARGET_OBJECTS:infer_lite_src_mid>)
if(MSVC)
set_target_properties(mindspore-infer-lite PROPERTIES PREFIX lib)
set_target_properties(mindspore-infer-lite PROPERTIES IMPORT_PREFIX lib)
set_target_properties(mindspore-infer-lite PROPERTIES IMPORT_SUFFIX .dll.lib)
endif()
target_link_libraries(mindspore-infer-lite cpu_kernel_mid nnacl_mid cpu_ops_mid)
if(SUPPORT_TRAIN)
target_link_libraries(mindspore-infer-lite train_cpu_kernel_mid)
endif()
if(MSLITE_ENABLE_MINDRT)
target_link_libraries(mindspore-infer-lite mindrt_mid)
endif()
if(SUPPORT_TENSORRT)
target_link_libraries(mindspore-infer-lite tensorrt_kernel_mid cuda_kernel_mid gpu_distribution_collective)
else()
target_link_libraries(mindspore-infer-lite tensorrt_stub)
endif()
if(MSLITE_GPU_BACKEND STREQUAL opencl)
target_link_libraries(mindspore-infer-lite opencl_kernel_mid)
endif()
if(SUPPORT_NPU)
target_link_libraries(mindspore-infer-lite npu_kernel_mid)
endif()
if(PLATFORM_ARM32 OR PLATFORM_ARM64 AND NOT TARGET_HIMIX
AND NOT TARGET_MIX210 AND NOT TARGET_OHOS_LITE AND NOT MACHINE_LINUX_ARM64)
target_link_libraries(mindspore-infer-lite log)
endif()
if(MSLITE_MINDDATA_IMPLEMENT STREQUAL "lite")
target_link_libraries(mindspore-infer-lite minddata_eager_mid minddata-lite)
endif()
########################## build optimize and float16 library #################################
if(PLATFORM_ARM)
if(PLATFORM_ARM64 AND NOT TARGET_HIMIX AND NOT TARGET_MIX210 AND NOT MACHINE_LINUX_ARM64)
target_link_libraries(mindspore-infer-lite cpu_opt_kernel_mid nnacl_optimize_mid)
endif()
if(MSLITE_ENABLE_FP16)
target_link_libraries(mindspore-infer-lite cpu_fp16_kernel_mid nnacl_fp16_mid)
endif()
endif()
if(MSLITE_ENABLE_RUNTIME_CONVERT)
target_link_libraries(mindspore-infer-lite
quantizer_mid fusion_mid proto_mid graph_pass_mid preprocess_mid
cpu_kernel_mid ccsrc_src_mid converter_src_mid lite_exporter_mid
config_parser_mid mslite_converter_plugin mindspore_core coder_mid
mindir_serializer_mid mindspore::protobuf ${SECUREC_LIBRARY})
target_link_libraries(mindspore-infer-lite
ccsrc_debug_common_mid_ mindir_proto_mid _mindspore_transform_express_ir_obj)
if(MSLITE_ENABLE_ACL)
target_link_libraries(mindspore-infer-lite lite_acl_mid
mindspore_shared_lib)
endif()
endif()
if(SUPPORT_CUDA AND NOT MSLITE_ENABLE_CLOUD_FUSION_INFERENCE)
target_link_libraries(mindspore-infer-lite cuda_lite_kernel_mid cuda_ops mindspore_core)
endif()
if(MSLITE_ENABLE_RUNTIME_GLOG)
target_link_libraries(mindspore-infer-lite mindspore::glog)
endif()
if(NOT WIN32)
target_link_libraries(mindspore-infer-lite dl)
endif()
if(ENABLE_MODEL_OBF)
target_link_libraries(mindspore-infer-lite ${OBF_LIB_DIR}/libmsdeobfuscator-lite.so)
endif()

135
mindspore/lite/src/extendrt/mock/lite_runtime/converters.cc Normal file
View File

@ -0,0 +1,135 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "extendrt/mock/lite_runtime/converters.h"
#include "src/common/log_adapter.h"
#include "src/common/utils.h"
namespace mindspore {
constexpr static int kMaxNumOfDevices = 3;
constexpr static int kDefaultThreadNumTwo = 2;
constexpr static int kDefaultThreadNumFour = 4;
constexpr static int kCoreNumThreshold = 32;
void ContextUtils::SetContextAttr(int32_t thread_num, int32_t inter_op_parallel_num, bool enable_parallel,
const std::vector<int32_t> &affinity_core_list,
const std::shared_ptr<Delegate> &delegate, lite::InnerContext *inner_context,
bool float_mode) {
inner_context->thread_num_ = thread_num;
inner_context->inter_op_parallel_num_ = inter_op_parallel_num;
inner_context->enable_parallel_ = enable_parallel;
inner_context->affinity_core_list_ = affinity_core_list;
inner_context->delegate = delegate;
inner_context->float_mode = float_mode;
}
Status ContextUtils::AddCpuDevice(const std::shared_ptr<Allocator> &allocator, int affinity_mode, bool enable_fp16,
const std::string &provider, const std::string &provider_device,
lite::InnerContext *inner_context) {
inner_context->allocator = allocator;
if (!IsAffinityModeValid(affinity_mode)) {
MS_LOG(ERROR) << "Invalid affinity mode, only supports 0:no affinities, 1:big cores first, 2:little cores first.";
return kLiteInputParamInvalid;
}
lite::DeviceInfo device_info;
device_info.cpu_device_info_ = {enable_fp16, static_cast<lite::CpuBindMode>(affinity_mode)};
inner_context->device_list_.push_back({lite::DT_CPU, device_info, provider, provider_device, allocator});
return kSuccess;
}
Status ContextUtils::AddGpuDevice(bool enable_fp16, uint32_t device_id, int rank_id, int group_size,
bool enable_gl_texture, void *gl_context, void *gl_display,
const std::string &provider, const std::string &provider_device,
const std::shared_ptr<Allocator> &allocator, lite::InnerContext *inner_context) {
lite::DeviceInfo device_info;
device_info.gpu_device_info_ = {enable_fp16, device_id, rank_id, group_size,
enable_gl_texture, gl_context, gl_display};
inner_context->device_list_.push_back({lite::DT_GPU, device_info, provider, provider_device, allocator});
return kSuccess;
}
Status ContextUtils::AddNpuDevice(int frequency, lite::InnerContext *inner_context) {
lite::DeviceInfo device_info;
device_info.npu_device_info_ = {frequency};
inner_context->device_list_.push_back({lite::DT_NPU, device_info});
return kSuccess;
}
Status ContextUtils::AddAscendDevice(lite::InnerContext *inner_context, DeviceInfoContext *device) {
lite::DeviceInfo device_info;
auto ascend_context = device->Cast<AscendDeviceInfo>();
device_info.ascend_device_info_ = {ascend_context->GetDeviceID(), ascend_context->GetDynamicBatchSize(),
ascend_context->GetDynamicImageSize()};
inner_context->device_list_.push_back({lite::DT_ASCEND, device_info});
return kSuccess;
}
void ContextUtils::ResetDefaultThreadNum(Context *context) {
if (context->GetThreadNum() != 0) {
return;
}
MS_LOG(INFO) << "thread num is 0, will set the optimal number of threads";
#if defined(__ANDROID__) || defined(MS_COMPILE_IOS)
context->SetThreadNum(kDefaultThreadNumTwo);
MS_LOG(INFO) << "Set the number of threads to " << kDefaultThreadNumTwo;
return;
#endif
auto core_num = lite::GetCoreNum();
if (core_num <= kCoreNumThreshold) {
context->SetThreadNum(kDefaultThreadNumTwo);
MS_LOG(INFO) << "Set the number of threads to " << kDefaultThreadNumTwo;
} else {
context->SetThreadNum(kDefaultThreadNumFour);
MS_LOG(INFO) << "Set the number of threads to " << kDefaultThreadNumFour;
}
return;
}
lite::InnerContext *ContextUtils::Convert(Context *context) {
auto inner_context = std::make_unique<lite::InnerContext>();
if ((context == nullptr) || (inner_context == nullptr)) {
MS_LOG(ERROR) << "Invalid context pointers.";
return nullptr;
}
ResetDefaultThreadNum(context);
auto device_list = context->MutableDeviceInfo();
if (device_list.size() == 0 || device_list.size() > kMaxNumOfDevices) {
MS_LOG(ERROR) << "Device num, support min: 1, max: " << kMaxNumOfDevices;
return nullptr;
}
SetContextAttr(context->GetThreadNum(), context->GetInterOpParallelNum(), context->GetEnableParallel(),
context->GetThreadAffinityCoreList(), context->GetDelegate(), inner_context.get(),
context->GetMultiModalHW());
inner_context->device_list_.clear();
Status ret = kLiteError;
for (auto &device : device_list) {
MS_CHECK_TRUE_RET(device != nullptr, nullptr);
if (device->GetDeviceType() == kCPU) {
auto cpu_context = device->Cast<CPUDeviceInfo>();
if (cpu_context->GetAllocator() == nullptr) {
cpu_context->SetAllocator(Allocator::Create());
}
ret = AddCpuDevice(cpu_context->GetAllocator(), context->GetThreadAffinityMode(), cpu_context->GetEnableFP16(),
cpu_context->GetProvider(), cpu_context->GetProviderDevice(), inner_context.get());
}
if (ret != kSuccess) {
MS_LOG(ERROR) << "Add device failed!";
return nullptr;
}
}
return inner_context.release();
}
} // namespace mindspore

65
mindspore/lite/src/extendrt/mock/lite_runtime/converters.h Normal file
View File

@ -0,0 +1,65 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_SRC_EXTENDRT_CXX_API_CONVERTERS_H_
#define MINDSPORE_LITE_SRC_EXTENDRT_CXX_API_CONVERTERS_H_
#include <vector>
#include <string>
#include <memory>
#include "include/api/context.h"
#include "include/api/status.h"
#include "include/api/cfg.h"
#include "include/train/train_cfg.h"
#include "src/runtime/inner_context.h"
namespace mindspore {
class ContextUtils {
public:
static lite::InnerContext *Convert(Context *context);
private:
static void SetContextAttr(int32_t thread_num, int32_t inter_op_parallel_num, bool enable_parallel,
const std::vector<int32_t> &affinity_core_list, const std::shared_ptr<Delegate> &delegate,
lite::InnerContext *inner_context, bool float_mode = false);
static Status AddCpuDevice(const std::shared_ptr<Allocator> &allocator, int affinity_mode, bool enable_fp16,
const std::string &provider, const std::string &provider_device,
lite::InnerContext *inner_context);
static Status AddGpuDevice(bool enable_fp16, uint32_t device_id, int rank_id, int group_size, bool enable_gl_texture,
void *gl_context, void *gl_display, const std::string &provider,
const std::string &provider_device, const std::shared_ptr<Allocator> &allocator,
lite::InnerContext *inner_context);
static Status AddNpuDevice(int frequency, lite::InnerContext *inner_context);
static Status AddAscendDevice(lite::InnerContext *inner_context, DeviceInfoContext *device);
static bool IsAffinityModeValid(int affinity_mode) {
return affinity_mode >= lite::NO_BIND && affinity_mode <= lite::MID_CPU;
}
static void ResetDefaultThreadNum(Context *context);
};
inline lite::QuantizationType A2L_ConvertQT(mindspore::QuantizationType qt) {
if (qt == kNoQuant) {
return lite::QT_NONE;
}
if (qt == kWeightQuant) {
return lite::QT_WEIGHT;
}
return lite::QT_DEFAULT;
}
Status A2L_ConvertConfig(const TrainCfg *a_train_cfg, lite::TrainCfg *l_train_cfg);
} // namespace mindspore
#endif // MINDSPORE_LITE_SRC_EXTENDRT_CXX_API_CONVERTERS_H_

92
mindspore/lite/src/extendrt/mock/lite_runtime/populate/arithmetic_populate.cc Normal file
View File

@ -0,0 +1,92 @@
/**
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "extendrt/mock/lite_runtime/populate/arithmetic_populate.h"
#include "extendrt/mock/lite_runtime/populate/base_operator_populate_register.h"
#include "ops/base_operator.h"
#include "ops/primitive_c.h"
using mindspore::schema::PrimitiveType_BiasAddGrad;
using mindspore::schema::PrimitiveType_Equal;
using mindspore::schema::PrimitiveType_FloorDiv;
using mindspore::schema::PrimitiveType_FloorMod;
using mindspore::schema::PrimitiveType_Greater;
using mindspore::schema::PrimitiveType_GreaterEqual;
using mindspore::schema::PrimitiveType_Less;
using mindspore::schema::PrimitiveType_LessEqual;
using mindspore::schema::PrimitiveType_LogicalAnd;
using mindspore::schema::PrimitiveType_LogicalOr;
using mindspore::schema::PrimitiveType_Maximum;
using mindspore::schema::PrimitiveType_MaximumGrad;
using mindspore::schema::PrimitiveType_Minimum;
using mindspore::schema::PrimitiveType_MinimumGrad;
using mindspore::schema::PrimitiveType_Mod;
using mindspore::schema::PrimitiveType_NotEqual;
using mindspore::schema::PrimitiveType_RealDiv;
using mindspore::schema::PrimitiveType_SquaredDifference;
namespace mindspore {
ArithmeticParameter *PopulateArithmeticCommonPara(void *base_operator) {
MS_CHECK_TRUE_RET(base_operator != nullptr, nullptr);
auto base_operator_ptr = static_cast<ops::BaseOperator *>(base_operator);
if (base_operator_ptr == nullptr) {
MS_LOG(ERROR) << "cast to BaseOperator failed";
return nullptr;
}
auto *param = reinterpret_cast<ArithmeticParameter *>(malloc(sizeof(ArithmeticParameter)));
if (param == nullptr) {
MS_LOG(ERROR) << "malloc ArithmeticParameter failed.";
return nullptr;
}
memset(param, 0, sizeof(ArithmeticParameter));
auto prim_c = base_operator_ptr->GetPrim();
auto op_type_str = prim_c->instance_name();
auto type = BaseOperatorPopulateRegistry::GetInstance()->TypeStrToType(op_type_str);
param->op_parameter_.type_ = type;
param->broadcasting_ = false;
param->ndim_ = 0;
param->activation_type_ = 0;
return param;
}
OpParameter *PopulateArithmetic(void *base_operator) {
ArithmeticParameter *param = PopulateArithmeticCommonPara(base_operator);
if (param == nullptr) {
MS_LOG(ERROR) << "PopulateArithmeticCommonPara failed.";
return nullptr;
}
return reinterpret_cast<OpParameter *>(param);
}
REG_BASE_POPULATE(PrimitiveType_MinimumGrad, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_MaximumGrad, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_RealDiv, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_LogicalAnd, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_LogicalOr, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_Equal, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_NotEqual, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_Less, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_LessEqual, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_Greater, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_GreaterEqual, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_Maximum, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_Minimum, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_FloorDiv, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_FloorMod, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_Mod, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_SquaredDifference, PopulateArithmetic)
REG_BASE_POPULATE(PrimitiveType_BiasAddGrad, PopulateArithmetic)
} // namespace mindspore

25
mindspore/lite/src/extendrt/mock/lite_runtime/populate/arithmetic_populate.h Normal file
View File

@ -0,0 +1,25 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_SRC_COMMON_OPS_POPULATE_ARITHMETIC_POPULATE_H_
#define MINDSPORE_LITE_SRC_COMMON_OPS_POPULATE_ARITHMETIC_POPULATE_H_
#include "nnacl/arithmetic.h"
namespace mindspore {
ArithmeticParameter *PopulateArithmeticCommonPara(void *prim);
OpParameter *PopulateArithmetic(void *primitive);
} // namespace mindspore
#endif // MINDSPORE_LITE_SRC_COMMON_OPS_POPULATE_ARITHMETIC_POPULATE_H_

24
mindspore/lite/src/extendrt/mock/lite_runtime/populate/base_operator_populate_register.cc Normal file
View File

@ -0,0 +1,24 @@
/**
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "extendrt/mock/lite_runtime/populate/base_operator_populate_register.h"
namespace mindspore {
BaseOperatorPopulateRegistry *BaseOperatorPopulateRegistry::GetInstance() {
static BaseOperatorPopulateRegistry registry;
return &registry;
}
} // namespace mindspore

99
mindspore/lite/src/extendrt/mock/lite_runtime/populate/base_operator_populate_register.h Normal file
View File

@ -0,0 +1,99 @@
/**
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_SRC_COMMON_OPS_POPULATE_POPULATE_REGISTER_H_
#define MINDSPORE_LITE_SRC_COMMON_OPS_POPULATE_POPULATE_REGISTER_H_
#include <map>
#include <vector>
#include <string>
#include "schema/model_generated.h"
#include "nnacl/op_base.h"
#include "src/common/common.h"
#include "src/common/log_adapter.h"
#include "src/common/prim_util.h"
#include "src/common/version_manager.h"
#include "src/common/utils.h"
#include "src/common/log_util.h"
namespace mindspore {
constexpr int kOffsetTwo = 2;
constexpr int kOffsetThree = 3;
constexpr size_t kMinShapeSizeTwo = 2;
constexpr size_t kMinShapeSizeFour = 4;
typedef OpParameter *(*BaseOperator2Parameter)(void *base_operator);
static const std::vector<schema::PrimitiveType> string_op = {
schema::PrimitiveType_CustomExtractFeatures, schema::PrimitiveType_CustomNormalize,
schema::PrimitiveType_CustomPredict, schema::PrimitiveType_HashtableLookup,
schema::PrimitiveType_LshProjection, schema::PrimitiveType_SkipGram};
class BaseOperatorPopulateRegistry {
public:
static BaseOperatorPopulateRegistry *GetInstance();
void InsertParameterMap(int type, BaseOperator2Parameter creator, int version = lite::SCHEMA_CUR) {
parameters_[lite::GenPrimVersionKey(type, version)] = creator;
std::string str = schema::EnumNamePrimitiveType(static_cast<schema::PrimitiveType>(type));
str_to_type_map_[str] = type;
}
BaseOperator2Parameter GetParameterCreator(int type, int version = lite::SCHEMA_CUR) {
BaseOperator2Parameter param_creator = nullptr;
auto iter = parameters_.find(lite::GenPrimVersionKey(type, version));
if (iter == parameters_.end()) {
#ifdef STRING_KERNEL_CLIP
if (lite::IsContain(string_op, static_cast<schema::PrimitiveType>(type))) {
MS_LOG(ERROR) << unsupport_string_tensor_log;
return nullptr;
}
#endif
MS_LOG(ERROR) << "Unsupported parameter type in Create : "
<< schema::EnumNamePrimitiveType(static_cast<schema::PrimitiveType>(type));
return nullptr;
}
param_creator = iter->second;
return param_creator;
}
int TypeStrToType(const std::string &type_str) {
auto iter = str_to_type_map_.find(type_str);
if (iter == str_to_type_map_.end()) {
MS_LOG(ERROR) << "Unknown type string to type " << type_str;
return schema::PrimitiveType_NONE;
}
return iter->second;
}
protected:
// key:type * 1000 + schema_version
std::map<int, BaseOperator2Parameter> parameters_;
std::map<std::string, int> str_to_type_map_;
};
class BaseRegistry {
public:
BaseRegistry(int primitive_type, BaseOperator2Parameter creator, int version = lite::SCHEMA_CUR) noexcept {
BaseOperatorPopulateRegistry::GetInstance()->InsertParameterMap(primitive_type, creator, version);
}
~BaseRegistry() = default;
};
#define REG_BASE_POPULATE(primitive_type, creator) \
static BaseRegistry g_##primitive_type##base_populate(primitive_type, creator);
} // namespace mindspore
#endif // MINDSPORE_LITE_SRC_COMMON_OPS_POPULATE_POPULATE_REGISTER_H_

2
mindspore/lite/src/extendrt/session/delegate_session.cc
View File

@ -25,7 +25,7 @@
namespace mindspore {
Status DelegateSession::Init(const std::shared_ptr<Context> context) { return kSuccess; }
Status DelegateSession::CompileGraph(FuncGraphPtr graph) { return kSuccess; }
Status DelegateSession::CompileGraph(FuncGraphPtr graph, const void *data, size_t size) { return kSuccess; }
Status DelegateSession::RunGraph() { return kSuccess; }
Status DelegateSession::RunGraph(const std::vector<tensor::TensorPtr> &inputs,

2
mindspore/lite/src/extendrt/session/delegate_session.h
View File

@ -30,7 +30,7 @@ class DelegateSession : public InferSession {
virtual ~DelegateSession() = default;
Status Init(const std::shared_ptr<Context> context) override;
Status CompileGraph(FuncGraphPtr graph) override;
Status CompileGraph(FuncGraphPtr graph, const void *data = nullptr, size_t size = 0) override;
Status RunGraph() override;
Status RunGraph(const std::vector<tensor::TensorPtr> &inputs, std::vector<tensor::TensorPtr> *outputs) override;
Status Resize(const std::vector<tensor::TensorPtr> &inputs, const std::vector<std::vector<int64_t>> &dims) override;

3
mindspore/lite/src/extendrt/session/graph_executor_session.cc
View File

@ -28,7 +28,8 @@ Status GraphExecutorSession::Init(const std::shared_ptr<Context> context) {
kernel_graph_utils_ = std::make_shared<mindspore::KernelGraphUtils>();
return kSuccess;
}
Status GraphExecutorSession::CompileGraph(FuncGraphPtr graph) {
Status GraphExecutorSession::CompileGraph(FuncGraphPtr graph, const void *data, size_t size) {
MS_LOG(INFO) << "GraphExecutorSession::CompileGraph";
std::vector<KernelGraphPtr> all_out_graph;
kernel_graph_ = kernel_graph_utils_->ConstructKernelGraph(graph, &all_out_graph, mindspore::device::DeviceType::kCPU);

2
mindspore/lite/src/extendrt/session/graph_executor_session.h
View File

@ -34,7 +34,7 @@ class GraphExecutorSession : public DelegateSession {
virtual ~GraphExecutorSession() = default;
Status Init(const std::shared_ptr<Context> context) override;
Status CompileGraph(FuncGraphPtr graph) override;
Status CompileGraph(FuncGraphPtr graph, const void *data = nullptr, size_t size = 0) override;
Status RunGraph() override;
Status RunGraph(const std::vector<tensor::TensorPtr> &inputs, std::vector<tensor::TensorPtr> *outputs) override;
Status Resize(const std::vector<tensor::TensorPtr> &inputs, const std::vector<std::vector<int64_t>> &dims) override;

306
mindspore/lite/src/extendrt/session/lite_infer_session.cc Normal file
View File

@ -0,0 +1,306 @@
/**
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <algorithm>
#include <functional>
#include <string>
#include <vector>
#include "src/extendrt/session/lite_infer_session.h"
#include "extendrt/mock/lite_runtime/converters.h"
#include "extendrt/session/factory.h"
#include "extendrt/utils/runtime_utils.h"
#include "extendrt/utils/tensor_utils.h"
namespace mindspore {
const size_t tensor_max_size = 0x1000000;
Status LiteInferSession::Init(const std::shared_ptr<Context> context) {
MS_LOG(INFO) << "SingleOpInferSession::Init";
context_ = context;
lite_session_ = CreateLiteSession(ContextUtils::Convert(context_.get()));
MS_EXCEPTION_IF_NULL(lite_session_);
return kSuccess;
}
Status LiteInferSession::CompileGraph(FuncGraphPtr graph, const void *data, size_t size) {
// Lite infer session do not use graph, just use data and size
MS_LOG(INFO) << "LiteInferSession::CompileGraph";
MS_EXCEPTION_IF_NULL(data);
MS_EXCEPTION_IF_ZERO("size", size);
lite_session_ = CreateLiteSession(ContextUtils::Convert(context_.get()));
MS_EXCEPTION_IF_NULL(lite_session_);
auto ret = lite_session_->LoadModelAndCompileByBuf(static_cast<const char *>(data), kMindIR, size);
if (ret != RET_OK) {
MS_LOG(EXCEPTION) << "load model and compile failed";
}
return kSuccess;
}
void LiteInferSession::ResetTensorData(std::vector<void *> old_data, const std::vector<lite::Tensor *> &tensors) {
for (size_t j = 0; j < old_data.size(); j++) {
tensors.at(j)->set_data(old_data.at(j));
}
}
std::vector<MSTensor> LiteInferSession::GetLiteSessionOutputs() {
std::vector<MSTensor> empty;
if (lite_session_ == nullptr) {
MS_LOG(ERROR) << "Session is null.";
return empty;
}
std::vector<MSTensor> res;
auto names = lite_session_->GetOutputTensorNames();
if (names.empty()) {
MS_LOG(ERROR) << "The output tensor name of this model is null.";
return empty;
}
auto outputs = lite_session_->GetOutputs();
if (outputs.empty()) {
MS_LOG(ERROR) << "The outputs of model is null.";
return empty;
}
if (names.size() != outputs.size()) {
MS_LOG(ERROR) << "The size of outputs dose not match the size of names.";
return empty;
}
res.resize(names.size());
for (size_t i = 0; i < names.size(); i++) {
auto impl = std::make_shared<LiteTensorImpl>(outputs[names[i]]);
if (impl == nullptr || impl->lite_tensor() == nullptr) {
MS_LOG(ERROR) << "Create tensor failed.";
return empty;
}
auto tensor = MSTensor(impl);
if (tensor == nullptr) {
MS_LOG(ERROR) << "Create tensor failed.";
return empty;
}
res[i] = tensor;
}
return res;
}
std::vector<int32_t> LiteInferSession::TruncateShape(const std::vector<int64_t> &shape, TypeId type, size_t data_len,
bool verify_size) {
std::vector<int32_t> empty;
if (shape.empty()) {
return empty;
}
std::vector<int32_t> truncated_shape;
truncated_shape.resize(shape.size());
size_t element_size = lite::DataTypeSize(type);
for (size_t i = 0; i < shape.size(); i++) {
auto dim = shape[i];
if (dim < 0 || dim > INT_MAX || (dim != 0 && element_size > INT_MAX / static_cast<size_t>(dim))) {
MS_LOG(ERROR) << "Invalid shape!dim: " << dim << ", element_size: " << element_size;
return empty;
} else {
element_size *= static_cast<size_t>(dim);
truncated_shape[i] = static_cast<int32_t>(dim);
}
}
if (verify_size) {
if (element_size != data_len) {
MS_LOG(ERROR) << "Invalid data size!element_size: " << element_size << ", data_len: " << data_len;
return empty;
}
}
return truncated_shape;
}
Status LiteInferSession::RunGraph() {
auto ret = lite_session_->RunGraph();
return static_cast<StatusCode>(ret);
}
Status LiteInferSession::RunGraph(const std::vector<tensor::TensorPtr> &inputs,
std::vector<tensor::TensorPtr> *outputs) {
MS_LOG(INFO) << "SingleOpInferSession::RunGraph with input and outputs";
MS_EXCEPTION_IF_NULL(outputs);
MS_EXCEPTION_IF_NULL(lite_session_);
auto input_tensors = lite_session_->GetInputs();
if (input_tensors.empty()) {
MS_LOG(EXCEPTION) << "Failed to get input tensor.";
}
if (input_tensors.size() != inputs.size()) {
MS_LOG(EXCEPTION) << "Wrong input size.";
}
std::vector<void *> old_data;
for (size_t i = 0; i < inputs.size(); i++) {
auto input = input_tensors.at(i);
auto user_input = inputs.at(i);
if (user_input->data_type() != input->data_type()) {
ResetTensorData(old_data, input_tensors);
MS_LOG(EXCEPTION) << "Tensor " << user_input->id() << " has a different data type from input"
<< input->tensor_name() << ".";
}
if (user_input->data_c() == nullptr) {
ResetTensorData(old_data, input_tensors);
MS_LOG(EXCEPTION) << "Tensor " << user_input->id() << " has no data.";
}
old_data.push_back(input->data());
if (input->data_type() == kObjectTypeString) {
#ifndef STRING_KERNEL_CLIP
std::vector<int32_t> shape =
TruncateShape(user_input->shape_c(), input->data_type(), user_input->DataSize(), false);
if (shape.empty() && !(user_input->shape_c().empty())) {
ResetTensorData(old_data, input_tensors);
MS_LOG(EXCEPTION) << "Input dims of tensor " << user_input->id() << " is invalid.";
}
input->set_shape(shape);
input->set_data(user_input->data_c());
#else
MS_LOG(ERROR) << unsupport_string_tensor_log;
return kLiteError;
#endif
} else {
if (user_input->data_c() != input->data()) {
if (input->Size() != user_input->Size()) {
ResetTensorData(old_data, input_tensors);
#ifndef ENABLE_LITE_ACL
MS_LOG(EXCEPTION) << "Tensor " << user_input->id() << " has wrong data size.";
#else
MS_LOG(WARNING) << "Please check tensor " << user_input->id()
<< " has been modified data size by DVPP method.";
std::vector<int> truncate_shape = {static_cast<int>(user_input->DataSize())};
input->set_shape(truncate_shape);
#endif
}
input->set_data(user_input->data_c());
}
}
}
auto ret = RunGraph();
ResetTensorData(old_data, input_tensors);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Run graph failed.";
return ret;
}
MS_LOG(DEBUG) << "Run graph success.";
auto res = GetLiteSessionOutputs();
if (res.empty()) {
MS_LOG(DEBUG) << "Empty outputs.";
return kLiteError;
}
outputs->clear();
*outputs = TensorUtils::MSTensorToTensorPtr(res);
return kSuccess;
}
Status LiteInferSession::Resize(const std::vector<tensor::TensorPtr> &inputs,
const std::vector<std::vector<int64_t>> &dims) {
return kSuccess;
}
std::vector<tensor::TensorPtr> LiteInferSession::GetOutputs() {
auto outputs = lite_session_->GetOutputs();
std::vector<tensor::TensorPtr> output_tensors;
for (auto &iter : outputs) {
auto output = iter.second;
auto type_id = output->data_type();
auto shape = output->shape();
ShapeVector shape_vec;
std::transform(shape.begin(), shape.end(), std::back_inserter(shape_vec),
[](int s) { return static_cast<int64_t>(s); });
auto data = output->data();
auto data_size = output->Size();
auto tensor_ptr = std::make_shared<mindspore::tensor::Tensor>(type_id, shape_vec, data, data_size);
output_tensors.emplace_back(tensor_ptr);
}
return output_tensors;
}
std::vector<tensor::TensorPtr> LiteInferSession::GetInputs() {
auto inputs = lite_session_->GetInputs();
std::vector<tensor::TensorPtr> input_tensors;
for (auto &input : inputs) {
auto type_id = input->data_type();
auto shape = input->shape();
ShapeVector shape_vec;
std::transform(shape.begin(), shape.end(), std::back_inserter(shape_vec),
[](int s) { return static_cast<int64_t>(s); });
auto data = input->data();
auto data_size = input->Size();
auto tensor_ptr = std::make_shared<mindspore::tensor::Tensor>(type_id, shape_vec, data, data_size);
input_tensors.emplace_back(tensor_ptr);
}
return input_tensors;
}
std::vector<std::string> LiteInferSession::GetOutputNames() {
auto outputs = lite_session_->GetOutputs();
std::vector<std::string> output_names;
std::transform(outputs.begin(), outputs.end(), std::back_inserter(output_names),
[](auto iter) { return iter.first; });
return output_names;
}
std::vector<std::string> LiteInferSession::GetInputNames() { return ConvertToTensorNames(lite_session_->GetInputs()); }
tensor::TensorPtr LiteInferSession::GetOutputByTensorName(const std::string &tensorName) { return nullptr; }
tensor::TensorPtr LiteInferSession::GetInputByTensorName(const std::string &name) { return nullptr; }
std::shared_ptr<lite::LiteSession> LiteInferSession::CreateLiteSession(lite::InnerContext *context) {
auto session = std::make_shared<lite::LiteSession>();
if (session == nullptr) {
MS_LOG(ERROR) << "create session failed";
delete context;
return nullptr;
}
auto ret = session->Init(context);
if (ret != mindspore::lite::RET_OK) {
MS_LOG(ERROR) << "init session failed";
return nullptr;
}
return session;
}
std::vector<std::string> LiteInferSession::ConvertToTensorNames(
const std::vector<mindspore::lite::Tensor *> &lite_tensors) {
std::vector<std::string> tensor_names;
std::transform(lite_tensors.begin(), lite_tensors.end(), std::back_inserter(tensor_names),
[](mindspore::lite::Tensor *lite_tensor) {
MS_EXCEPTION_IF_NULL(lite_tensor);
return lite_tensor->tensor_name();
});
return tensor_names;
}
std::vector<tensor::TensorPtr> LiteInferSession::ConvertToTensors(
const std::vector<mindspore::lite::Tensor *> &lite_tensors) {
std::vector<tensor::TensorPtr> tensors;
for (auto lite_tensor : lite_tensors) {
auto type_id = lite_tensor->data_type();
auto shape = lite_tensor->shape();
ShapeVector shape_vec;
std::transform(shape.begin(), shape.end(), std::back_inserter(shape_vec),
[](int s) { return static_cast<int64_t>(s); });
auto data = lite_tensor->data();
auto data_size = lite_tensor->Size();
auto tensor_ptr = std::make_shared<mindspore::tensor::Tensor>(type_id, shape_vec, data, data_size);
tensors.emplace_back(tensor_ptr);
}
return tensors;
}
static std::shared_ptr<InferSession> LiteInferSessionCreator(const SessionConfig &config) {
return std::make_shared<LiteInferSession>(config.context_);
}
REG_SESSION(kLiteInferSession, LiteInferSessionCreator);
} // namespace mindspore

64
mindspore/lite/src/extendrt/session/lite_infer_session.h Normal file
View File

@ -0,0 +1,64 @@
/**
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_EXTENDRT_SINGLE_OP_SESSION_H_
#define MINDSPORE_LITE_EXTENDRT_SINGLE_OP_SESSION_H_
#include <string>
#include <memory>
#include <vector>
#include "src/extendrt/infer_session.h"
#include "runtime/lite_session.h"
namespace mindspore {
class LiteInferSession : public InferSession {
public:
LiteInferSession() = default;
explicit LiteInferSession(const std::shared_ptr<Context> context) : context_(context) {}
virtual ~LiteInferSession() = default;
Status Init(const std::shared_ptr<Context> context) override;
Status CompileGraph(FuncGraphPtr graph, const void *data = nullptr, size_t size = 0) override;
Status RunGraph() override;
Status RunGraph(const std::vector<tensor::TensorPtr> &inputs, std::vector<tensor::TensorPtr> *outputs) override;
Status Resize(const std::vector<tensor::TensorPtr> &inputs, const std::vector<std::vector<int64_t>> &dims) override;
std::vector<tensor::TensorPtr> GetOutputs() override;
std::vector<tensor::TensorPtr> GetInputs() override;
std::vector<std::string> GetOutputNames() override;
std::vector<std::string> GetInputNames() override;
tensor::TensorPtr GetOutputByTensorName(const std::string &tensorName) override;
tensor::TensorPtr GetInputByTensorName(const std::string &name) override;
private:
std::shared_ptr<lite::LiteSession> CreateLiteSession(lite::InnerContext *context);
std::vector<MSTensor> GetLiteSessionOutputs();
void ResetTensorData(std::vector<void *> old_data, const std::vector<lite::Tensor *> &tensors);
std::vector<int32_t> TruncateShape(const std::vector<int64_t> &shape, enum TypeId type, size_t data_len,
bool verify_size);
std::vector<std::string> ConvertToTensorNames(const std::vector<mindspore::lite::Tensor *> &lite_tensors);
std::vector<tensor::TensorPtr> ConvertToTensors(const std::vector<mindspore::lite::Tensor *> &lite_tensors);
private:
std::shared_ptr<lite::LiteSession> lite_session_;
std::shared_ptr<Context> context_;
std::vector<tensor::TensorPtr> inputs_;
std::vector<std::string> input_names_;
std::vector<tensor::TensorPtr> outputs_;
std::vector<std::string> output_names_;
};
} // namespace mindspore
#endif // MINDSPORE_LITE_EXTENDRT_SINGLE_OP_SESSION_H_

3
mindspore/lite/src/extendrt/session/type.h
View File

@ -22,10 +22,11 @@
#include "include/api/delegate.h"
namespace mindspore {
enum SessionType { kDefaultSession = 0, kSingleOpSession, kDelegateSession, kNoneSession };
enum SessionType { kDefaultSession = 0, kSingleOpSession, kLiteInferSession, kDelegateSession, kNoneSession };
struct SessionConfig {
SessionType type_;
const std::shared_ptr<Context> context_;
std::vector<std::shared_ptr<Delegate>> delegates_;
};
} // namespace mindspore

2
mindspore/lite/src/extendrt/single_op_session.cc
View File

@ -49,7 +49,7 @@ Status SingleOpInferSession::Init(const std::shared_ptr<Context> context) {
return kSuccess;
}
Status SingleOpInferSession::CompileGraph(FuncGraphPtr graph) {
Status SingleOpInferSession::CompileGraph(FuncGraphPtr graph, const void *data, size_t size) {
MS_LOG(INFO) << "SingleOpInferSession::CompileGraph";
std::vector<KernelGraphPtr> all_out_graph;
kernel_graph_ = kernel_graph_utils_->ConstructKernelGraph(graph, &all_out_graph, mindspore::device::DeviceType::kCPU);

3
mindspore/lite/src/extendrt/single_op_session.h
View File

@ -21,7 +21,6 @@
#include <vector>
#include "src/extendrt/infer_session.h"
// #include "backend/common/session/session_basic.h"
#include "extendrt/utils/kernel_graph_utils.h"
namespace mindspore {
@ -30,7 +29,7 @@ class SingleOpInferSession : public InferSession {
SingleOpInferSession() = default;
virtual ~SingleOpInferSession() = default;
Status Init(const std::shared_ptr<Context> context) override;
Status CompileGraph(FuncGraphPtr graph) override;
Status CompileGraph(FuncGraphPtr graph, const void *data = nullptr, size_t size = 0) override;
Status RunGraph() override;
Status RunGraph(const std::vector<tensor::TensorPtr> &inputs, std::vector<tensor::TensorPtr> *outputs) override;
Status Resize(const std::vector<tensor::TensorPtr> &inputs, const std::vector<std::vector<int64_t>> &dims) override;

5
mindspore/lite/src/extendrt/utils/cloud_infer_test.cc
View File

@ -34,8 +34,9 @@ int main(int argc, const char **argv) {
mindspore::Model ms_model;
auto context = std::make_shared<mindspore::Context>();
auto &device_info = context->MutableDeviceInfo();
auto device_context = std::make_shared<mindspore::GPUDeviceInfo>();
auto device_context = std::make_shared<mindspore::CPUDeviceInfo>();
device_context->SetProvider("tensorrt");
device_context->SetAllocator(nullptr);
device_info.emplace_back(device_context);
ms_model.Build(flags.model_file_, mindspore::kMindIR, context);
@ -62,7 +63,7 @@ int main(int argc, const char **argv) {
}
for (auto output : outputs) {
auto z = static_cast<int *>(output.MutableData());
auto z = static_cast<bool *>(output.MutableData());
for (int i = 0; i < test_input_shape; i++) {
std::cout << z[i] << " ";
}

1
mindspore/lite/src/extendrt/utils/kernel_graph_utils.cc
View File

@ -123,7 +123,6 @@ KernelGraphPtr KernelGraphUtils::ConstructKernelGraphFromNodeList(const AnfNodeP
}
graph->set_device_target(device_target);
// add a make_tuple at the end of graph as output
// graph->set_output(ConstructOutput(outputs, graph));
FuncGraphManagerPtr manager = MakeManager({graph});
if (manager) {
manager->AddFuncGraph(graph);

8
mindspore/lite/src/runtime/lite_session.cc
View File

@ -1652,10 +1652,10 @@ int lite::LiteSession::LoadModelAndCompileByBuf(const char *model_buf, mindspore
MS_CHECK_FALSE_MSG(status != RET_OK, RET_ERROR, "InitPackWeightByBuf failed.");
auto ret = CompileGraph(model);
model->buf = nullptr;
if (buf_model_type == mindspore::ModelType::kMindIR) {
delete[] lite_buf;
lite_buf = nullptr;
}
// if (buf_model_type == mindspore::ModelType::kMindIR) {
// delete[] lite_buf;
// lite_buf = nullptr;
// }
if (ret != lite::RET_OK) {
MS_LOG(ERROR) << "Compile model failed";
delete model;

1
mindspore/lite/tools/converter/CMakeLists.txt
View File

@ -155,6 +155,7 @@ set(LITE_SRC ${API_SRC}
${SRC_DIR}/runtime/pack_weight_manager.cc
${SRC_DIR}/runtime/huffman_decode.cc
${SRC_DIR}/extendrt/delegate/tensorrt/distribution/distribution_base.cc
${LITE_DIR}/src/extendrt/mock/lite_runtime/populate/base_operator_populate_register.cc
${SRC_DIR}/control_flow/control_flow_scheduler.cc
${SRC_DIR}/control_flow/control_subgraph_creator.cc
)