From 9d2434ad0684f7b3bb27fd72437c640a90d558aa Mon Sep 17 00:00:00 2001 From: liu lili Date: Thu, 1 Dec 2022 16:10:46 +0800 Subject: [PATCH] mindspore lite: change litert to plugin --- mindspore/lite/src/extendrt/CMakeLists.txt | 9 ++- .../model_pool/model_parallel_runner_impl.cc | 4 + .../graph_executor/litert/CMakeLists.txt | 2 +- .../litert/litert_plugin_impl.cc | 20 +++++ .../litert/litert_plugin_impl.h | 31 ++++++++ .../delegate/plugin/litert_executor_plugin.cc | 77 +++++++++++++++++++ .../delegate/plugin/litert_executor_plugin.h | 42 ++++++++++ .../tensorrt/tensorrt_graph_executor.cc | 3 +- .../src/extendrt/delegate_graph_executor.cc | 18 ++--- mindspore/lite/src/extendrt/infer_session.cc | 7 +- .../src/extendrt/utils/func_graph_utils.cc | 43 ++++++++++- .../src/extendrt/utils/func_graph_utils.h | 3 + 12 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 mindspore/lite/src/extendrt/delegate/graph_executor/litert/litert_plugin_impl.cc create mode 100644 mindspore/lite/src/extendrt/delegate/graph_executor/litert/litert_plugin_impl.h create mode 100644 mindspore/lite/src/extendrt/delegate/plugin/litert_executor_plugin.cc create mode 100644 mindspore/lite/src/extendrt/delegate/plugin/litert_executor_plugin.h diff --git a/mindspore/lite/src/extendrt/CMakeLists.txt b/mindspore/lite/src/extendrt/CMakeLists.txt index 855bbd9acd4..d237deaae07 100644 --- a/mindspore/lite/src/extendrt/CMakeLists.txt +++ b/mindspore/lite/src/extendrt/CMakeLists.txt @@ -40,6 +40,7 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE) ${CMAKE_CURRENT_SOURCE_DIR}/../common/file_utils.cc ${CMAKE_CURRENT_SOURCE_DIR}/../common/utils.cc ${CMAKE_CURRENT_SOURCE_DIR}/../common/graph_util.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../common/config_infos.cc ${CMAKE_CURRENT_SOURCE_DIR}/subgraph_kernel.cc ${CMAKE_CURRENT_SOURCE_DIR}/numa_adapter.cc ${CMAKE_CURRENT_SOURCE_DIR}/kernel/cpu/less_test_kernel_mod.cc @@ -57,10 +58,12 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE) ${CMAKE_CURRENT_SOURCE_DIR}/session/factory.cc ${CMAKE_CURRENT_SOURCE_DIR}/delegate/factory.cc ${CMAKE_CURRENT_SOURCE_DIR}/delegate/plugin/tensorrt_executor_plugin.cc + ${CMAKE_CURRENT_SOURCE_DIR}/delegate/plugin/litert_executor_plugin.cc ${CMAKE_CURRENT_SOURCE_DIR}/delegate/tensorrt/distribution/distribution_base.cc ${CMAKE_CURRENT_SOURCE_DIR}/delegate_graph_executor.cc - ${CMAKE_CURRENT_SOURCE_DIR}/convert/runtime_convert.cc ${CMAKE_CURRENT_SOURCE_DIR}/session/optimizer/tensorrt_optimizer.cc + ${CMAKE_CURRENT_SOURCE_DIR}/delegate/graph_executor/litert/func_graph_reuse_manager.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../litert/pack_weight_manager.cc ) if(MSLITE_ENABLE_BFC_MEMORY) set(MSLITE_EXTEND_RUNTIME_SRC ${MSLITE_EXTEND_RUNTIME_SRC} @@ -158,8 +161,6 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE) mindspore-kernel-graph _mindspore_backend_common_optimizer_obj _mindspore_backend_common_pass_obj) target_link_libraries(mindspore-extendrt mindspore_core mindspore::protobuf) - add_dependencies(mindspore-extendrt msplugin-ge-litert) - target_link_libraries(mindspore-extendrt msplugin-ge-litert) target_link_libraries(mindspore-extendrt_static mindspore_infer_shared_lib_obj) target_link_libraries(mindspore-extendrt_static mindspore-infer-anfalgo mindspore-kernel-graph _mindspore_backend_common_optimizer_obj @@ -206,7 +207,7 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE) add_subdirectory(convert) endif() - set(TEST_CLOUD_INFER on) + set(TEST_CLOUD_INFER off) if(TEST_CLOUD_INFER) set(CLOUD_INFER_TEST_SRC ${CLOUD_INFER_TEST_SRC} diff --git a/mindspore/lite/src/extendrt/cxx_api/model_pool/model_parallel_runner_impl.cc b/mindspore/lite/src/extendrt/cxx_api/model_pool/model_parallel_runner_impl.cc index 2681ecd77c0..e284c4ecb30 100644 --- a/mindspore/lite/src/extendrt/cxx_api/model_pool/model_parallel_runner_impl.cc +++ b/mindspore/lite/src/extendrt/cxx_api/model_pool/model_parallel_runner_impl.cc @@ -33,11 +33,13 @@ Status ModelParallelRunnerImpl::Init(const std::string &model_path, MS_LOG(ERROR) << "new model pool failed, model pool is nullptr."; return kLiteNullptr; } +#ifndef ENABLE_CLOUD_FUSION_INFERENCE if (!PlatformInstructionSetSupportCheck()) { delete model_pool_; model_pool_ = nullptr; return kLiteNotSupport; } +#endif auto status = model_pool_->InitByPath(model_path, runner_config); if (status != kSuccess) { MS_LOG(ERROR) << "ModelParallelRunner init failed."; @@ -63,11 +65,13 @@ Status ModelParallelRunnerImpl::Init(const void *model_data, size_t data_size, MS_LOG(ERROR) << "new model pool failed, model pool is nullptr."; return kLiteNullptr; } +#ifndef ENABLE_CLOUD_FUSION_INFERENCE if (!PlatformInstructionSetSupportCheck()) { delete model_pool_; model_pool_ = nullptr; return kLiteNotSupport; } +#endif auto status = model_pool_->InitByBuf(static_cast(model_data), data_size, runner_config); if (status != kSuccess) { MS_LOG(ERROR) << "model runner init failed."; diff --git a/mindspore/lite/src/extendrt/delegate/graph_executor/litert/CMakeLists.txt b/mindspore/lite/src/extendrt/delegate/graph_executor/litert/CMakeLists.txt index a0eca1eed71..aa0f2f5ef40 100644 --- a/mindspore/lite/src/extendrt/delegate/graph_executor/litert/CMakeLists.txt +++ b/mindspore/lite/src/extendrt/delegate/graph_executor/litert/CMakeLists.txt @@ -316,7 +316,7 @@ endif() set(MSLITE_GE_LITERT_SRC ${MSLITE_GE_LITERT_SRC} ${LITE_DIR}/src/extendrt/delegate/graph_executor/litert/graph_executor.cc - ${LITE_DIR}/src/extendrt/delegate/graph_executor/litert/func_graph_reuse_manager.cc + ${LITE_DIR}/src/extendrt/delegate/graph_executor/litert/litert_plugin_impl.cc ) set(LITE_SRC ${LITE_SRC} ${MSLITE_GE_LITERT_SRC}) diff --git a/mindspore/lite/src/extendrt/delegate/graph_executor/litert/litert_plugin_impl.cc b/mindspore/lite/src/extendrt/delegate/graph_executor/litert/litert_plugin_impl.cc new file mode 100644 index 00000000000..75e43fc088a --- /dev/null +++ b/mindspore/lite/src/extendrt/delegate/graph_executor/litert/litert_plugin_impl.cc @@ -0,0 +1,20 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "extendrt/delegate/graph_executor/litert/litert_plugin_impl.h" + +mindspore::infer::LiteRTExecutorPluginImplBase *CreateLiteRTPluginImpl() { + return new mindspore::infer::LiteRTPluginImpl(); +} diff --git a/mindspore/lite/src/extendrt/delegate/graph_executor/litert/litert_plugin_impl.h b/mindspore/lite/src/extendrt/delegate/graph_executor/litert/litert_plugin_impl.h new file mode 100644 index 00000000000..50acbffc1a3 --- /dev/null +++ b/mindspore/lite/src/extendrt/delegate/graph_executor/litert/litert_plugin_impl.h @@ -0,0 +1,31 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_GRAPH_EXECUTOR_LITERT_GRAPH_EXECUTO_LITERT_PLUGIN_IMPL_H_ +#define MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_GRAPH_EXECUTOR_LITERT_GRAPH_EXECUTO_LITERT_PLUGIN_IMPL_H_ +#include "include/api/status.h" +#include "utils/log_adapter.h" +#include "extendrt/delegate/plugin/litert_executor_plugin.h" + +namespace mindspore::infer { +class LiteRTPluginImpl : public LiteRTExecutorPluginImplBase { + public: + LiteRTPluginImpl() = default; + ~LiteRTPluginImpl() = default; +}; +} // namespace mindspore::infer + +extern "C" MS_API mindspore::infer::LiteRTExecutorPluginImplBase *CreateLiteRTPluginImpl(); +#endif // MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_GRAPH_EXECUTOR_LITERT_GRAPH_EXECUTO_LITERT_PLUGIN_IMPL_H_ diff --git a/mindspore/lite/src/extendrt/delegate/plugin/litert_executor_plugin.cc b/mindspore/lite/src/extendrt/delegate/plugin/litert_executor_plugin.cc new file mode 100644 index 00000000000..df56aa2149f --- /dev/null +++ b/mindspore/lite/src/extendrt/delegate/plugin/litert_executor_plugin.cc @@ -0,0 +1,77 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "extendrt/delegate/plugin/litert_executor_plugin.h" +#include "utils/log_adapter.h" +#if !defined(_WIN32) +#include "extendrt/cxx_api/dlutils.h" +#endif + +namespace mindspore::infer { +namespace { +constexpr auto kLiteRtPluginSoName = "libmsplugin-ge-litert.so"; +constexpr auto kFunCreateLiteRTPluginImp = "CreateLiteRTPluginImpl"; +} // namespace +LiteRTExecutorPlugin::LiteRTExecutorPlugin() = default; +LiteRTExecutorPlugin::~LiteRTExecutorPlugin() { +#if !defined(_WIN32) + MS_LOG(DEBUG) << "~LiteRTExecutorPlugin() begin."; + DLSoClose(handle_); + MS_LOG(DEBUG) << "~LiteRTExecutorPlugin() end."; +#endif +} + +LiteRTExecutorPlugin &LiteRTExecutorPlugin::GetInstance() { + static LiteRTExecutorPlugin instance; + return instance; +} + +bool LiteRTExecutorPlugin::Register() { +#if !defined(_WIN32) + if (is_registered_) { + return true; + } + std::string plugin_path; + auto ret = DLSoPath({"libmindspore-lite.so", "_c_lite"}, kLiteRtPluginSoName, &plugin_path); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Get real path of " << kLiteRtPluginSoName << " failed."; + return false; + } + MS_LOG(INFO) << "Find litert plugin so success, path = " << plugin_path; + void *function = nullptr; + ret = DLSoOpen(plugin_path, kFunCreateLiteRTPluginImp, &handle_, &function); + if (ret != kSuccess) { + MS_LOG(ERROR) << "DLSoOpen failed, so path: " << plugin_path; + return false; + } + auto create_kernel_func = reinterpret_cast(function); + if (create_kernel_func == nullptr) { + MS_LOG(ERROR) << "Cast " << kFunCreateLiteRTPluginImp << " failed."; + return false; + } + auto plugin_impl = create_kernel_func(); + if (plugin_impl == nullptr) { + MS_LOG(ERROR) << "Create custom TensorRT kernel failed."; + return false; + } + is_registered_ = true; + MS_LOG(INFO) << "Register litert plugin success."; +#endif + return true; +} +} // namespace mindspore::infer diff --git a/mindspore/lite/src/extendrt/delegate/plugin/litert_executor_plugin.h b/mindspore/lite/src/extendrt/delegate/plugin/litert_executor_plugin.h new file mode 100644 index 00000000000..b8033f89ba2 --- /dev/null +++ b/mindspore/lite/src/extendrt/delegate/plugin/litert_executor_plugin.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_LITE_SRC_EXTENDRT_LITERT_EXECUTOR_PLUGIN_H_ +#define MINDSPORE_LITE_SRC_EXTENDRT_LITERT_EXECUTOR_PLUGIN_H_ +#include "include/api/status.h" +#include "utils/log_adapter.h" +#include "mindapi/base/macros.h" + +namespace mindspore::infer { +class MS_API LiteRTExecutorPlugin { + public: + static LiteRTExecutorPlugin &GetInstance(); + bool Register(); + + private: + LiteRTExecutorPlugin(); + ~LiteRTExecutorPlugin(); + + void *handle_ = nullptr; + bool is_registered_ = false; +}; + +class LiteRTExecutorPluginImplBase { + public: + LiteRTExecutorPluginImplBase() = default; + virtual ~LiteRTExecutorPluginImplBase() = default; +}; +} // namespace mindspore::infer +#endif // MINDSPORE_LITE_SRC_EXTENDRT_LITERT_EXECUTOR_PLUGIN_H_ diff --git a/mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_graph_executor.cc b/mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_graph_executor.cc index 940029a4a5f..601ed8fce2b 100644 --- a/mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_graph_executor.cc +++ b/mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_graph_executor.cc @@ -31,6 +31,7 @@ #include "src/extendrt/utils/kernel_graph_utils.h" #include "common/config_infos.h" #include "tools/optimizer/common/gllo_utils.h" +#include "src/extendrt/utils/func_graph_utils.h" namespace mindspore::lite { namespace { @@ -149,7 +150,7 @@ Status GetAbstractArgsFromCNode(const CNodePtr &cnode, std::vectorclear(); - auto input_nodes = opt::GetNodeInputs(cnode); + auto input_nodes = mindspore::GetNodeInputs(cnode); for (auto &tensor_id : input_nodes) { auto it = std::find_if(tensor_info_list.begin(), tensor_info_list.end(), [&tensor_id](const NodeWithOutputIndex &index) { return index.kernel_index == tensor_id; }); diff --git a/mindspore/lite/src/extendrt/delegate_graph_executor.cc b/mindspore/lite/src/extendrt/delegate_graph_executor.cc index 3d2c6516737..c45826acf50 100644 --- a/mindspore/lite/src/extendrt/delegate_graph_executor.cc +++ b/mindspore/lite/src/extendrt/delegate_graph_executor.cc @@ -17,21 +17,21 @@ #include #include #include "src/extendrt/subgraph_kernel.h" -#include "tools/common/func_graph_subgraph.h" +// #include "tools/common/func_graph_subgraph.h" #include "ops/fusion/partial_fusion.h" namespace mindspore { // Graph sink delegate, the whole FuncGraph as a node to execute. void GraphSinkDelegate::ReplaceNodes(const std::shared_ptr &graph) { sink_graph_ = graph; // replace the whole graph to a partial node. - lite::SubGraph helper(graph); - auto nodes = graph->order_list(); - std::set cnodeset; - for (auto it = nodes.begin(); it != nodes.end(); it++) { - cnodeset.emplace(*it); - } - helper.Reset(cnodeset); - helper.ApplySubGraph(); + // lite::SubGraph helper(graph); + // auto nodes = graph->order_list(); + // std::set cnodeset; + // for (auto it = nodes.begin(); it != nodes.end(); it++) { + // cnodeset.emplace(*it); + // } + // helper.Reset(cnodeset); + // helper.ApplySubGraph(); return; } diff --git a/mindspore/lite/src/extendrt/infer_session.cc b/mindspore/lite/src/extendrt/infer_session.cc index c5c572615a6..99532e867ee 100644 --- a/mindspore/lite/src/extendrt/infer_session.cc +++ b/mindspore/lite/src/extendrt/infer_session.cc @@ -26,6 +26,7 @@ #include "extendrt/delegate/factory.h" #include "extendrt/session/factory.h" #include "extendrt/delegate/plugin/tensorrt_executor_plugin.h" +#include "extendrt/delegate/plugin/litert_executor_plugin.h" namespace mindspore { static const std::vector ms_infer_cut_list = {prim::kPrimReturn, prim::kPrimPartial, @@ -129,7 +130,11 @@ void InferSession::HandleContext(const std::shared_ptr &context) { continue; } auto provider = cpu_device->GetProvider(); - if (provider.empty()) { + if (provider.empty() || provider == default_cpu_provider) { + if (!infer::LiteRTExecutorPlugin::GetInstance().Register()) { + MS_LOG_WARNING << "Failed to register LiteRT plugin"; + return; + } cpu_device->SetProvider(default_cpu_provider); } continue; diff --git a/mindspore/lite/src/extendrt/utils/func_graph_utils.cc b/mindspore/lite/src/extendrt/utils/func_graph_utils.cc index b3d86526af3..c987a2c8048 100644 --- a/mindspore/lite/src/extendrt/utils/func_graph_utils.cc +++ b/mindspore/lite/src/extendrt/utils/func_graph_utils.cc @@ -22,12 +22,11 @@ #include #include "src/extendrt/utils/func_graph_utils.h" -#include "include/common/utils/anfalgo.h" #include "include/common/utils/convert_utils.h" #include "mindspore/ccsrc/backend/common/optimizer/helper.h" -#include "tools/optimizer/common/gllo_utils.h" namespace mindspore { +const PrimitivePtr kPrimMakeTupleV2 = std::make_shared("make_tuple"); ValuePtr FuncGraphUtils::GetNodeValuePtr(AnfNodePtr input_node) { if (input_node == nullptr) { return nullptr; @@ -115,6 +114,42 @@ bool FuncGraphUtils::GetCNodeOperator(const mindspore::CNodePtr &cnode, return true; } +bool CheckPrimitiveType(const AnfNodePtr &node, const PrimitivePtr &primitive_type) { + if (node == nullptr || primitive_type == nullptr) { + return false; + } + if (node->isa()) { + auto cnode = node->cast(); + return IsPrimitive(cnode->input(kAnfPrimitiveIndex), primitive_type); + } else if (node->isa()) { + return IsPrimitive(node, primitive_type); + } + return false; +} + +std::vector GetNodeInputs(const AnfNodePtr &anf_node) { + if (!anf_node) { + return {}; + } + if (!anf_node->isa()) { + return {{anf_node, 0}}; + } + auto cnode = anf_node->cast(); + std::vector inputs; + size_t input_num = common::AnfAlgo::GetInputTensorNum(cnode); + for (size_t input_idx = 0; input_idx < input_num; ++input_idx) { + const auto &pre_node_output = common::AnfAlgo::GetPrevNodeOutput(cnode, input_idx); + auto pre_node = pre_node_output.first; + if (CheckPrimitiveType(pre_node, prim::kPrimMakeTuple) || CheckPrimitiveType(pre_node, kPrimMakeTupleV2)) { + auto tuple_inputs = GetNodeInputs(pre_node); + std::copy(tuple_inputs.begin(), tuple_inputs.end(), std::back_inserter(inputs)); + } else { + inputs.push_back(pre_node_output); + } + } + return inputs; +} + bool FuncGraphUtils::GetCNodeInputsOutputs(const mindspore::CNodePtr &cnode, std::vector *input_tensors, std::vector *output_tensors) { @@ -123,7 +158,7 @@ bool FuncGraphUtils::GetCNodeInputsOutputs(const mindspore::CNodePtr &cnode, return false; } // Makeup input tensors. - *input_tensors = opt::GetNodeInputs(cnode); + *input_tensors = GetNodeInputs(cnode); // Makeup output tensors. output_tensors->clear(); auto output_num = common::AnfAlgo::GetOutputTensorNum(cnode); @@ -161,7 +196,7 @@ bool FuncGraphUtils::GetFuncGraphOutputs(const FuncGraphPtr &func_graph, std::ve MS_LOG(ERROR) << "Input func_graph or outputs cannot be nullptr"; return false; } - *outputs = opt::GetNodeInputs(func_graph->get_return()); + *outputs = GetNodeInputs(func_graph->get_return()); return true; } diff --git a/mindspore/lite/src/extendrt/utils/func_graph_utils.h b/mindspore/lite/src/extendrt/utils/func_graph_utils.h index e0f0db48dea..411a4c4cd88 100644 --- a/mindspore/lite/src/extendrt/utils/func_graph_utils.h +++ b/mindspore/lite/src/extendrt/utils/func_graph_utils.h @@ -28,11 +28,14 @@ #include "ir/func_graph.h" #include "include/api/data_type.h" #include "mindspore/ccsrc/kernel/kernel.h" +#include "include/common/utils/anfalgo.h" namespace mindspore { using AnfWithOutIndex = std::pair; using kernel::BaseOperatorPtr; +std::vector GetNodeInputs(const AnfNodePtr &anf_node); + class FuncGraphUtils { public: static tensor::TensorPtr GetConstNodeValue(AnfNodePtr input_node);