mindspore-Ecosystem
/
mindspore
21
0
Fork 27
Code Issues Pull Requests 6 Projects Releases Wiki Activity

mindspore lite: change litert to plugin

This commit is contained in:
liu lili 2022-12-01 16:10:46 +08:00
parent 6d9fdaacc1
commit 9d2434ad06
12 changed files with 239 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
mindspore/lite/src/extendrt/CMakeLists.txt
View File

@ -40,6 +40,7 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE)
${CMAKE_CURRENT_SOURCE_DIR}/../common/file_utils.cc
${CMAKE_CURRENT_SOURCE_DIR}/../common/utils.cc
${CMAKE_CURRENT_SOURCE_DIR}/../common/graph_util.cc
${CMAKE_CURRENT_SOURCE_DIR}/../common/config_infos.cc
${CMAKE_CURRENT_SOURCE_DIR}/subgraph_kernel.cc
${CMAKE_CURRENT_SOURCE_DIR}/numa_adapter.cc
${CMAKE_CURRENT_SOURCE_DIR}/kernel/cpu/less_test_kernel_mod.cc
@ -57,10 +58,12 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE)
${CMAKE_CURRENT_SOURCE_DIR}/session/factory.cc
${CMAKE_CURRENT_SOURCE_DIR}/delegate/factory.cc
${CMAKE_CURRENT_SOURCE_DIR}/delegate/plugin/tensorrt_executor_plugin.cc
${CMAKE_CURRENT_SOURCE_DIR}/delegate/plugin/litert_executor_plugin.cc
${CMAKE_CURRENT_SOURCE_DIR}/delegate/tensorrt/distribution/distribution_base.cc
${CMAKE_CURRENT_SOURCE_DIR}/delegate_graph_executor.cc
${CMAKE_CURRENT_SOURCE_DIR}/convert/runtime_convert.cc
${CMAKE_CURRENT_SOURCE_DIR}/session/optimizer/tensorrt_optimizer.cc
${CMAKE_CURRENT_SOURCE_DIR}/delegate/graph_executor/litert/func_graph_reuse_manager.cc
${CMAKE_CURRENT_SOURCE_DIR}/../litert/pack_weight_manager.cc
)
if(MSLITE_ENABLE_BFC_MEMORY)
set(MSLITE_EXTEND_RUNTIME_SRC ${MSLITE_EXTEND_RUNTIME_SRC}
@ -158,8 +161,6 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE)
mindspore-kernel-graph _mindspore_backend_common_optimizer_obj
_mindspore_backend_common_pass_obj)
target_link_libraries(mindspore-extendrt mindspore_core mindspore::protobuf)
add_dependencies(mindspore-extendrt msplugin-ge-litert)
target_link_libraries(mindspore-extendrt msplugin-ge-litert)
target_link_libraries(mindspore-extendrt_static mindspore_infer_shared_lib_obj)
target_link_libraries(mindspore-extendrt_static mindspore-infer-anfalgo
mindspore-kernel-graph _mindspore_backend_common_optimizer_obj
@ -206,7 +207,7 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE)
add_subdirectory(convert)
endif()
set(TEST_CLOUD_INFER on)
set(TEST_CLOUD_INFER off)
if(TEST_CLOUD_INFER)
set(CLOUD_INFER_TEST_SRC ${CLOUD_INFER_TEST_SRC}

4
mindspore/lite/src/extendrt/cxx_api/model_pool/model_parallel_runner_impl.cc
View File

@ -33,11 +33,13 @@ Status ModelParallelRunnerImpl::Init(const std::string &model_path,
MS_LOG(ERROR) << "new model pool failed, model pool is nullptr.";
return kLiteNullptr;
}
#ifndef ENABLE_CLOUD_FUSION_INFERENCE
if (!PlatformInstructionSetSupportCheck()) {
delete model_pool_;
model_pool_ = nullptr;
return kLiteNotSupport;
}
#endif
auto status = model_pool_->InitByPath(model_path, runner_config);
if (status != kSuccess) {
MS_LOG(ERROR) << "ModelParallelRunner init failed.";
@ -63,11 +65,13 @@ Status ModelParallelRunnerImpl::Init(const void *model_data, size_t data_size,
MS_LOG(ERROR) << "new model pool failed, model pool is nullptr.";
return kLiteNullptr;
}
#ifndef ENABLE_CLOUD_FUSION_INFERENCE
if (!PlatformInstructionSetSupportCheck()) {
delete model_pool_;
model_pool_ = nullptr;
return kLiteNotSupport;
}
#endif
auto status = model_pool_->InitByBuf(static_cast<const char *>(model_data), data_size, runner_config);
if (status != kSuccess) {
MS_LOG(ERROR) << "model runner init failed.";

2
mindspore/lite/src/extendrt/delegate/graph_executor/litert/CMakeLists.txt
View File

@ -316,7 +316,7 @@ endif()
set(MSLITE_GE_LITERT_SRC
${MSLITE_GE_LITERT_SRC}
${LITE_DIR}/src/extendrt/delegate/graph_executor/litert/graph_executor.cc
${LITE_DIR}/src/extendrt/delegate/graph_executor/litert/func_graph_reuse_manager.cc
${LITE_DIR}/src/extendrt/delegate/graph_executor/litert/litert_plugin_impl.cc
)
set(LITE_SRC ${LITE_SRC} ${MSLITE_GE_LITERT_SRC})

20
mindspore/lite/src/extendrt/delegate/graph_executor/litert/litert_plugin_impl.cc Normal file
View File

@ -0,0 +1,20 @@
/**
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "extendrt/delegate/graph_executor/litert/litert_plugin_impl.h"
mindspore::infer::LiteRTExecutorPluginImplBase *CreateLiteRTPluginImpl() {
return new mindspore::infer::LiteRTPluginImpl();
}

31
mindspore/lite/src/extendrt/delegate/graph_executor/litert/litert_plugin_impl.h Normal file
View File

@ -0,0 +1,31 @@
/**
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_GRAPH_EXECUTOR_LITERT_GRAPH_EXECUTO_LITERT_PLUGIN_IMPL_H_
#define MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_GRAPH_EXECUTOR_LITERT_GRAPH_EXECUTO_LITERT_PLUGIN_IMPL_H_
#include "include/api/status.h"
#include "utils/log_adapter.h"
#include "extendrt/delegate/plugin/litert_executor_plugin.h"
namespace mindspore::infer {
class LiteRTPluginImpl : public LiteRTExecutorPluginImplBase {
public:
LiteRTPluginImpl() = default;
~LiteRTPluginImpl() = default;
};
} // namespace mindspore::infer
extern "C" MS_API mindspore::infer::LiteRTExecutorPluginImplBase *CreateLiteRTPluginImpl();
#endif // MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_GRAPH_EXECUTOR_LITERT_GRAPH_EXECUTO_LITERT_PLUGIN_IMPL_H_

77
mindspore/lite/src/extendrt/delegate/plugin/litert_executor_plugin.cc Normal file
View File

@ -0,0 +1,77 @@
/**
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <string>
#include "extendrt/delegate/plugin/litert_executor_plugin.h"
#include "utils/log_adapter.h"
#if !defined(_WIN32)
#include "extendrt/cxx_api/dlutils.h"
#endif
namespace mindspore::infer {
namespace {
constexpr auto kLiteRtPluginSoName = "libmsplugin-ge-litert.so";
constexpr auto kFunCreateLiteRTPluginImp = "CreateLiteRTPluginImpl";
} // namespace
LiteRTExecutorPlugin::LiteRTExecutorPlugin() = default;
LiteRTExecutorPlugin::~LiteRTExecutorPlugin() {
#if !defined(_WIN32)
MS_LOG(DEBUG) << "~LiteRTExecutorPlugin() begin.";
DLSoClose(handle_);
MS_LOG(DEBUG) << "~LiteRTExecutorPlugin() end.";
#endif
}
LiteRTExecutorPlugin &LiteRTExecutorPlugin::GetInstance() {
static LiteRTExecutorPlugin instance;
return instance;
}
bool LiteRTExecutorPlugin::Register() {
#if !defined(_WIN32)
if (is_registered_) {
return true;
}
std::string plugin_path;
auto ret = DLSoPath({"libmindspore-lite.so", "_c_lite"}, kLiteRtPluginSoName, &plugin_path);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Get real path of " << kLiteRtPluginSoName << " failed.";
return false;
}
MS_LOG(INFO) << "Find litert plugin so success, path = " << plugin_path;
void *function = nullptr;
ret = DLSoOpen(plugin_path, kFunCreateLiteRTPluginImp, &handle_, &function);
if (ret != kSuccess) {
MS_LOG(ERROR) << "DLSoOpen failed, so path: " << plugin_path;
return false;
}
auto create_kernel_func = reinterpret_cast<mindspore::infer::LiteRTExecutorPluginImplBase *(*)(void)>(function);
if (create_kernel_func == nullptr) {
MS_LOG(ERROR) << "Cast " << kFunCreateLiteRTPluginImp << " failed.";
return false;
}
auto plugin_impl = create_kernel_func();
if (plugin_impl == nullptr) {
MS_LOG(ERROR) << "Create custom TensorRT kernel failed.";
return false;
}
is_registered_ = true;
MS_LOG(INFO) << "Register litert plugin success.";
#endif
return true;
}
} // namespace mindspore::infer

42
mindspore/lite/src/extendrt/delegate/plugin/litert_executor_plugin.h Normal file
View File

@ -0,0 +1,42 @@
/**
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_SRC_EXTENDRT_LITERT_EXECUTOR_PLUGIN_H_
#define MINDSPORE_LITE_SRC_EXTENDRT_LITERT_EXECUTOR_PLUGIN_H_
#include "include/api/status.h"
#include "utils/log_adapter.h"
#include "mindapi/base/macros.h"
namespace mindspore::infer {
class MS_API LiteRTExecutorPlugin {
public:
static LiteRTExecutorPlugin &GetInstance();
bool Register();
private:
LiteRTExecutorPlugin();
~LiteRTExecutorPlugin();
void *handle_ = nullptr;
bool is_registered_ = false;
};
class LiteRTExecutorPluginImplBase {
public:
LiteRTExecutorPluginImplBase() = default;
virtual ~LiteRTExecutorPluginImplBase() = default;
};
} // namespace mindspore::infer
#endif // MINDSPORE_LITE_SRC_EXTENDRT_LITERT_EXECUTOR_PLUGIN_H_

3
mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_graph_executor.cc
View File

@ -31,6 +31,7 @@
#include "src/extendrt/utils/kernel_graph_utils.h"
#include "common/config_infos.h"
#include "tools/optimizer/common/gllo_utils.h"
#include "src/extendrt/utils/func_graph_utils.h"
namespace mindspore::lite {
namespace {
@ -149,7 +150,7 @@ Status GetAbstractArgsFromCNode(const CNodePtr &cnode, std::vector<NodeWithOutpu
MS_EXCEPTION_IF_NULL(*base_operator);
// Makeup input tensors.
input_tensors->clear();
auto input_nodes = opt::GetNodeInputs(cnode);
auto input_nodes = mindspore::GetNodeInputs(cnode);
for (auto &tensor_id : input_nodes) {
auto it = std::find_if(tensor_info_list.begin(), tensor_info_list.end(),
[&tensor_id](const NodeWithOutputIndex &index) { return index.kernel_index == tensor_id; });

18
mindspore/lite/src/extendrt/delegate_graph_executor.cc
View File

@ -17,21 +17,21 @@
#include <set>
#include <memory>
#include "src/extendrt/subgraph_kernel.h"
#include "tools/common/func_graph_subgraph.h"
// #include "tools/common/func_graph_subgraph.h"
#include "ops/fusion/partial_fusion.h"
namespace mindspore {
// Graph sink delegate, the whole FuncGraph as a node to execute.
void GraphSinkDelegate::ReplaceNodes(const std::shared_ptr<FuncGraph> &graph) {
sink_graph_ = graph;
// replace the whole graph to a partial node.
lite::SubGraph helper(graph);
auto nodes = graph->order_list();
std::set<CNodePtr> cnodeset;
for (auto it = nodes.begin(); it != nodes.end(); it++) {
cnodeset.emplace(*it);
}
helper.Reset(cnodeset);
helper.ApplySubGraph();
// lite::SubGraph helper(graph);
// auto nodes = graph->order_list();
// std::set<CNodePtr> cnodeset;
// for (auto it = nodes.begin(); it != nodes.end(); it++) {
// cnodeset.emplace(*it);
// }
// helper.Reset(cnodeset);
// helper.ApplySubGraph();
return;
}

7
mindspore/lite/src/extendrt/infer_session.cc
View File

@ -26,6 +26,7 @@
#include "extendrt/delegate/factory.h"
#include "extendrt/session/factory.h"
#include "extendrt/delegate/plugin/tensorrt_executor_plugin.h"
#include "extendrt/delegate/plugin/litert_executor_plugin.h"
namespace mindspore {
static const std::vector<PrimitivePtr> ms_infer_cut_list = {prim::kPrimReturn, prim::kPrimPartial,
@ -129,7 +130,11 @@ void InferSession::HandleContext(const std::shared_ptr<Context> &context) {
continue;
}
auto provider = cpu_device->GetProvider();
if (provider.empty()) {
if (provider.empty() || provider == default_cpu_provider) {
if (!infer::LiteRTExecutorPlugin::GetInstance().Register()) {
MS_LOG_WARNING << "Failed to register LiteRT plugin";
return;
}
cpu_device->SetProvider(default_cpu_provider);
}
continue;

43
mindspore/lite/src/extendrt/utils/func_graph_utils.cc
View File

@ -22,12 +22,11 @@
#include <memory>
#include "src/extendrt/utils/func_graph_utils.h"
#include "include/common/utils/anfalgo.h"
#include "include/common/utils/convert_utils.h"
#include "mindspore/ccsrc/backend/common/optimizer/helper.h"
#include "tools/optimizer/common/gllo_utils.h"
namespace mindspore {
const PrimitivePtr kPrimMakeTupleV2 = std::make_shared<Primitive>("make_tuple");
ValuePtr FuncGraphUtils::GetNodeValuePtr(AnfNodePtr input_node) {
if (input_node == nullptr) {
return nullptr;
@ -115,6 +114,42 @@ bool FuncGraphUtils::GetCNodeOperator(const mindspore::CNodePtr &cnode,
return true;
}
bool CheckPrimitiveType(const AnfNodePtr &node, const PrimitivePtr &primitive_type) {
if (node == nullptr || primitive_type == nullptr) {
return false;
}
if (node->isa<CNode>()) {
auto cnode = node->cast<CNodePtr>();
return IsPrimitive(cnode->input(kAnfPrimitiveIndex), primitive_type);
} else if (node->isa<ValueNode>()) {
return IsPrimitive(node, primitive_type);
}
return false;
}
std::vector<common::KernelWithIndex> GetNodeInputs(const AnfNodePtr &anf_node) {
if (!anf_node) {
return {};
}
if (!anf_node->isa<CNode>()) {
return {{anf_node, 0}};
}
auto cnode = anf_node->cast<CNodePtr>();
std::vector<common::KernelWithIndex> inputs;
size_t input_num = common::AnfAlgo::GetInputTensorNum(cnode);
for (size_t input_idx = 0; input_idx < input_num; ++input_idx) {
const auto &pre_node_output = common::AnfAlgo::GetPrevNodeOutput(cnode, input_idx);
auto pre_node = pre_node_output.first;
if (CheckPrimitiveType(pre_node, prim::kPrimMakeTuple) || CheckPrimitiveType(pre_node, kPrimMakeTupleV2)) {
auto tuple_inputs = GetNodeInputs(pre_node);
std::copy(tuple_inputs.begin(), tuple_inputs.end(), std::back_inserter(inputs));
} else {
inputs.push_back(pre_node_output);
}
}
return inputs;
}
bool FuncGraphUtils::GetCNodeInputsOutputs(const mindspore::CNodePtr &cnode,
std::vector<AnfWithOutIndex> *input_tensors,
std::vector<AnfWithOutIndex> *output_tensors) {
@ -123,7 +158,7 @@ bool FuncGraphUtils::GetCNodeInputsOutputs(const mindspore::CNodePtr &cnode,
return false;
}
// Makeup input tensors.
*input_tensors = opt::GetNodeInputs(cnode);
*input_tensors = GetNodeInputs(cnode);
// Makeup output tensors.
output_tensors->clear();
auto output_num = common::AnfAlgo::GetOutputTensorNum(cnode);
@ -161,7 +196,7 @@ bool FuncGraphUtils::GetFuncGraphOutputs(const FuncGraphPtr &func_graph, std::ve
MS_LOG(ERROR) << "Input func_graph or outputs cannot be nullptr";
return false;
}
*outputs = opt::GetNodeInputs(func_graph->get_return());
*outputs = GetNodeInputs(func_graph->get_return());
return true;
}

3
mindspore/lite/src/extendrt/utils/func_graph_utils.h
View File

@ -28,11 +28,14 @@
#include "ir/func_graph.h"
#include "include/api/data_type.h"
#include "mindspore/ccsrc/kernel/kernel.h"
#include "include/common/utils/anfalgo.h"
namespace mindspore {
using AnfWithOutIndex = std::pair<AnfNodePtr, size_t>;
using kernel::BaseOperatorPtr;
std::vector<common::KernelWithIndex> GetNodeInputs(const AnfNodePtr &anf_node);
class FuncGraphUtils {
public:
static tensor::TensorPtr GetConstNodeValue(AnfNodePtr input_node);