From 164da34d995c792d2d2ae6a1b3434fd685e5b18f Mon Sep 17 00:00:00 2001 From: xuyongfei Date: Sat, 6 Aug 2022 21:09:52 +0800 Subject: [PATCH] Lite Cloud inference: TensorRT plugin --- cmake/package_lite.cmake | 12 ++ mindspore/ccsrc/utils/anfalgo.cc | 5 +- mindspore/lite/src/CMakeLists.txt | 8 +- mindspore/lite/src/extendrt/CMakeLists.txt | 20 +--- mindspore/lite/src/extendrt/cxx_api/dlutils.h | 27 ++++- .../src/extendrt/cxx_api/model/model_impl.cc | 5 +- .../lite/src/extendrt/delegate/factory.cc | 23 ++++ .../lite/src/extendrt/delegate/factory.h | 11 +- .../delegate/graph_executor/factory.cc | 23 ++++ .../delegate/graph_executor/factory.h | 9 +- .../plugin/tensorrt_executor_plugin.cc | 95 ++++++++++++++++ .../plugin/tensorrt_executor_plugin.h | 49 ++++++++ .../extendrt/delegate/tensorrt/CMakeLists.txt | 25 ++++- .../tensorrt/op/tensorrt_executor_plugin.h | 106 ++++++++++++++++++ .../delegate/tensorrt/op/tensorrt_op.h | 2 +- ...ase_impl.cc => op_registration_factory.cc} | 19 ++-- .../tensorrt/op_registration_factory.h | 54 +++++++++ .../tensorrt/tensorrt_graph_executor.cc | 1 - .../delegate/tensorrt/tensorrt_plugin_impl.cc | 28 +++++ .../delegate/tensorrt/tensorrt_plugin_impl.h | 33 ++++++ mindspore/lite/src/extendrt/infer_session.cc | 31 ++++- mindspore/lite/src/extendrt/infer_session.h | 1 + .../extendrt/mock/lite_runtime/CMakeLists.txt | 6 - .../lite/src/extendrt/session/factory.cc | 42 +++++++ mindspore/lite/src/extendrt/session/factory.h | 19 +--- .../session/graph_executor_session.cc | 12 +- mindspore/lite/tools/benchmark/CMakeLists.txt | 1 - mindspore/lite/tools/converter/CMakeLists.txt | 1 + .../mindir_exporter/mindir_serializer.cc | 4 - 29 files changed, 577 insertions(+), 95 deletions(-) create mode 100644 mindspore/lite/src/extendrt/delegate/factory.cc create mode 100644 mindspore/lite/src/extendrt/delegate/graph_executor/factory.cc create mode 100644 mindspore/lite/src/extendrt/delegate/plugin/tensorrt_executor_plugin.cc create mode 100644 mindspore/lite/src/extendrt/delegate/plugin/tensorrt_executor_plugin.h create mode 100644 mindspore/lite/src/extendrt/delegate/tensorrt/op/tensorrt_executor_plugin.h rename mindspore/lite/src/extendrt/delegate/tensorrt/{distribution/distribution_base_impl.cc => op_registration_factory.cc} (58%) create mode 100644 mindspore/lite/src/extendrt/delegate/tensorrt/op_registration_factory.h create mode 100644 mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_plugin_impl.cc create mode 100644 mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_plugin_impl.h create mode 100644 mindspore/lite/src/extendrt/session/factory.cc diff --git a/cmake/package_lite.cmake b/cmake/package_lite.cmake index a9c7456749d..aded3608d5f 100644 --- a/cmake/package_lite.cmake +++ b/cmake/package_lite.cmake @@ -414,6 +414,10 @@ if(PLATFORM_ARM64) install(FILES ${TOP_DIR}/mindspore/lite/build/src/extendrt/kernel/ascend/libascend_kernel_plugin.so DESTINATION ${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME}) endif() + if(MSLITE_GPU_BACKEND STREQUAL tensorrt) + install(FILES ${TOP_DIR}/mindspore/lite/build/src/extendrt/delegate/tensorrt/libtensorrt_plugin.so + DESTINATION ${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME}) + endif() else() install(FILES ${TOP_DIR}/mindspore/lite/build/src/${MINDSPORE_LITE_LIB_NAME}.so DESTINATION ${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME}) @@ -645,6 +649,10 @@ elseif(PLATFORM_ARM32) install(FILES ${TOP_DIR}/mindspore/lite/build/src/extendrt/kernel/ascend/libascend_kernel_plugin.so DESTINATION ${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME}) endif() + if(MSLITE_GPU_BACKEND STREQUAL tensorrt) + install(FILES ${TOP_DIR}/mindspore/lite/build/src/extendrt/delegate/tensorrt/libtensorrt_plugin.so + DESTINATION ${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME}) + endif() else() install(FILES ${TOP_DIR}/mindspore/lite/build/src/${MINDSPORE_LITE_LIB_NAME}.so DESTINATION ${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME}) @@ -823,6 +831,10 @@ else() install(FILES ${TOP_DIR}/mindspore/lite/build/src/extendrt/kernel/ascend/libascend_kernel_plugin.so DESTINATION ${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME}) endif() + if(MSLITE_GPU_BACKEND STREQUAL tensorrt) + install(FILES ${TOP_DIR}/mindspore/lite/build/src/extendrt/delegate/tensorrt/libtensorrt_plugin.so + DESTINATION ${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME}) + endif() else() install(FILES ${TOP_DIR}/mindspore/lite/build/src/${MINDSPORE_LITE_LIB_NAME}.so DESTINATION ${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME}) diff --git a/mindspore/ccsrc/utils/anfalgo.cc b/mindspore/ccsrc/utils/anfalgo.cc index 079f5fe03a9..c62dfa8c168 100644 --- a/mindspore/ccsrc/utils/anfalgo.cc +++ b/mindspore/ccsrc/utils/anfalgo.cc @@ -214,7 +214,10 @@ size_t AnfAlgo::GetTupleGetItemOutIndex(const CNodePtr &tuple_get_item) { MS_EXCEPTION_IF_NULL(output_index_value_node); auto value_node = output_index_value_node->cast(); MS_EXCEPTION_IF_NULL(value_node); - return LongToSize(GetValue(value_node->value())); + auto value = value_node->value(); + MS_EXCEPTION_IF_NULL(value); + auto idx = value->isa() ? GetValue(value) : GetValue(value); + return LongToSize(idx); } KernelWithIndex AnfAlgo::VisitKernel(const AnfNodePtr &anf_node, size_t index) { diff --git a/mindspore/lite/src/CMakeLists.txt b/mindspore/lite/src/CMakeLists.txt index 49fc38a1f58..45c83d1b2cb 100644 --- a/mindspore/lite/src/CMakeLists.txt +++ b/mindspore/lite/src/CMakeLists.txt @@ -490,18 +490,18 @@ if(SUPPORT_TENSORRT) include_directories(${TENSORRT_PATH}/include) include_directories(${CUDA_PATH}/include) add_subdirectory(litert/delegate/tensorrt) + target_link_libraries(mindspore-lite tensorrt_kernel_mid cuda_kernel_mid gpu_distribution_collective) + target_link_libraries(mindspore-lite_static tensorrt_kernel_mid cuda_kernel_mid gpu_distribution_collective) endif() - target_link_libraries(mindspore-lite tensorrt_kernel_mid cuda_kernel_mid gpu_distribution_collective) - target_link_libraries(mindspore-lite_static tensorrt_kernel_mid cuda_kernel_mid gpu_distribution_collective) else() if(NOT MSLITE_ENABLE_CLOUD_FUSION_INFERENCE) set(TENSORRT_STUB ${CMAKE_CURRENT_SOURCE_DIR}/litert/delegate/tensorrt/distribution/distribution_base.cc ) add_library(tensorrt_stub OBJECT ${TENSORRT_STUB}) + target_link_libraries(mindspore-lite tensorrt_stub) + target_link_libraries(mindspore-lite_static tensorrt_stub) endif() - target_link_libraries(mindspore-lite tensorrt_stub) - target_link_libraries(mindspore-lite_static tensorrt_stub) endif() if(MSLITE_GPU_BACKEND STREQUAL opencl) diff --git a/mindspore/lite/src/extendrt/CMakeLists.txt b/mindspore/lite/src/extendrt/CMakeLists.txt index ec7ed4b336a..9fa2ecc036d 100644 --- a/mindspore/lite/src/extendrt/CMakeLists.txt +++ b/mindspore/lite/src/extendrt/CMakeLists.txt @@ -50,6 +50,11 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE) ${CMAKE_CURRENT_SOURCE_DIR}/delegate/graph_executor/delegate.cc ${CMAKE_CURRENT_SOURCE_DIR}/session/delegate_session.cc ${CMAKE_CURRENT_SOURCE_DIR}/session/graph_executor_session.cc + ${CMAKE_CURRENT_SOURCE_DIR}/session/factory.cc + ${CMAKE_CURRENT_SOURCE_DIR}/delegate/factory.cc + ${CMAKE_CURRENT_SOURCE_DIR}/delegate/graph_executor/factory.cc + ${CMAKE_CURRENT_SOURCE_DIR}/delegate/plugin/tensorrt_executor_plugin.cc + ${CMAKE_CURRENT_SOURCE_DIR}/delegate/tensorrt/distribution/distribution_base.cc ) if(NOT MSLITE_ENABLE_ACL) @@ -173,21 +178,8 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE) endif() if(SUPPORT_TENSORRT) - add_compile_definitions(GPU_TENSORRT) - set(TENSORRT_PATH $ENV{TENSORRT_PATH}) - set(CUDA_PATH $ENV{CUDA_HOME}) - set(TENSORRT_LIB_PATH ${TENSORRT_PATH}/lib) - set(CUDA_LIB_PATH ${CUDA_PATH}/lib64) - include_directories(${TENSORRT_PATH}/include) - include_directories(${CUDA_PATH}/include) + add_definitions(-DSUPPORT_TENSORRT) add_subdirectory(delegate/tensorrt) - target_link_libraries(mindspore-extendrt tensorrt_kernel_mid cuda_kernel_mid gpu_distribution_collective) - else() - set(TENSORRT_STUB - ${CMAKE_CURRENT_SOURCE_DIR}/delegate/tensorrt/distribution/distribution_base.cc - ) - add_library(tensorrt_stub OBJECT ${TENSORRT_STUB}) - target_link_libraries(mindspore-extendrt tensorrt_stub) endif() set(TEST_CLOUD_INFER on) diff --git a/mindspore/lite/src/extendrt/cxx_api/dlutils.h b/mindspore/lite/src/extendrt/cxx_api/dlutils.h index 435a1eb324b..3f777df59c6 100644 --- a/mindspore/lite/src/extendrt/cxx_api/dlutils.h +++ b/mindspore/lite/src/extendrt/cxx_api/dlutils.h @@ -70,15 +70,32 @@ inline Status DLSoPath(const std::string &benchmark_so, const std::string &targe inline Status DLSoOpen(const std::string &dl_path, const std::string &func_name, void **handle, void **function) { // do dlopen and export functions from c_dataengine + if (handle == nullptr) { + MS_LOG(WARNING) << "Input parameter handle cannot be nullptr"; + return Status(kMEFailed, "Input parameter handle cannot be nullptr"); + } *handle = dlopen(dl_path.c_str(), RTLD_LAZY | RTLD_LOCAL); + auto get_dl_error = []() -> std::string { + auto error = dlerror(); + return error == nullptr ? "" : error; + }; if (*handle == nullptr) { - return Status(kMEFailed, "dlopen failed, the pointer[handle] is null."); + auto error = get_dl_error(); + MS_LOG(WARNING) << "dlopen " << dl_path << " failed, error: " << error; + return Status(kMEFailed, "dlopen " + dl_path + " failed, error: " + error); } - - *function = dlsym(*handle, func_name.c_str()); - if (*function == nullptr) { - return Status(kMEFailed, "Could not find " + func_name + " in " + dl_path); + if (!func_name.empty()) { + if (function == nullptr) { + MS_LOG(WARNING) << "Input parameter function cannot be nullptr"; + return Status(kMEFailed, "Input parameter function cannot be nullptr"); + } + *function = dlsym(*handle, func_name.c_str()); + if (*function == nullptr) { + auto error = get_dl_error(); + MS_LOG(WARNING) << "Could not find " + func_name + " in " + dl_path + ", error: " << error; + return Status(kMEFailed, "Could not find " + func_name + " in " + dl_path + ", error: " + error); + } } return kSuccess; } diff --git a/mindspore/lite/src/extendrt/cxx_api/model/model_impl.cc b/mindspore/lite/src/extendrt/cxx_api/model/model_impl.cc index 4ffa9c0f7ff..480c01a7b58 100644 --- a/mindspore/lite/src/extendrt/cxx_api/model/model_impl.cc +++ b/mindspore/lite/src/extendrt/cxx_api/model/model_impl.cc @@ -34,7 +34,10 @@ Status ModelImpl::Build(const void *model_data, size_t data_size, ModelType mode if (session_ == nullptr) { return kLiteNullptr; } - session_->Init(model_context); + ret = session_->Init(model_context); + if (ret != kSuccess) { + return ret; + } if (MsContext::GetInstance() == nullptr) { MS_LOG(INFO) << "MsContext::GetInstance() is nullptr."; MsContext::device_type_seter([](std::shared_ptr &device_type_seter) { diff --git a/mindspore/lite/src/extendrt/delegate/factory.cc b/mindspore/lite/src/extendrt/delegate/factory.cc new file mode 100644 index 00000000000..31d265ee98d --- /dev/null +++ b/mindspore/lite/src/extendrt/delegate/factory.cc @@ -0,0 +1,23 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "extendrt/delegate/factory.h" + +namespace mindspore { +DelegateRegistry &DelegateRegistry::GetInstance() { + static DelegateRegistry instance; + return instance; +} +} // namespace mindspore diff --git a/mindspore/lite/src/extendrt/delegate/factory.h b/mindspore/lite/src/extendrt/delegate/factory.h index 4d168c87867..ddb3c596fea 100644 --- a/mindspore/lite/src/extendrt/delegate/factory.h +++ b/mindspore/lite/src/extendrt/delegate/factory.h @@ -28,15 +28,12 @@ namespace mindspore { typedef std::shared_ptr (*DelegateCreator)(const std::shared_ptr &config); -class DelegateRegistry { +class MS_API DelegateRegistry { public: DelegateRegistry() = default; virtual ~DelegateRegistry() = default; - static DelegateRegistry *GetInstance() { - static DelegateRegistry instance; - return &instance; - } + static DelegateRegistry &GetInstance(); void RegDelegate(const mindspore::DeviceType &device_type, const std::string &provider, DelegateCreator creator) { auto it = creator_map_.find(device_type); @@ -52,7 +49,7 @@ class DelegateRegistry { std::shared_ptr GetDelegate(const mindspore::DeviceType &device_type, const std::string &provider, const std::shared_ptr &config) { // first find graph executor delegate - auto graph_executor_delegate = GraphExecutorRegistry::GetInstance()->GetDelegate(device_type, provider, config); + auto graph_executor_delegate = GraphExecutorRegistry::GetInstance().GetDelegate(device_type, provider, config); if (graph_executor_delegate != nullptr) { return graph_executor_delegate; } @@ -76,7 +73,7 @@ class DelegateRegistry { class DelegateRegistrar { public: DelegateRegistrar(const mindspore::DeviceType &device_type, const std::string &provider, DelegateCreator creator) { - DelegateRegistry::GetInstance()->RegDelegate(device_type, provider, creator); + DelegateRegistry::GetInstance().RegDelegate(device_type, provider, creator); } ~DelegateRegistrar() = default; }; diff --git a/mindspore/lite/src/extendrt/delegate/graph_executor/factory.cc b/mindspore/lite/src/extendrt/delegate/graph_executor/factory.cc new file mode 100644 index 00000000000..48d9290b209 --- /dev/null +++ b/mindspore/lite/src/extendrt/delegate/graph_executor/factory.cc @@ -0,0 +1,23 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "extendrt/delegate/graph_executor/factory.h" + +namespace mindspore { +GraphExecutorRegistry &GraphExecutorRegistry::GetInstance() { + static GraphExecutorRegistry instance; + return instance; +} +} // namespace mindspore diff --git a/mindspore/lite/src/extendrt/delegate/graph_executor/factory.h b/mindspore/lite/src/extendrt/delegate/graph_executor/factory.h index e17e5720284..9e93d51f5b4 100644 --- a/mindspore/lite/src/extendrt/delegate/graph_executor/factory.h +++ b/mindspore/lite/src/extendrt/delegate/graph_executor/factory.h @@ -27,15 +27,12 @@ namespace mindspore { typedef std::shared_ptr (*GraphExecutorCreator)( const std::shared_ptr &config); -class GraphExecutorRegistry { +class MS_API GraphExecutorRegistry { public: GraphExecutorRegistry() = default; virtual ~GraphExecutorRegistry() = default; - static GraphExecutorRegistry *GetInstance() { - static GraphExecutorRegistry instance; - return &instance; - } + static GraphExecutorRegistry &GetInstance(); void RegGraphExecutor(const mindspore::DeviceType &device_type, const std::string &provider, GraphExecutorCreator creator) { @@ -84,7 +81,7 @@ class GraphExecutorRegistrar { public: GraphExecutorRegistrar(const mindspore::DeviceType &device_type, const std::string &provider, GraphExecutorCreator creator) { - GraphExecutorRegistry::GetInstance()->RegGraphExecutor(device_type, provider, creator); + GraphExecutorRegistry::GetInstance().RegGraphExecutor(device_type, provider, creator); } ~GraphExecutorRegistrar() = default; }; diff --git a/mindspore/lite/src/extendrt/delegate/plugin/tensorrt_executor_plugin.cc b/mindspore/lite/src/extendrt/delegate/plugin/tensorrt_executor_plugin.cc new file mode 100644 index 00000000000..a28c397d26f --- /dev/null +++ b/mindspore/lite/src/extendrt/delegate/plugin/tensorrt_executor_plugin.cc @@ -0,0 +1,95 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "extendrt/delegate/plugin/tensorrt_executor_plugin.h" +#include +#include "utils/log_adapter.h" +#if !defined(_WIN32) +#include "extendrt/cxx_api/dlutils.h" +#endif + +namespace mindspore::lite { +namespace { +constexpr auto kTensorRtPluginSoName = "libtensorrt_plugin.so"; +constexpr auto kFunCreateTRTPluginImp = "CreateTensorRTPluginImpl"; +} // namespace +TensorRTPlugin::TensorRTPlugin() = default; +TensorRTPlugin::~TensorRTPlugin() { +#if !defined(_WIN32) + MS_LOG(DEBUG) << "~AscendKernelPlugin() begin."; + DLSoClose(handle_); + MS_LOG(DEBUG) << "~AscendKernelPlugin() end."; +#endif +} + +TensorRTPlugin &TensorRTPlugin::GetInstance() { + static TensorRTPlugin instance; + return instance; +} + +bool TensorRTPlugin::Register() { +#if !defined(_WIN32) + if (is_registered_) { + return true; + } + std::string plugin_path; + auto ret = DLSoPath("libmindspore-extendrt.so", kTensorRtPluginSoName, &plugin_path); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Get real path of " << kTensorRtPluginSoName << " failed."; + return false; + } + MS_LOG(INFO) << "Find tensorrt plugin so success, path = " << plugin_path; + void *function = nullptr; + ret = DLSoOpen(plugin_path, kFunCreateTRTPluginImp, &handle_, &function); + if (ret != kSuccess) { + MS_LOG(ERROR) << "DLSoOpen failed, so path: " << plugin_path; + return false; + } + auto create_kernel_func = reinterpret_cast(function); + if (create_kernel_func == nullptr) { + MS_LOG(ERROR) << "Cast " << kFunCreateTRTPluginImp << " failed."; + return false; + } + auto plugin_impl = create_kernel_func(); + if (plugin_impl == nullptr) { + MS_LOG(ERROR) << "Create custom TensorRT kernel failed."; + return false; + } + group_size_ = plugin_impl->GetGPUGroupSize(); + rank_id_ = plugin_impl->GetRankID(); + is_registered_ = true; + MS_LOG(INFO) << "Register tensorrt plugin success."; +#endif + return true; +} + +int TensorRTPlugin::GetGPUGroupSize() { +#ifdef SUPPORT_TENSORRT + if (!is_registered_) { + Register(); + } +#endif + return group_size_; +} + +int TensorRTPlugin::GetRankID() { +#ifdef SUPPORT_TENSORRT + if (!is_registered_) { + Register(); + } +#endif + return rank_id_; +} +} // namespace mindspore::lite diff --git a/mindspore/lite/src/extendrt/delegate/plugin/tensorrt_executor_plugin.h b/mindspore/lite/src/extendrt/delegate/plugin/tensorrt_executor_plugin.h new file mode 100644 index 00000000000..e910628adfb --- /dev/null +++ b/mindspore/lite/src/extendrt/delegate/plugin/tensorrt_executor_plugin.h @@ -0,0 +1,49 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_LITE_SRC_EXTENDRT_TENSORRT_PLUGIN_H_ +#define MINDSPORE_LITE_SRC_EXTENDRT_TENSORRT_PLUGIN_H_ +#include "include/api/status.h" +#include "utils/log_adapter.h" +#include "utils/visible.h" + +namespace mindspore::lite { +class MS_API TensorRTPlugin { + public: + static TensorRTPlugin &GetInstance(); + bool Register(); + + int GetGPUGroupSize(); + int GetRankID(); + + private: + TensorRTPlugin(); + ~TensorRTPlugin(); + + void *handle_ = nullptr; + bool is_registered_ = false; + int group_size_ = 1; + int rank_id_ = 0; +}; + +class TensorRTPluginImplBase { + public: + TensorRTPluginImplBase() = default; + virtual ~TensorRTPluginImplBase() = default; + virtual int GetGPUGroupSize() const = 0; + virtual int GetRankID() const = 0; +}; +} // namespace mindspore::lite +#endif // MINDSPORE_LITE_SRC_EXTENDRT_TENSORRT_PLUGIN_H_ diff --git a/mindspore/lite/src/extendrt/delegate/tensorrt/CMakeLists.txt b/mindspore/lite/src/extendrt/delegate/tensorrt/CMakeLists.txt index 3beeb7a6515..2816a04eee2 100644 --- a/mindspore/lite/src/extendrt/delegate/tensorrt/CMakeLists.txt +++ b/mindspore/lite/src/extendrt/delegate/tensorrt/CMakeLists.txt @@ -1,6 +1,16 @@ -include_directories(${TENSORRT_PATH}/include) -include_directories(${CUDA_PATH}/include) +set(CUDA_PATH $ENV{CUDA_HOME}) +include_directories(${CCSRC_DIR}/plugin/device/gpu/kernel) +set(CUDA_VERSION 11.1) +set(CUDA_LIB_PATH ${CUDA_PATH}/lib64) include_directories(${CUDA_PATH}) +include_directories(${CUDA_PATH}/include) +find_package(CUDA) + +add_compile_definitions(GPU_TENSORRT) +set(TENSORRT_PATH $ENV{TENSORRT_PATH}) +set(TENSORRT_LIB_PATH ${TENSORRT_PATH}/lib) +include_directories(${TENSORRT_PATH}/include) + include_directories(${CCSRC_DIR}/plugin/device/cpu/kernel) include_directories(${CCSRC_DIR}/../) include_directories(${CCSRC_DIR}/plugin/device/gpu/kernel/cuda_impl/cuda_ops) @@ -46,6 +56,8 @@ file(GLOB TENSORRT_RUNTIME_SRC LIST_DIRECTORIES false ${CMAKE_CURRENT_SOURCE_DIR}/cuda_impl/*.cc ${CMAKE_CURRENT_SOURCE_DIR}/../../../extendrt/delegate/delegate_utils.cc ${CCSRC_DIR}/plugin/device/gpu/kernel/cuda_impl/cuda_ops/cuda_device_info.cc + ${CCSRC_DIR}/plugin/device/cpu/kernel/nnacl/nnacl_common.c + ${TOP_DIR}/mindspore/lite/src/common/file_utils.cc ) # include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../parameter_cache) @@ -71,15 +83,18 @@ set_target_properties(libnvinfer PROPERTIES IMPORTED_LOCATION ${TENSORRT_LIB_PAT add_library(libcublas SHARED IMPORTED) set_target_properties(libcublas PROPERTIES IMPORTED_LOCATION ${CUDA_LIB_PATH}/libcublas.so) -add_library(tensorrt_kernel_mid OBJECT ${TENSORRT_RUNTIME_SRC}) +add_library(tensorrt_plugin SHARED ${TENSORRT_RUNTIME_SRC}) -add_dependencies(tensorrt_kernel_mid fbs_src) +add_dependencies(tensorrt_plugin fbs_src) target_link_libraries( - tensorrt_kernel_mid + tensorrt_plugin libcudart libcublas libnvinfer ) add_subdirectory(cuda_impl) + +target_link_libraries(tensorrt_plugin cuda_kernel_mid gpu_distribution_collective) +target_link_libraries(tensorrt_plugin mindspore-extendrt mindspore_core) diff --git a/mindspore/lite/src/extendrt/delegate/tensorrt/op/tensorrt_executor_plugin.h b/mindspore/lite/src/extendrt/delegate/tensorrt/op/tensorrt_executor_plugin.h new file mode 100644 index 00000000000..69f0a704d25 --- /dev/null +++ b/mindspore/lite/src/extendrt/delegate/tensorrt/op/tensorrt_executor_plugin.h @@ -0,0 +1,106 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_TENSORRT_OP_TENSORRT_PLUGIN_H_ +#define MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_TENSORRT_OP_TENSORRT_PLUGIN_H_ +#include +#include +#include "src/common/log_adapter.h" +#include "include/errorcode.h" +#include "NvInferRuntimeCommon.h" +#include + +namespace mindspore::lite { +void SerializeValue(void **buffer, const void *value, size_t cpy_size); +void DeserializeValue(void const **buffer, size_t *buffer_size, void *value, size_t cpy_size); +class TensorRTPlugin : public nvinfer1::IPluginV2DynamicExt { + public: + TensorRTPlugin(const std::string &layer_name, const std::string &plugin_name, uint32_t device_id = 0) + : layer_name_(layer_name), plugin_name_(plugin_name), device_id_(device_id) {} + + // It doesn't make sense to make GeluPluginDynamic without arguments, so we delete + // default constructor. + TensorRTPlugin() = delete; + + // IPluginV2DynamicExt Methods + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, const nvinfer1::DimsExprs *inputs, int nbInputs, + nvinfer1::IExprBuilder &exprBuilder) noexcept override; + bool supportsFormatCombination(int pos, const nvinfer1::PluginTensorDesc *tensorsDesc, int nbInputs, + int nbOutputs) noexcept override; + void configurePlugin(const nvinfer1::DynamicPluginTensorDesc *in, int nbInputs, + const nvinfer1::DynamicPluginTensorDesc *out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(const nvinfer1::PluginTensorDesc *inputs, int nbInputs, + const nvinfer1::PluginTensorDesc *outputs, int nbOutputs) const noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType(int index, const nvinfer1::DataType *inputTypes, int nbInputs) const + noexcept override; + + // IPluginV2 Methods + const char *getPluginType() const noexcept override; + const char *getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void *buffer) const noexcept override; + void destroy() noexcept override; + void setPluginNamespace(const char *pluginNamespace) noexcept override; + const char *getPluginNamespace() const noexcept override; + + protected: + std::string layer_name_; + std::string name_space_; + std::string plugin_version_{"1"}; + std::string plugin_name_; + uint32_t device_id_{0}; +}; + +template +class TensorRTPluginCreater : public nvinfer1::IPluginCreator { + public: + explicit TensorRTPluginCreater(const std::string &plugin_name) : plugin_name_(plugin_name) { + // Fill PluginFieldCollection with PluginField arguments metadata + field_collection_.nbFields = fields_.size(); + field_collection_.fields = fields_.data(); + } + + const char *getPluginName() const noexcept override { return plugin_name_.c_str(); } + + const char *getPluginVersion() const noexcept override { return plugin_version_.c_str(); } + + const nvinfer1::PluginFieldCollection *getFieldNames() noexcept override { return &field_collection_; } + + void setPluginNamespace(const char *pluginNamespace) noexcept override { name_space_ = std::string(pluginNamespace); } + + const char *getPluginNamespace() const noexcept override { return name_space_.c_str(); } + + nvinfer1::IPluginV2 *createPlugin(const char *name, const nvinfer1::PluginFieldCollection *fc) noexcept { + return new (std::nothrow) T(name, fc); + } + + nvinfer1::IPluginV2 *deserializePlugin(const char *name, const void *serialData, size_t serialLength) noexcept { + return new (std::nothrow) T(name, serialData, serialLength); + } + + protected: + static nvinfer1::PluginFieldCollection field_collection_; + static std::vector fields_; + std::string name_space_; + std::string plugin_version_{"1"}; + std::string plugin_name_; +}; +} // namespace mindspore::lite +#endif // MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_TENSORRT_OP_TENSORRT_PLUGIN_H_ diff --git a/mindspore/lite/src/extendrt/delegate/tensorrt/op/tensorrt_op.h b/mindspore/lite/src/extendrt/delegate/tensorrt/op/tensorrt_op.h index c05b0e7ac68..c94c58be904 100644 --- a/mindspore/lite/src/extendrt/delegate/tensorrt/op/tensorrt_op.h +++ b/mindspore/lite/src/extendrt/delegate/tensorrt/op/tensorrt_op.h @@ -26,7 +26,7 @@ #include "include/errorcode.h" #include "src/extendrt/delegate/tensorrt/tensorrt_context.h" #include "src/extendrt/delegate/tensorrt/tensorrt_utils.h" -#include "src/litert/delegate/auto_registration_factory.h" +#include "src/extendrt/delegate/tensorrt/op_registration_factory.h" #include "src/extendrt/delegate/tensorrt/tensor_info.h" #include "src/common/log_util.h" #include "ops/base_operator.h" diff --git a/mindspore/lite/src/extendrt/delegate/tensorrt/distribution/distribution_base_impl.cc b/mindspore/lite/src/extendrt/delegate/tensorrt/op_registration_factory.cc similarity index 58% rename from mindspore/lite/src/extendrt/delegate/tensorrt/distribution/distribution_base_impl.cc rename to mindspore/lite/src/extendrt/delegate/tensorrt/op_registration_factory.cc index 2a668207b8b..e23fd66b2f2 100644 --- a/mindspore/lite/src/extendrt/delegate/tensorrt/distribution/distribution_base_impl.cc +++ b/mindspore/lite/src/extendrt/delegate/tensorrt/op_registration_factory.cc @@ -1,5 +1,5 @@ /** - * Copyright 2021 Huawei Technologies Co., Ltd + * Copyright 2022 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,16 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#include "src/extendrt/delegate/tensorrt/distribution/distribution_base.h" -#include -#include -#include -#include "plugin/device/gpu/hal/device/distribution/collective_wrapper.h" -#include "src/extendrt/delegate/tensorrt/tensorrt_utils.h" +#include "extendrt/delegate/tensorrt/op_registration_factory.h" +#include "extendrt/delegate/tensorrt/op/tensorrt_op.h" namespace mindspore::lite { -int GetGPUGroupSize() { return GetGroupSize(NCCL_WORLD_GROUP); } - -int GetRankID() { return GetRankIDByGroup(NCCL_WORLD_GROUP); } +template <> +TensorRTRegistrationFactory &TensorRTRegistrationFactory::Get() { + static TensorRTRegistrationFactory obj; + return obj; +} } // namespace mindspore::lite diff --git a/mindspore/lite/src/extendrt/delegate/tensorrt/op_registration_factory.h b/mindspore/lite/src/extendrt/delegate/tensorrt/op_registration_factory.h new file mode 100644 index 00000000000..705ae8cdba8 --- /dev/null +++ b/mindspore/lite/src/extendrt/delegate/tensorrt/op_registration_factory.h @@ -0,0 +1,54 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_AUTO_REGISTRATION_FACTORY_H_ +#define MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_AUTO_REGISTRATION_FACTORY_H_ + +#include + +namespace mindspore::lite { +template +class AutoRegistrationFactory { + public: + struct AutoRegister { + AutoRegister(KeyType k, CreatorType creator) { + AutoRegistrationFactory::Get().Insert(k, creator); + } + }; + static AutoRegistrationFactory &Get(); + bool HasKey(KeyType k) const { return key2creator_.find(k) != key2creator_.end(); } + CreatorType GetCreator(KeyType k) { return key2creator_[k]; } + + private: + bool Insert(KeyType k, CreatorType creator) { + if (HasKey(k)) { + return false; + } + return key2creator_.emplace(k, creator).second; + } + std::unordered_map key2creator_; +}; + +#define AUTO_REGISTRATION_FACTORY_JOIN(a, b) a##b + +#define AUTO_REGISTRATION_FACTORY_UNIQUE_NAME_JOIN(a, b) AUTO_REGISTRATION_FACTORY_JOIN(a, b) + +#define AUTO_REGISTRATION_FACTORY_UNIQUE_NAME AUTO_REGISTRATION_FACTORY_UNIQUE_NAME_JOIN(g_, __COUNTER__) + +#define REGISTER_CLASS_CREATOR(KeyType, k, CreatorType, creator) \ + static AutoRegistrationFactory::AutoRegister AUTO_REGISTRATION_FACTORY_UNIQUE_NAME(k, creator); +} // namespace mindspore::lite + +#endif // MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_AUTO_REGISTRATION_FACTORY_H_ diff --git a/mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_graph_executor.cc b/mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_graph_executor.cc index 02f00d22746..6d2c3c812b9 100644 --- a/mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_graph_executor.cc +++ b/mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_graph_executor.cc @@ -24,7 +24,6 @@ #include #include "ccsrc/kernel/kernel.h" #include "src/extendrt/delegate/delegate_utils.h" -#include "src/litert/delegate/auto_registration_factory.h" #include "src/extendrt/delegate/graph_executor/factory.h" #include "ccsrc/kernel/common_utils.h" diff --git a/mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_plugin_impl.cc b/mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_plugin_impl.cc new file mode 100644 index 00000000000..40ea316a892 --- /dev/null +++ b/mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_plugin_impl.cc @@ -0,0 +1,28 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "extendrt/delegate/tensorrt/tensorrt_plugin_impl.h" +#include "extendrt/delegate/tensorrt/distribution/distribution_base.h" +// #include "plugin/device/gpu/hal/device/distribution/collective_wrapper.h" + +namespace mindspore::lite { +int TensorRTPluginImpl::GetGPUGroupSize() const { return 1; } // GetGroupSize(NCCL_WORLD_GROUP); + +int TensorRTPluginImpl::GetRankID() const { return 0; } // GetRankIDByGroup(NCCL_WORLD_GROUP); +} // namespace mindspore::lite + +mindspore::lite::TensorRTPluginImplBase *CreateTensorRTPluginImpl() { + return new mindspore::lite::TensorRTPluginImpl(); +} diff --git a/mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_plugin_impl.h b/mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_plugin_impl.h new file mode 100644 index 00000000000..15f91fe56ff --- /dev/null +++ b/mindspore/lite/src/extendrt/delegate/tensorrt/tensorrt_plugin_impl.h @@ -0,0 +1,33 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_LITE_SRC_EXTENDRT_TENSORRT_PLUGIN_IMPL_H_ +#define MINDSPORE_LITE_SRC_EXTENDRT_TENSORRT_PLUGIN_IMPL_H_ +#include "include/api/status.h" +#include "utils/log_adapter.h" +#include "extendrt/delegate/plugin/tensorrt_executor_plugin.h" + +namespace mindspore::lite { +class TensorRTPluginImpl : public TensorRTPluginImplBase { + public: + TensorRTPluginImpl() = default; + ~TensorRTPluginImpl() = default; + int GetGPUGroupSize() const; + int GetRankID() const; +}; +} // namespace mindspore::lite + +extern "C" MS_API mindspore::lite::TensorRTPluginImplBase *CreateTensorRTPluginImpl(); +#endif // MINDSPORE_LITE_SRC_EXTENDRT_KERNEL_ASCEND_ASCEND_KERNEL_API_H_ diff --git a/mindspore/lite/src/extendrt/infer_session.cc b/mindspore/lite/src/extendrt/infer_session.cc index f552cb5d4c3..43b9c9a27f5 100644 --- a/mindspore/lite/src/extendrt/infer_session.cc +++ b/mindspore/lite/src/extendrt/infer_session.cc @@ -27,6 +27,7 @@ #include "extendrt/delegate/factory.h" #include "extendrt/delegate/graph_executor/factory.h" #include "extendrt/session/factory.h" +#include "extendrt/delegate/plugin/tensorrt_executor_plugin.h" namespace mindspore { static const std::vector ms_infer_cut_list = {prim::kPrimReturn, prim::kPrimPartial, @@ -85,8 +86,34 @@ std::vector DefaultInferSession::GetInputNames() { return std::vect tensor::TensorPtr DefaultInferSession::GetOutputByTensorName(const std::string &tensorName) { return nullptr; } tensor::TensorPtr DefaultInferSession::GetInputByTensorName(const std::string &name) { return nullptr; } std::shared_ptr InferSession::CreateSession(const std::shared_ptr context) { + HandleGPUContext(context); auto config = SelectSessionArg(context); - return SessionRegistry::GetInstance()->GetSession(config.type_, config); + return SessionRegistry::GetInstance().GetSession(config.type_, config); +} + +void InferSession::HandleGPUContext(const std::shared_ptr &context) { + if (!context) { + return; + } + constexpr auto default_gpu_provider = "tensorrt"; + auto device_infos = context->MutableDeviceInfo(); + for (auto &device_info : device_infos) { + if (!device_info || device_info->GetDeviceType() != kGPU) { + continue; + } + auto gpu_device = device_info->Cast(); + if (!gpu_device) { + continue; + } + auto provider = gpu_device->GetProvider(); + if (provider.empty() || provider == default_gpu_provider) { + if (!lite::TensorRTPlugin::GetInstance().Register()) { + MS_LOG_WARNING << "Failed to register TensorRT plugin"; + return; + } + gpu_device->SetProvider(default_gpu_provider); + } + } } SessionConfig InferSession::SelectSessionArg(const std::shared_ptr &context) { @@ -102,7 +129,7 @@ SessionConfig InferSession::SelectSessionArg(const std::shared_ptr &con // delegate init MS_EXCEPTION_IF_NULL(device_context); // get graph executor delegate - auto delegate = mindspore::DelegateRegistry::GetInstance()->GetDelegate( + auto delegate = mindspore::DelegateRegistry::GetInstance().GetDelegate( device_context->GetDeviceType(), device_context->GetProvider(), delegate_config); if (delegate == nullptr) { continue; diff --git a/mindspore/lite/src/extendrt/infer_session.h b/mindspore/lite/src/extendrt/infer_session.h index 8224ebf6cee..b83e45af37f 100644 --- a/mindspore/lite/src/extendrt/infer_session.h +++ b/mindspore/lite/src/extendrt/infer_session.h @@ -51,6 +51,7 @@ class InferSession : public std::enable_shared_from_this { protected: FuncGraphPtr graph_; compile::GraphPartitionPtr partition_; + static void HandleGPUContext(const std::shared_ptr &context); }; } // namespace mindspore #endif diff --git a/mindspore/lite/src/extendrt/mock/lite_runtime/CMakeLists.txt b/mindspore/lite/src/extendrt/mock/lite_runtime/CMakeLists.txt index e8eaf22e055..165000a6c63 100644 --- a/mindspore/lite/src/extendrt/mock/lite_runtime/CMakeLists.txt +++ b/mindspore/lite/src/extendrt/mock/lite_runtime/CMakeLists.txt @@ -301,12 +301,6 @@ if(MSLITE_ENABLE_MINDRT) target_link_libraries(mindspore-infer-lite mindrt_mid) endif() -if(SUPPORT_TENSORRT) - target_link_libraries(mindspore-infer-lite tensorrt_kernel_mid cuda_kernel_mid gpu_distribution_collective) -else() - target_link_libraries(mindspore-infer-lite tensorrt_stub) -endif() - if(MSLITE_GPU_BACKEND STREQUAL opencl) target_link_libraries(mindspore-infer-lite opencl_kernel_mid) endif() diff --git a/mindspore/lite/src/extendrt/session/factory.cc b/mindspore/lite/src/extendrt/session/factory.cc new file mode 100644 index 00000000000..c7f5df6a98e --- /dev/null +++ b/mindspore/lite/src/extendrt/session/factory.cc @@ -0,0 +1,42 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "extendrt/session/factory.h" +#include +#include + +#include "extendrt/session/type.h" +#include "extendrt/infer_session.h" + +namespace mindspore { +SessionRegistry &SessionRegistry::GetInstance() { + static SessionRegistry instance; + return instance; +} + +void SessionRegistry::RegSession(const mindspore::SessionType &session_type, + std::function(const SessionConfig &)> creator) { + session_map_[session_type] = creator; +} + +std::shared_ptr SessionRegistry::GetSession(const mindspore::SessionType &session_type, + const SessionConfig &config) { + auto it = session_map_.find(session_type); + if (it == session_map_.end()) { + return nullptr; + } + return it->second(config); +} +} // namespace mindspore diff --git a/mindspore/lite/src/extendrt/session/factory.h b/mindspore/lite/src/extendrt/session/factory.h index e9af52f0f51..920b80a61ce 100644 --- a/mindspore/lite/src/extendrt/session/factory.h +++ b/mindspore/lite/src/extendrt/session/factory.h @@ -28,23 +28,12 @@ class SessionRegistry { SessionRegistry() = default; virtual ~SessionRegistry() = default; - static SessionRegistry *GetInstance() { - static SessionRegistry instance; - return &instance; - } + static SessionRegistry &GetInstance(); void RegSession(const mindspore::SessionType &session_type, - std::function(const SessionConfig &)> creator) { - session_map_[session_type] = creator; - } + std::function(const SessionConfig &)> creator); - std::shared_ptr GetSession(const mindspore::SessionType &session_type, const SessionConfig &config) { - auto it = session_map_.find(session_type); - if (it == session_map_.end()) { - return nullptr; - } - return it->second(config); - } + std::shared_ptr GetSession(const mindspore::SessionType &session_type, const SessionConfig &config); private: mindspore::HashMap(const SessionConfig &)>> session_map_; @@ -54,7 +43,7 @@ class SessionRegistrar { public: SessionRegistrar(const mindspore::SessionType &session_type, std::function(const SessionConfig &)> creator) { - SessionRegistry::GetInstance()->RegSession(session_type, creator); + SessionRegistry::GetInstance().RegSession(session_type, creator); } ~SessionRegistrar() = default; }; diff --git a/mindspore/lite/src/extendrt/session/graph_executor_session.cc b/mindspore/lite/src/extendrt/session/graph_executor_session.cc index a06f36c3b22..b95beac3dd8 100644 --- a/mindspore/lite/src/extendrt/session/graph_executor_session.cc +++ b/mindspore/lite/src/extendrt/session/graph_executor_session.cc @@ -23,9 +23,6 @@ #include "src/extendrt/utils/kernel_build_utils.h" namespace mindspore { -namespace { -constexpr auto kAttrGraphOutputNames = "graph_output_names"; -} Status GraphExecutorSession::Init(const std::shared_ptr context) { MS_LOG(INFO) << "GraphExecutorSession::Init"; kernel_graph_utils_ = std::make_shared(); @@ -44,13 +41,6 @@ Status GraphExecutorSession::CompileGraph(FuncGraphPtr graph, const void *data, if (graph_executor_->CompileGraph(kernel_graph_, options_)) { kernel_graph_utils_->GetModelInputsInfo(kernel_graph_->graph_id(), &inputs_, &input_names_); kernel_graph_utils_->GetModelOutputsInfo(kernel_graph_->graph_id(), &outputs_, &output_names_); - - if (graph->has_attr(kAttrGraphOutputNames)) { - auto names_attr = graph->get_attr(kAttrGraphOutputNames); - MS_EXCEPTION_IF_NULL(names_attr); - output_names_ = GetValue>(names_attr); - MS_LOG_INFO << "get output names from graph attr, output names: " << output_names_; - } return kSuccess; } return kCoreFailed; @@ -91,7 +81,7 @@ tensor::TensorPtr GraphExecutorSession::GetOutputByTensorName(const std::string tensor::TensorPtr GraphExecutorSession::GetInputByTensorName(const std::string &name) { for (size_t i = 0; i < input_names_.size(); i++) { if (input_names_[i] == name) { - return outputs_[i]; + return inputs_[i]; } } return nullptr; diff --git a/mindspore/lite/tools/benchmark/CMakeLists.txt b/mindspore/lite/tools/benchmark/CMakeLists.txt index 4ef46840738..51ec5489926 100644 --- a/mindspore/lite/tools/benchmark/CMakeLists.txt +++ b/mindspore/lite/tools/benchmark/CMakeLists.txt @@ -57,7 +57,6 @@ set(COMMON_SRC ${CMAKE_CURRENT_SOURCE_DIR}/../common/opengl_util.cc ${CMAKE_CURRENT_SOURCE_DIR}/../../src/common/file_utils.cc ${CMAKE_CURRENT_SOURCE_DIR}/../../src/common/utils.cc - ${CMAKE_CURRENT_SOURCE_DIR}/../../../ccsrc/plugin/device/cpu/kernel/nnacl/nnacl_common.c ) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../../lite) diff --git a/mindspore/lite/tools/converter/CMakeLists.txt b/mindspore/lite/tools/converter/CMakeLists.txt index 9777249c542..180d9ecf4f4 100644 --- a/mindspore/lite/tools/converter/CMakeLists.txt +++ b/mindspore/lite/tools/converter/CMakeLists.txt @@ -152,6 +152,7 @@ set(LITE_SRC ${API_SRC} ${SRC_DIR}/litert/pack_weight_manager.cc ${SRC_DIR}/litert/huffman_decode.cc ${SRC_DIR}/extendrt/delegate/tensorrt/distribution/distribution_base.cc + ${SRC_DIR}/extendrt/delegate/plugin/tensorrt_executor_plugin.cc ${LITE_DIR}/src/extendrt/mock/lite_runtime/populate/base_operator_populate_register.cc ${SRC_DIR}/control_flow/control_flow_scheduler.cc ${SRC_DIR}/control_flow/control_subgraph_creator.cc diff --git a/mindspore/lite/tools/mindir_exporter/mindir_serializer.cc b/mindspore/lite/tools/mindir_exporter/mindir_serializer.cc index 17be2989386..fec51621dbe 100644 --- a/mindspore/lite/tools/mindir_exporter/mindir_serializer.cc +++ b/mindspore/lite/tools/mindir_exporter/mindir_serializer.cc @@ -105,10 +105,6 @@ int MindIRSerializer::Save(const std::shared_ptr ¶m, const Fu MS_LOG(ERROR) << "parse path failed."; return ret; } - // todo - func_graph->set_attr("graph_output_names", - MakeValue(ConverterInnerContext::GetInstance()->GetGraphOutputTensorNames())); - ret = RemoveQuantParameterHolder(func_graph); if (ret != RET_OK && ret != RET_NO_CHANGE) { MS_LOG(ERROR) << "remove quant parameter holder failed.";