From b25cc0256f866cceec86474cf695931227550e49 Mon Sep 17 00:00:00 2001 From: zhoufeng Date: Wed, 16 Nov 2022 14:51:04 +0800 Subject: [PATCH] support all cuda version in a whl package Signed-off-by: zhoufeng --- cmake/package.cmake | 77 +++++++++---------- cmake/package_mac.cmake | 18 ++--- cmake/package_script.cmake | 1 - cmake/package_tar.cmake | 2 +- cmake/package_win.cmake | 27 ++----- mindspore/ccsrc/CMakeLists.txt | 4 +- mindspore/ccsrc/cxx_api/CMakeLists.txt | 66 ++++++++-------- .../ccsrc/minddata/dataset/CMakeLists.txt | 7 +- .../dataset/engine/cache/CMakeLists.txt | 4 +- .../ccsrc/minddata/mindrecord/CMakeLists.txt | 2 +- .../ccsrc/plugin/device/ascend/CMakeLists.txt | 28 +++---- .../hal/hccl_adapter/plugin/CMakeLists.txt | 8 +- .../ascend/kernel/aicpu/aicpu_kernel_load.cc | 5 +- .../kernel/aicpu/aicpu_ops/CMakeLists.txt | 4 +- .../ccsrc/plugin/device/gpu/CMakeLists.txt | 12 ++- .../device/gpu/hal/device/cuda_driver.cc | 7 ++ .../gpu/hal/hardware/gpu_device_context.cc | 18 ++++- .../mindspore/communication/_comm_helper.py | 7 +- .../communication/_hccl_management.py | 2 +- scripts/build/merge_whl_package.sh | 1 - setup.py | 14 +--- 21 files changed, 153 insertions(+), 161 deletions(-) diff --git a/cmake/package.cmake b/cmake/package.cmake index 333d1cf3dea..6b0d21d4a8f 100644 --- a/cmake/package.cmake +++ b/cmake/package.cmake @@ -19,45 +19,18 @@ set(CPACK_PYTHON_VERSION ${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}) if(ENABLE_GPU) set(CPACK_MS_BACKEND "ms") - set(CPACK_MS_TARGET "gpu or cpu") - if(BUILD_DEV_MODE) - # providing cuda11 version of dev package only - set(CPACK_MS_PACKAGE_NAME "mindspore-cuda11-dev") - else() - set(CPACK_MS_PACKAGE_NAME "mindspore-gpu") - endif() elseif(ENABLE_D) set(CPACK_MS_BACKEND "ms") - set(CPACK_MS_TARGET "ascend or cpu") - if(BUILD_DEV_MODE) - set(CPACK_MS_PACKAGE_NAME "mindspore-ascend-dev") - else() - set(CPACK_MS_PACKAGE_NAME "mindspore-ascend") - endif() elseif(ENABLE_CPU) set(CPACK_MS_BACKEND "ms") - set(CPACK_MS_TARGET "cpu") - if(BUILD_DEV_MODE) - set(CPACK_MS_PACKAGE_NAME "mindspore-dev") - else() - set(CPACK_MS_PACKAGE_NAME "mindspore") - endif() -elseif(ENABLE_ACL) - set(CPACK_MS_BACKEND "debug") - set(CPACK_MS_TARGET "ascend or gpu or cpu") - if(BUILD_DEV_MODE) - set(CPACK_MS_PACKAGE_NAME "mindspore-ascend-dev") - else() - set(CPACK_MS_PACKAGE_NAME "mindspore-ascend") - endif() else() set(CPACK_MS_BACKEND "debug") - set(CPACK_MS_TARGET "ascend or gpu or cpu") - if(BUILD_DEV_MODE) - set(CPACK_MS_PACKAGE_NAME "mindspore-dev") - else() - set(CPACK_MS_PACKAGE_NAME "mindspore") - endif() +endif() +if(BUILD_DEV_MODE) + # providing cuda11 version of dev package only + set(CPACK_MS_PACKAGE_NAME "mindspore-dev") +else() + set(CPACK_MS_PACKAGE_NAME "mindspore") endif() include(CPack) @@ -114,9 +87,10 @@ endif() if(ENABLE_GPU) install( - TARGETS mindspore_gpu + TARGETS mindspore_gpu LIBRARY DESTINATION ${INSTALL_PLUGIN_DIR} COMPONENT mindspore + NAMELINK_SKIP ) endif() @@ -201,17 +175,17 @@ if(ENABLE_GPU) if(ENABLE_MPI) install( TARGETS nvidia_collective - DESTINATION ${INSTALL_LIB_DIR} + DESTINATION ${INSTALL_PLUGIN_DIR}/gpu${CUDA_VERSION} COMPONENT mindspore ) if(CMAKE_SYSTEM_NAME MATCHES "Linux" AND GPU_BACKEND_CUDA) - install(FILES ${nccl_LIBPATH}/libnccl.so.2.7.6 DESTINATION ${INSTALL_LIB_DIR} + install(FILES ${nccl_LIBPATH}/libnccl.so.2.7.6 DESTINATION ${INSTALL_PLUGIN_DIR}/gpu${CUDA_VERSION} RENAME libnccl.so.2 COMPONENT mindspore) endif() endif() install( TARGETS cuda_ops - DESTINATION ${INSTALL_LIB_DIR} + DESTINATION ${INSTALL_PLUGIN_DIR}/gpu${CUDA_VERSION} COMPONENT mindspore ) endif() @@ -220,7 +194,7 @@ if(ENABLE_D) if(ENABLE_MPI) install( TARGETS ascend_collective - DESTINATION ${INSTALL_LIB_DIR} + DESTINATION ${INSTALL_PLUGIN_DIR}/ascend COMPONENT mindspore ) endif() @@ -245,14 +219,14 @@ if(ENABLE_D OR ENABLE_ACL) if(ENABLE_D) install( TARGETS hccl_plugin - DESTINATION ${INSTALL_LIB_DIR} + DESTINATION ${INSTALL_PLUGIN_DIR}/ascend COMPONENT mindspore ) endif() if(ENABLE_ACL) install( TARGETS dvpp_utils - DESTINATION ${INSTALL_LIB_DIR} + DESTINATION ${INSTALL_PLUGIN_DIR}/ascend COMPONENT mindspore ) endif() @@ -329,6 +303,29 @@ if(ENABLE_AKG AND CMAKE_SYSTEM_NAME MATCHES "Linux") DESTINATION ${INSTALL_PY_DIR}/ COMPONENT mindspore ) + if(ENABLE_CPU AND NOT ENABLE_GPU AND NOT ENABLE_D) + install( + TARGETS akg + DESTINATION ${INSTALL_PLUGIN_DIR}/cpu + COMPONENT mindspore + ) + endif() + + if(ENABLE_GPU) + install( + TARGETS akg + DESTINATION ${INSTALL_PLUGIN_DIR}/gpu${CUDA_VERSION} + COMPONENT mindspore + ) + endif() + + if(ENABLE_D) + install( + TARGETS akg + DESTINATION ${INSTALL_PLUGIN_DIR}/ascend + COMPONENT mindspore + ) + endif() endif() if(EXISTS ${CMAKE_SOURCE_DIR}/mindspore/python/mindspore/dataset) diff --git a/cmake/package_mac.cmake b/cmake/package_mac.cmake index 3a37b6122b4..7c5550b45ff 100644 --- a/cmake/package_mac.cmake +++ b/cmake/package_mac.cmake @@ -18,23 +18,15 @@ set(CPACK_PYTHON_EXE ${Python3_EXECUTABLE}) set(CPACK_PYTHON_VERSION ${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}) set(CPACK_OSX_DEPLOYMENT_TARGET ${CMAKE_OSX_DEPLOYMENT_TARGET}) - if(ENABLE_CPU) set(CPACK_MS_BACKEND "ms") - set(CPACK_MS_TARGET "cpu") - if(BUILD_DEV_MODE) - set(CPACK_MS_PACKAGE_NAME "mindspore-dev") - else() - set(CPACK_MS_PACKAGE_NAME "mindspore") - endif() else() set(CPACK_MS_BACKEND "debug") - set(CPACK_MS_TARGET "ascend or gpu or cpu") - if(BUILD_DEV_MODE) - set(CPACK_MS_PACKAGE_NAME "mindspore-dev") - else() - set(CPACK_MS_PACKAGE_NAME "mindspore") - endif() +endif() +if(BUILD_DEV_MODE) + set(CPACK_MS_PACKAGE_NAME "mindspore-dev") +else() + set(CPACK_MS_PACKAGE_NAME "mindspore") endif() include(CPack) diff --git a/cmake/package_script.cmake b/cmake/package_script.cmake index 1f229ca7a3e..98673808ae8 100644 --- a/cmake/package_script.cmake +++ b/cmake/package_script.cmake @@ -70,7 +70,6 @@ execute_process( string(REPLACE " " "" GIT_COMMIT_ID ${GIT_COMMIT_ID}) set(ENV{BACKEND_POLICY} ${CPACK_MS_BACKEND}) -set(ENV{BACKEND_TARGET} ${CPACK_MS_TARGET}) set(ENV{MS_PACKAGE_NAME} ${CPACK_MS_PACKAGE_NAME}) set(ENV{COMMIT_ID} ${GIT_COMMIT_ID}) diff --git a/cmake/package_tar.cmake b/cmake/package_tar.cmake index 82e7a5865de..345d5fc40b3 100644 --- a/cmake/package_tar.cmake +++ b/cmake/package_tar.cmake @@ -100,7 +100,7 @@ if(ENABLE_MINDDATA) if(ENABLE_ACL) install( TARGETS dvpp_utils - DESTINATION ${INSTALL_LIB_DIR} + DESTINATION ${INSTALL_PLUGIN_DIR} COMPONENT mindspore ) endif() diff --git a/cmake/package_win.cmake b/cmake/package_win.cmake index 6a6958c7bc2..b71bb2372d7 100644 --- a/cmake/package_win.cmake +++ b/cmake/package_win.cmake @@ -17,29 +17,15 @@ set(CPACK_PYTHON_VERSION ${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}) if(ENABLE_GPU) set(CPACK_MS_BACKEND "ms") - set(CPACK_MS_TARGET "gpu or cpu") - if(BUILD_DEV_MODE) - # providing cuda11 version of dev package only - set(CPACK_MS_PACKAGE_NAME "mindspore_cuda11_dev") - else() - set(CPACK_MS_PACKAGE_NAME "mindspore_gpu") - endif() elseif(ENABLE_CPU) set(CPACK_MS_BACKEND "ms") - set(CPACK_MS_TARGET "cpu") - if(BUILD_DEV_MODE) - set(CPACK_MS_PACKAGE_NAME "mindspore_dev") - else() - set(CPACK_MS_PACKAGE_NAME "mindspore") - endif() else() set(CPACK_MS_BACKEND "debug") - set(CPACK_MS_TARGET "ascend or gpu or cpu") - if(BUILD_DEV_MODE) - set(CPACK_MS_PACKAGE_NAME "mindspore_dev") - else() - set(CPACK_MS_PACKAGE_NAME "mindspore") - endif() +endif() +if(BUILD_DEV_MODE) + set(CPACK_MS_PACKAGE_NAME "mindspore_dev") +else() + set(CPACK_MS_PACKAGE_NAME "mindspore") endif() include(CPack) @@ -213,9 +199,10 @@ if(ENABLE_GPU) COMPONENT mindspore ) install( - TARGETS mindspore_gpu + TARGETS mindspore_gpu LIBRARY DESTINATION ${INSTALL_PLUGIN_DIR} COMPONENT mindspore + NAMELINK_SKIP ) endif() diff --git a/mindspore/ccsrc/CMakeLists.txt b/mindspore/ccsrc/CMakeLists.txt index 29322dbfef6..b3107f745ad 100644 --- a/mindspore/ccsrc/CMakeLists.txt +++ b/mindspore/ccsrc/CMakeLists.txt @@ -162,7 +162,6 @@ if(ENABLE_GPU AND GPU_BACKEND_CUDA) if(NOT CUPTI_INCLUDE_DIRS OR CUPTI_INCLUDE_DIRS STREQUAL "") set(CUPTI_INCLUDE_DIRS ${CUDA_PATH}/extras/CUPTI/include) endif() - set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:${CUDA_PATH}/lib64) message("CUDA_PATH: ${CUDA_PATH}") message("CUDA_INCLUDE_DIRS: ${CUDA_INCLUDE_DIRS}") message("CUDNN_INCLUDE_PATH: ${CUDNN_INCLUDE_PATH}") @@ -512,7 +511,6 @@ elseif(CMAKE_SYSTEM_NAME MATCHES "Windows") else() MESSAGE(FATAL_ERROR "other platform: ${CMAKE_SYSTEM_NAME}") endif() -set(MINDSPORE_RPATH ${ORIGIN_PATH}/lib:${ORIGIN_PATH}/lib/plugin:${ORIGIN_PATH}/..:${MINDSPORE_RPATH}) if(ENABLE_D) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/plugin/device/ascend) @@ -553,7 +551,7 @@ else() target_link_libraries(_c_expression PRIVATE mindspore::pybind11_module) endif() -set_target_properties(_c_expression PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) +set_target_properties(_c_expression PROPERTIES INSTALL_RPATH ${ORIGIN_PATH}/lib:${ORIGIN_PATH}/lib/plugin) if(USE_GLOG) target_link_libraries(_c_expression PRIVATE mindspore::glog) endif() diff --git a/mindspore/ccsrc/cxx_api/CMakeLists.txt b/mindspore/ccsrc/cxx_api/CMakeLists.txt index 22a07638587..12f19c74ff3 100644 --- a/mindspore/ccsrc/cxx_api/CMakeLists.txt +++ b/mindspore/ccsrc/cxx_api/CMakeLists.txt @@ -54,7 +54,7 @@ if(ENABLE_D OR ENABLE_GPU) list(APPEND MSLIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/akg_kernel_register.cc) endif() -if(ENABLE_ACL AND NOT BUILD_LITE) +if(NOT BUILD_LITE) list(APPEND MSLIB_SRC_DEPEND ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/frontend/parallel/strategy_checkpoint/parallel_strategy_checkpoint.cc) list(APPEND MSLIB_SRC_DEPEND ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/frontend/parallel/tensor_layout/array.cc) @@ -108,33 +108,34 @@ elseif(MODE_ASCEND_ACL) if(MS_BUILD_GRPC) target_link_libraries(mindspore_shared_lib PRIVATE mindspore::grpc++) endif() + set_target_properties(mindspore_shared_lib PROPERTIES OUTPUT_NAME mindspore) +else() + if(CMAKE_SYSTEM_NAME MATCHES "Linux") + # wheel package and ut + add_library(api_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/types.cc + ${CMAKE_CURRENT_SOURCE_DIR}/context.cc + ${CMAKE_CURRENT_SOURCE_DIR}/cell.cc + ${CMAKE_CURRENT_SOURCE_DIR}/serialization.cc + ${CMAKE_CURRENT_SOURCE_DIR}/graph/graph.cc + ${CMAKE_CURRENT_SOURCE_DIR}/graph/graph_data.cc + ${CMAKE_CURRENT_SOURCE_DIR}/model/model.cc + ${CMAKE_SOURCE_DIR}/mindspore/core/utils/status.cc + ) + target_link_libraries(api_lib PRIVATE mindspore_shared_lib -Wl,--no-as-needed ${PYTHON_LIBRARIES}) + set_target_properties(api_lib PROPERTIES OUTPUT_NAME mindspore) + set_target_properties(api_lib PROPERTIES INSTALL_RPATH "$ORIGIN") + endif() endif() if(ENABLE_D OR ENABLE_GPU) target_link_libraries(mindspore_shared_lib PRIVATE -Wl,--as-needed ${SECUREC_LIBRARY} mindspore_backend mindspore_core mindspore_common proto_input mindspore::protobuf) - - if(CMAKE_SYSTEM_NAME MATCHES "Linux") - add_library(api_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/types.cc - ${CMAKE_CURRENT_SOURCE_DIR}/context.cc - ${CMAKE_CURRENT_SOURCE_DIR}/cell.cc - ${CMAKE_CURRENT_SOURCE_DIR}/serialization.cc - ${CMAKE_CURRENT_SOURCE_DIR}/graph/graph.cc - ${CMAKE_CURRENT_SOURCE_DIR}/graph/graph_data.cc - ${CMAKE_CURRENT_SOURCE_DIR}/model/model.cc - ${CMAKE_SOURCE_DIR}/mindspore/core/utils/status.cc - ) - target_link_libraries(api_lib PRIVATE mindspore_shared_lib -Wl,--no-as-needed ${PYTHON_LIBRARIES}) - set_target_properties(api_lib PROPERTIES OUTPUT_NAME mindspore) - set_target_properties(api_lib PROPERTIES INSTALL_RPATH "$ORIGIN") - endif() else() if(BUILD_LITE) target_link_libraries(mindspore_shared_lib PRIVATE mindspore_core ${SECUREC_LIBRARY}) else() target_link_libraries(mindspore_shared_lib PRIVATE ${PY3_LIBG} ${SECUREC_LIBRARY} mindspore_backend mindspore mindspore_core mindspore_common proto_input mindspore::protobuf) - set_target_properties(mindspore_shared_lib PROPERTIES OUTPUT_NAME mindspore) endif() endif() @@ -175,20 +176,23 @@ endif() if(CMAKE_SYSTEM_NAME MATCHES "Linux") # duplicate, should be remove after backend decoupling is done - set(MINDSPORE_RPATH $ORIGIN) + set(MINDSPORE_SHARED_LIB_RPATH $ORIGIN) if(MODE_ASCEND_ACL) - set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/nnae/latest/lib64) - set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/ascend-toolkit/latest/lib64) - set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/latest/lib64) - set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/opp/op_impl/built-in/ai_core/tbe/op_tiling) - set(MINDSPORE_RPATH - ${MINDSPORE_RPATH}:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe/op_tiling) - set(MINDSPORE_RPATH - ${MINDSPORE_RPATH}:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe/op_tiling) - set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/latest/opp/op_impl/built-in/ai_core/tbe/op_tiling) - set(MINDSPORE_RPATH - ${MINDSPORE_RPATH}:/usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/op_tiling) - set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/latest/opp/built-in/op_impl/ai_core/tbe/op_tiling) + set(MINDSPORE_SHARED_LIB_RPATH ${MINDSPORE_SHARED_LIB_RPATH}:/usr/local/Ascend/nnae/latest/lib64) + set(MINDSPORE_SHARED_LIB_RPATH ${MINDSPORE_SHARED_LIB_RPATH}:/usr/local/Ascend/ascend-toolkit/latest/lib64) + set(MINDSPORE_SHARED_LIB_RPATH ${MINDSPORE_SHARED_LIB_RPATH}:/usr/local/Ascend/latest/lib64) + set(MINDSPORE_SHARED_LIB_RPATH + ${MINDSPORE_SHARED_LIB_RPATH}:/usr/local/Ascend/opp/op_impl/built-in/ai_core/tbe/op_tiling) + set(MINDSPORE_SHARED_LIB_RPATH + ${MINDSPORE_SHARED_LIB_RPATH}:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe/op_tiling) + set(MINDSPORE_SHARED_LIB_RPATH "${MINDSPORE_SHARED_LIB_RPATH}\ + :/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe/op_tiling") + set(MINDSPORE_SHARED_LIB_RPATH + ${MINDSPORE_SHARED_LIB_RPATH}:/usr/local/Ascend/latest/opp/op_impl/built-in/ai_core/tbe/op_tiling) + set(MINDSPORE_SHARED_LIB_RPATH "${MINDSPORE_SHARED_LIB_RPATH}\ + :/usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/op_tiling") + set(MINDSPORE_SHARED_LIB_RPATH + ${MINDSPORE_SHARED_LIB_RPATH}:/usr/local/Ascend/latest/opp/built-in/op_impl/ai_core/tbe/op_tiling) endif() - set_target_properties(mindspore_shared_lib PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) + set_target_properties(mindspore_shared_lib PROPERTIES INSTALL_RPATH ${MINDSPORE_SHARED_LIB_RPATH}) endif() diff --git a/mindspore/ccsrc/minddata/dataset/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/CMakeLists.txt index 55d98e10311..8f5dcf77f02 100644 --- a/mindspore/ccsrc/minddata/dataset/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/CMakeLists.txt @@ -70,10 +70,6 @@ endif() if(CMAKE_SYSTEM_NAME MATCHES "Darwin") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-abstract-non-virtual-dtor") -else() - if(NOT MSVC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath,$ORIGIN:$ORIGIN/lib") - endif() endif() set(MINDDATA_LOG_ADAPTER_SRC ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/util/md_log_adapter.cc) @@ -295,7 +291,8 @@ if(MS_BUILD_GRPC) endif() if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin" AND NOT MSLITE_ENABLE_CLOUD_MIND_DATA) - set_target_properties(_c_dataengine PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) + set_target_properties(_c_dataengine PROPERTIES INSTALL_RPATH + ${ORIGIN_PATH}:${ORIGIN_PATH}/lib:${ORIGIN_PATH}/lib/plugin) endif() if(CMAKE_SYSTEM_NAME MATCHES "Darwin") diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt index 42ac13e826e..09dba37e165 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt @@ -117,8 +117,8 @@ if(ENABLE_CACHE) add_dependencies(engine-cache-server generated_engine_files) - set_target_properties(cache_admin PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) - set_target_properties(cache_server PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) + set_target_properties(cache_admin PROPERTIES INSTALL_RPATH ${ORIGIN_PATH}/..:${ORIGIN_PATH}/../lib) + set_target_properties(cache_server PROPERTIES INSTALL_RPATH ${ORIGIN_PATH}/..:${ORIGIN_PATH}/../lib) else() ms_protobuf_generate(CACHE_PROTO_SRCS CACHE_PROTO_HDRS cache_grpc.proto) target_sources(engine-cache-client PUBLIC ${CACHE_PROTO_SRCS}) diff --git a/mindspore/ccsrc/minddata/mindrecord/CMakeLists.txt b/mindspore/ccsrc/minddata/mindrecord/CMakeLists.txt index d58570137e6..3a537cbf93f 100644 --- a/mindspore/ccsrc/minddata/mindrecord/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/mindrecord/CMakeLists.txt @@ -61,5 +61,5 @@ else() endif() if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin" AND NOT MSLITE_ENABLE_CLOUD_MIND_DATA) - set_target_properties(_c_mindrecord PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) + set_target_properties(_c_mindrecord PROPERTIES INSTALL_RPATH ${ORIGIN_PATH}:${ORIGIN_PATH}/lib) endif() diff --git a/mindspore/ccsrc/plugin/device/ascend/CMakeLists.txt b/mindspore/ccsrc/plugin/device/ascend/CMakeLists.txt index 61a721a187f..916c186c212 100644 --- a/mindspore/ccsrc/plugin/device/ascend/CMakeLists.txt +++ b/mindspore/ccsrc/plugin/device/ascend/CMakeLists.txt @@ -3,17 +3,17 @@ include_directories(${CMAKE_BINARY_DIR}) include_directories(${CMAKE_SOURCE_DIR}) include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset) -set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/nnae/latest/lib64) -set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/ascend-toolkit/latest/lib64) -set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/latest/lib64) -set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/opp/op_impl/built-in/ai_core/tbe/op_tiling) -set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe/op_tiling) -set(MINDSPORE_RPATH - ${MINDSPORE_RPATH}:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe/op_tiling) -set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/latest/opp/op_impl/built-in/ai_core/tbe/op_tiling) -set(MINDSPORE_RPATH - ${MINDSPORE_RPATH}:/usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/op_tiling) -set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/latest/opp/built-in/op_impl/ai_core/tbe/op_tiling) +set(ASCEND_RPATH ${ASCEND_RPATH}:/usr/local/Ascend/nnae/latest/lib64) +set(ASCEND_RPATH ${ASCEND_RPATH}:/usr/local/Ascend/ascend-toolkit/latest/lib64) +set(ASCEND_RPATH ${ASCEND_RPATH}:/usr/local/Ascend/latest/lib64) +set(ASCEND_RPATH ${ASCEND_RPATH}:/usr/local/Ascend/opp/op_impl/built-in/ai_core/tbe/op_tiling) +set(ASCEND_RPATH ${ASCEND_RPATH}:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe/op_tiling) +set(ASCEND_RPATH + ${ASCEND_RPATH}:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe/op_tiling) +set(ASCEND_RPATH ${ASCEND_RPATH}:/usr/local/Ascend/latest/opp/op_impl/built-in/ai_core/tbe/op_tiling) +set(ASCEND_RPATH + ${ASCEND_RPATH}:/usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/op_tiling) +set(ASCEND_RPATH ${ASCEND_RPATH}:/usr/local/Ascend/latest/opp/built-in/op_impl/ai_core/tbe/op_tiling) ### cxx api need file ### if(ENABLE_ACL) @@ -58,7 +58,8 @@ target_link_libraries(mindspore_ascend PRIVATE proto_input mindspore::protobuf) target_link_libraries(mindspore_ascend PRIVATE securec d_collective) target_link_libraries(mindspore_ascend PRIVATE mindspore::dnnl mindspore::mkldnn mindspore::ssl mindspore::crypto nnacl) -set_target_properties(mindspore_ascend PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) +set_target_properties(mindspore_ascend PROPERTIES INSTALL_RPATH + ${ORIGIN_PATH}:${ORIGIN_PATH}/ascend:${ORIGIN_PATH}/../:${ASCEND_RPATH}) if(ENABLE_DEBUGGER) # debugger: link grpc @@ -100,7 +101,8 @@ endif() if(ENABLE_D) if(ENABLE_MPI) - set_target_properties(_ascend_mpi PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) + set_target_properties(_ascend_mpi PROPERTIES INSTALL_RPATH + ${ORIGIN_PATH}:${ORIGIN_PATH}/lib:${ORIGIN_PATH}/lib/plugin/:${ASCEND_RPATH}) endif() endif() diff --git a/mindspore/ccsrc/plugin/device/ascend/hal/hccl_adapter/plugin/CMakeLists.txt b/mindspore/ccsrc/plugin/device/ascend/hal/hccl_adapter/plugin/CMakeLists.txt index 7b4f49105c5..62313ba2db2 100644 --- a/mindspore/ccsrc/plugin/device/ascend/hal/hccl_adapter/plugin/CMakeLists.txt +++ b/mindspore/ccsrc/plugin/device/ascend/hal/hccl_adapter/plugin/CMakeLists.txt @@ -2,11 +2,11 @@ add_library(hccl_plugin SHARED hccl_plugin.cc) target_include_directories(hccl_plugin PRIVATE ${CMAKE_BINARY_DIR}/proto/ge) add_dependencies(hccl_plugin graph) -set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/nnae/latest/lib64/plugin/opskernel) -set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel) -set(MINDSPORE_RPATH ${MINDSPORE_RPATH}:/usr/local/Ascend/latest/lib64/plugin/opskernel) +set(HCCL_RPATH ${HCCL_RPATH}:/usr/local/Ascend/nnae/latest/lib64/plugin/opskernel) +set(HCCL_RPATH ${HCCL_RPATH}:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel) +set(HCCL_RPATH ${HCCL_RPATH}:/usr/local/Ascend/latest/lib64/plugin/opskernel) -set_target_properties(hccl_plugin PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) +set_target_properties(hccl_plugin PROPERTIES INSTALL_RPATH ${HCCL_RPATH}) find_library(HCCL hccl ${ASCEND_CANN_RUNTIME_PATH} ${ASCEND_TOOLKIT_RUNTIME_PATH}) find_library(REGISTER register ${ASCEND_CANN_RUNTIME_PATH} ${ASCEND_TOOLKIT_RUNTIME_PATH}) diff --git a/mindspore/ccsrc/plugin/device/ascend/kernel/aicpu/aicpu_kernel_load.cc b/mindspore/ccsrc/plugin/device/ascend/kernel/aicpu/aicpu_kernel_load.cc index 23621a2eae5..0eddc91b75e 100644 --- a/mindspore/ccsrc/plugin/device/ascend/kernel/aicpu/aicpu_kernel_load.cc +++ b/mindspore/ccsrc/plugin/device/ascend/kernel/aicpu/aicpu_kernel_load.cc @@ -30,7 +30,6 @@ namespace mindspore { namespace kernel { -constexpr size_t kPluginSuffix = 6; bool AicpuOpKernelLoad::GetBinaryFileName(const std::string &so_name, const std::string &bin_folder_path, std::string *bin_file_path) { MS_EXCEPTION_IF_NULL(bin_file_path); @@ -106,7 +105,7 @@ bool AicpuOpKernelLoad::GetSoNeedLoadPath(std::string *file_path) const { MS_LOG(ERROR) << "Current path [" << cust_kernel_so_path << "] is invalid."; return false; } - auto real_cust_kernel_so_path = cust_kernel_so_path.substr(0, pos - kPluginSuffix); + auto real_cust_kernel_so_path = cust_kernel_so_path.substr(0, pos); if (real_cust_kernel_so_path.size() > PATH_MAX) { MS_LOG(ERROR) << "Current path [" << real_cust_kernel_so_path << "] is too long."; return false; @@ -126,7 +125,7 @@ bool AicpuOpKernelLoad::PackageBinaryFile(const std::string &so_name, } std::string bin_file_path; - ret = GetBinaryFileName(so_name, bin_folder_path, &bin_file_path); + ret = GetBinaryFileName(so_name, bin_folder_path + "/ascend", &bin_file_path); if (!ret) { MS_LOG(ERROR) << "GetBinaryFileName failed."; return false; diff --git a/mindspore/ccsrc/plugin/device/ascend/kernel/aicpu/aicpu_ops/CMakeLists.txt b/mindspore/ccsrc/plugin/device/ascend/kernel/aicpu/aicpu_ops/CMakeLists.txt index e9379bd721c..cfd656fa163 100644 --- a/mindspore/ccsrc/plugin/device/ascend/kernel/aicpu/aicpu_ops/CMakeLists.txt +++ b/mindspore/ccsrc/plugin/device/ascend/kernel/aicpu/aicpu_ops/CMakeLists.txt @@ -73,10 +73,10 @@ if(EXISTS ${CMAKE_C_COMPILER} AND EXISTS ${CMAKE_CXX_COMPILER}) -pthread ) - set(INSTALL_LIBRARY_DIR lib) + set(INSTALL_LIBRARY_DIR lib/plugin) install(TARGETS mindspore_aicpu_kernels OPTIONAL EXPORT mindspore_aicpu_kernels-targets - LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR} + LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR}/ascend ) endif() diff --git a/mindspore/ccsrc/plugin/device/gpu/CMakeLists.txt b/mindspore/ccsrc/plugin/device/gpu/CMakeLists.txt index 0533a4cb8b1..73ba2e0ddba 100644 --- a/mindspore/ccsrc/plugin/device/gpu/CMakeLists.txt +++ b/mindspore/ccsrc/plugin/device/gpu/CMakeLists.txt @@ -37,6 +37,7 @@ add_library(mindspore_axx_api_gpu_obj OBJECT ${API_INFER_GPU}) list(APPEND GPU_SUB_OBJECTS_SRC $) add_library(mindspore_gpu SHARED ${GPU_SUB_OBJECTS_SRC}) +set_target_properties(mindspore_gpu PROPERTIES SOVERSION ${CUDA_VERSION}) if(MSVC) target_link_options(mindspore_gpu PRIVATE /FORCE:UNRESOLVED) endif() @@ -44,7 +45,8 @@ target_link_libraries(mindspore_gpu PRIVATE mindspore_backend mindspore_core min target_link_libraries(mindspore_gpu PRIVATE mindspore_shared_lib) target_link_libraries(mindspore_gpu PRIVATE proto_input mindspore::protobuf) target_link_libraries(mindspore_gpu PRIVATE securec) -set_target_properties(mindspore_gpu PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) +set_target_properties(mindspore_gpu PROPERTIES INSTALL_RPATH + ${ORIGIN_PATH}:${ORIGIN_PATH}/gpu${CUDA_VERSION}:${ORIGIN_PATH}/../:/usr/local/cuda/lib64) target_link_libraries(mindspore_gpu PRIVATE mindspore::dnnl mindspore::mkldnn nnacl) if(NOT WIN32) target_link_libraries(mindspore_gpu PRIVATE mindspore::ssl mindspore::crypto) @@ -63,7 +65,9 @@ if(ENABLE_GPU) ${CUDA_PATH}/lib/x64/cuda.lib ${CUDA_PATH}/lib/x64/cusolver.lib ${CUDA_PATH}/lib/x64/cufft.lib - ${CUDA_PATH}/lib/x64/cusparse.lib) + ${CUDA_PATH}/lib/x64/cusparse.lib + ${CUDA_PATH}/lib/x64/libnvrtc.so + ) else() target_link_libraries(mindspore_gpu PRIVATE cuda_ops ${CUBLAS_LIBRARY_PATH} @@ -73,7 +77,9 @@ if(ENABLE_GPU) ${CUDA_PATH}/lib64/stubs/libcuda.so ${CUDA_PATH}/lib64/libcusolver.so ${CUDA_PATH}/lib64/libcufft.so - ${CUDA_PATH}/lib64/libcusparse.so) + ${CUDA_PATH}/lib64/libcusparse.so + ${CUDA_PATH}/lib64/libnvrtc.so + ) endif() if(CMAKE_SYSTEM_NAME MATCHES "Linux" AND ${CUDA_VERSION} VERSION_GREATER "11.0") diff --git a/mindspore/ccsrc/plugin/device/gpu/hal/device/cuda_driver.cc b/mindspore/ccsrc/plugin/device/gpu/hal/device/cuda_driver.cc index 90464cec160..64d16b1fed2 100644 --- a/mindspore/ccsrc/plugin/device/gpu/hal/device/cuda_driver.cc +++ b/mindspore/ccsrc/plugin/device/gpu/hal/device/cuda_driver.cc @@ -15,6 +15,7 @@ */ #include "plugin/device/gpu/hal/device/cuda_driver.h" +#include #include "utils/log_adapter.h" #include "include/common/utils/convert_utils.h" @@ -271,6 +272,12 @@ bool CudaDriver::SetDevice(int index) { "number 4."; return false; } + int major = 0; + int minor = 0; + auto curtc_ret = nvrtcVersion(&major, &minor); + if (curtc_ret == nvrtcResult::NVRTC_SUCCESS) { + MS_LOG(DEBUG) << "NVRTC version is " << major << "." << minor; + } return true; } } // namespace gpu diff --git a/mindspore/ccsrc/plugin/device/gpu/hal/hardware/gpu_device_context.cc b/mindspore/ccsrc/plugin/device/gpu/hal/hardware/gpu_device_context.cc index 379a8c4dd73..c09448aaebc 100644 --- a/mindspore/ccsrc/plugin/device/gpu/hal/hardware/gpu_device_context.cc +++ b/mindspore/ccsrc/plugin/device/gpu/hal/hardware/gpu_device_context.cc @@ -68,6 +68,21 @@ namespace mindspore { namespace device { namespace gpu { +namespace { +std::string GetCurrentDir() { +#ifndef _WIN32 + Dl_info dl_info; + if (dladdr(reinterpret_cast(GetCurrentDir), &dl_info) == 0) { + MS_LOG(WARNING) << "Get dladdr error"; + return ""; + } + std::string cur_so_path = dl_info.dli_fname; + return dirname(cur_so_path.data()); +#else + return ""; +#endif +} +} // namespace using KernelGraph = mindspore::session::KernelGraph; static thread_local bool cur_thread_device_inited{false}; @@ -759,7 +774,8 @@ uint32_t GPUKernelExecutor::GetRankID() const { bool GPUDeviceResManager::LoadCollectiveCommLib() { #ifdef ENABLE_MPI - std::string nvidia_comm_lib_name = "libnvidia_collective.so"; + std::string nvidia_comm_lib_name = GetCurrentDir() + "/gpu" + std::to_string(CUDA_VERSION / 1000) + "." + + std::to_string(CUDA_VERSION / 10 % 10) + "/libnvidia_collective.so"; auto loader = std::make_shared(nvidia_comm_lib_name); MS_EXCEPTION_IF_NULL(loader); if (!loader->Initialize()) { diff --git a/mindspore/python/mindspore/communication/_comm_helper.py b/mindspore/python/mindspore/communication/_comm_helper.py index 993d06e6a3c..dba33b5dd6e 100644 --- a/mindspore/python/mindspore/communication/_comm_helper.py +++ b/mindspore/python/mindspore/communication/_comm_helper.py @@ -20,7 +20,7 @@ from mindspore import context from mindspore.parallel._ps_context import _is_role_worker, _is_role_pserver, _is_role_sched, _is_ps_mode,\ _get_ps_context from mindspore import log as logger -from mindspore._c_expression import CollectiveManager, set_cluster_exit_with_exception +from mindspore._c_expression import CollectiveManager, set_cluster_exit_with_exception, MSContext HCCL_LIB = 'libhccl_plugin.so' @@ -29,7 +29,7 @@ def hccl_load_lib(): """load hccl lib""" try: base_dir = os.path.dirname(os.path.realpath(__file__)) - lib_path = os.path.join(base_dir, "../lib", HCCL_LIB) + lib_path = os.path.join(base_dir, "../lib/plugin", HCCL_LIB) ctypes.CDLL(lib_path) except Exception: raise RuntimeError('Get hccl lib error.') @@ -37,7 +37,8 @@ def hccl_load_lib(): _HCCL_TEST_AVAILABLE = False try: - hccl_load_lib() + if MSContext.get_instance().is_ascend_plugin_loaded(): + hccl_load_lib() except RuntimeError: _HCCL_TEST_AVAILABLE = True diff --git a/mindspore/python/mindspore/communication/_hccl_management.py b/mindspore/python/mindspore/communication/_hccl_management.py index fced7c4664c..0070fcba371 100644 --- a/mindspore/python/mindspore/communication/_hccl_management.py +++ b/mindspore/python/mindspore/communication/_hccl_management.py @@ -82,7 +82,7 @@ def load_lib(): """load hccl lib""" try: base_dir = os.path.dirname(os.path.realpath(__file__)) - lib_path = os.path.join(base_dir, "../lib", HCCL_LIB) + lib_path = os.path.join(base_dir, "../lib/plugin/ascend", HCCL_LIB) hccl_lib = ctypes.CDLL(lib_path) except Exception: raise RuntimeError('Get hccl lib error.') diff --git a/scripts/build/merge_whl_package.sh b/scripts/build/merge_whl_package.sh index f79b0a8a788..29f40e401c3 100755 --- a/scripts/build/merge_whl_package.sh +++ b/scripts/build/merge_whl_package.sh @@ -33,7 +33,6 @@ do if [ -d "./$i/mindspore/lib/plugin" ]; then \cp -rf ./$i/mindspore/lib/plugin/* $BSAE_PACKAGE_UNZIP_DIR/mindspore/lib/plugin fi; - find ./$i/mindspore/_ms_mpi.*.so -print0 | xargs -0 -I {} \cp -f {} $BSAE_PACKAGE_UNZIP_DIR/mindspore/ if [ -f "./$i/mindspore/lib/libmpi_collective.so" ]; then \cp -rf ./$i/mindspore/lib/libmpi_collective.so $BSAE_PACKAGE_UNZIP_DIR/mindspore/lib/ fi; diff --git a/setup.py b/setup.py index 20141c3b691..98a1e8e0b84 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,6 @@ from setuptools.command.egg_info import egg_info from setuptools.command.build_py import build_py backend_policy = os.getenv('BACKEND_POLICY') -device_target = os.getenv('BACKEND_TARGET') commit_id = os.getenv('COMMIT_ID').replace("\n", "") package_name = os.getenv('MS_PACKAGE_NAME').replace("\n", "") build_path = os.getenv('BUILD_PATH') @@ -58,10 +57,6 @@ def _write_package_name(file): file.write("__package_name__ = '{}'\n".format(package_name)) -def _write_device_target(file): - file.write("__device_target__ = '{}'\n".format(device_target)) - - def build_dependencies(): """generate python file""" version_file = os.path.join(pkg_dir, 'mindspore', 'version.py') @@ -80,14 +75,6 @@ def build_dependencies(): with open(config_file, 'w') as f: _write_config(f) - target = os.path.join(pkg_dir, 'mindspore', 'default_config.py') - with open(target, 'a') as f: - _write_device_target(f) - - target = os.path.join(pwd, 'mindspore/python/mindspore', 'default_config.py') - with open(target, 'a') as f: - _write_device_target(f) - package_info = os.path.join(pkg_dir, 'mindspore', 'default_config.py') with open(package_info, 'a') as f: _write_package_name(f) @@ -126,6 +113,7 @@ package_data = { '*.pdb', 'bin/*', 'lib/plugin/*', + 'lib/plugin/*/*', 'lib/*.so*', 'lib/*.a', 'lib/*.dylib*',