From 3b7452a0e33f630098f98c755b9654417a25a970 Mon Sep 17 00:00:00 2001 From: zhoufeng Date: Fri, 16 Dec 2022 15:21:05 +0800 Subject: [PATCH] support -f build only device plugins Signed-off-by: zhoufeng --- CMakeLists.txt | 7 +- build.sh | 5 +- cmake/options.cmake | 15 +++ cmake/package_plugin.cmake | 115 ++++++++++++++++++ cmake/utils.cmake | 48 +++++++- mindspore/ccsrc/CMakeLists.txt | 4 + .../kernels/image/dvpp/utils/CMakeLists.txt | 1 + .../ccsrc/plugin/device/ascend/CMakeLists.txt | 7 +- .../ccsrc/plugin/device/gpu/CMakeLists.txt | 13 +- mindspore/core/CMakeLists.txt | 4 +- scripts/build/build_mindspore.sh | 3 + scripts/build/merge_whl_package.sh | 2 + scripts/build/parse_device.sh | 1 + scripts/build/process_options.sh | 4 +- scripts/build/usage.sh | 1 + 15 files changed, 215 insertions(+), 15 deletions(-) create mode 100644 cmake/package_plugin.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 30d922c6b33..c9db97a10b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,7 +119,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") endif() include(${CMAKE_SOURCE_DIR}/cmake/init.cmake) -add_subdirectory(mindspore/ccsrc) +add_subdirectory_with_faster_option(mindspore/ccsrc) add_subdirectory(mindspore/core) if(ENABLE_TESTCASES OR ENABLE_CPP_ST) @@ -133,7 +133,10 @@ if(${VERSION_NUMBER} MATCHES ".*dev.*") message("building dev mode") set(BUILD_DEV_MODE ON) endif() -if(MODE_ASCEND_ACL) + +if(ONLY_BUILD_DEVICE_PLUGINS) + include(cmake/package_plugin.cmake) +elseif(MODE_ASCEND_ACL) include(cmake/package_tar.cmake) elseif(CMAKE_SYSTEM_NAME MATCHES "Windows") include(cmake/package_win.cmake) diff --git a/build.sh b/build.sh index e9d92c6e482..b812c7e1bad 100755 --- a/build.sh +++ b/build.sh @@ -92,7 +92,10 @@ else if [[ "X$ENABLE_ACL" == "Xon" ]] && [[ "X$ENABLE_D" == "Xoff" ]]; then echo "acl mode, skipping deploy phase" rm -rf ${BASEPATH}/output/_CPack_Packages/ - else + elif [[ "X$FASTER_BUILD_FOR_PLUGINS" == "Xon" ]]; then + echo "plugin mode, skipping deploy phase" + rm -rf ${BASEPATH}/output/_CPack_Packages/ + else cp -rf ${BUILD_PATH}/package/mindspore/lib ${BASEPATH}/mindspore/python/mindspore cp -rf ${BUILD_PATH}/package/mindspore/*.so ${BASEPATH}/mindspore/python/mindspore fi diff --git a/cmake/options.cmake b/cmake/options.cmake index d382ebf0ce0..ebc197e99e3 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -28,6 +28,21 @@ option(BUILD_DEV_MODE "MindSpore build nightly dev mode" OFF) option(ENABLE_FAST_HASH_TABLE "Enable use fast hash table instead of std ones" ON) option(USE_LLVM "use llvm" OFF) option(USE_MS_THREADPOOL_FOR_DNNL "use ms threadpool for onednn ops" ON) +option(ONLY_BUILD_DEVICE_PLUGINS "only build device plugins" OFF) + +if(ONLY_BUILD_DEVICE_PLUGINS) + if(NOT CMAKE_SYSTEM_NAME MATCHES "Linux") + set(ONLY_BUILD_DEVICE_PLUGINS OFF) + message(WARNING "-f is supported on only linux.") + endif() + if(ENABLE_CPU) + set(ENABLE_CPU OFF) + endif() +endif() + +if(NOT CMAKE_SYSTEM_NAME MATCHES "Linux") + set(ENABLE_MPI OFF) +endif() if(NOT CMAKE_SYSTEM_NAME MATCHES "Linux") set(USE_MS_THREADPOOL_FOR_DNNL OFF) diff --git a/cmake/package_plugin.cmake b/cmake/package_plugin.cmake new file mode 100644 index 00000000000..b84050e50d6 --- /dev/null +++ b/cmake/package_plugin.cmake @@ -0,0 +1,115 @@ +# include dependency +include(CMakePackageConfigHelpers) +include(GNUInstallDirs) + +# prepare output directory +file(REMOVE_RECURSE ${CMAKE_SOURCE_DIR}/output) +file(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/output) + +# cpack variables +string(TOLOWER linux_${CMAKE_HOST_SYSTEM_PROCESSOR} PLATFORM_NAME) +if(PYTHON_VERSION MATCHES "3.9") + set(CPACK_PACKAGE_FILE_NAME mindspore.py39) +elseif(PYTHON_VERSION MATCHES "3.8") + set(CPACK_PACKAGE_FILE_NAME mindspore.py38) +elseif(PYTHON_VERSION MATCHES "3.7") + set(CPACK_PACKAGE_FILE_NAME mindspore.py37) +else() + message("Could not find 'Python 3.9' OR 'Python 3.8' or 'Python 3.7'") + return() +endif() + +set(CPACK_GENERATOR "ZIP") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_SOURCE_DIR}/output) + +set(INSTALL_LIB_DIR ${CMAKE_INSTALL_LIBDIR} CACHE PATH "Installation directory for libraries") +set(INSTALL_BASE_DIR ".") +set(INSTALL_LIB_DIR "lib") +set(INSTALL_PLUGIN_DIR "${INSTALL_LIB_DIR}/plugin") + +# set package files +install( + TARGETS mindspore_shared_lib + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore +) + +if(ENABLE_D OR ENABLE_GPU) + install( + TARGETS api_lib + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore + ) +endif() + +if(ENABLE_D) + install( + TARGETS mindspore_ascend + DESTINATION ${INSTALL_PLUGIN_DIR} + COMPONENT mindspore + ) + if(ENABLE_MPI) + install( + TARGETS ascend_collective + DESTINATION ${INSTALL_PLUGIN_DIR}/ascend + COMPONENT mindspore + ) + endif() + install( + TARGETS hccl_plugin + DESTINATION ${INSTALL_PLUGIN_DIR}/ascend + COMPONENT mindspore + ) +endif() + +if(ENABLE_ACL) + install( + TARGETS dvpp_utils + DESTINATION ${INSTALL_PLUGIN_DIR}/ascend + COMPONENT mindspore + ) +endif() + +if(ENABLE_GPU) + install( + TARGETS mindspore_gpu + DESTINATION ${INSTALL_PLUGIN_DIR} + COMPONENT mindspore + ) + if(ENABLE_MPI) + install( + TARGETS nvidia_collective + DESTINATION ${INSTALL_PLUGIN_DIR}/gpu${CUDA_VERSION} + COMPONENT mindspore + ) + if(CMAKE_SYSTEM_NAME MATCHES "Linux" AND GPU_BACKEND_CUDA) + install(FILES ${nccl_LIBPATH}/libnccl.so.2.7.6 DESTINATION ${INSTALL_PLUGIN_DIR}/gpu${CUDA_VERSION} + RENAME libnccl.so.2 COMPONENT mindspore) + endif() + endif() + install( + TARGETS cuda_ops + DESTINATION ${INSTALL_PLUGIN_DIR}/gpu${CUDA_VERSION} + COMPONENT mindspore + ) +endif() + +if(ENABLE_AKG AND CMAKE_SYSTEM_NAME MATCHES "Linux") + if(ENABLE_GPU) + install( + TARGETS akg + DESTINATION ${INSTALL_PLUGIN_DIR}/gpu${CUDA_VERSION} + COMPONENT mindspore + ) + endif() + + if(ENABLE_D) + install( + TARGETS akg + DESTINATION ${INSTALL_PLUGIN_DIR}/ascend + COMPONENT mindspore + ) + endif() +endif() + +include(CPack) diff --git a/cmake/utils.cmake b/cmake/utils.cmake index 34bf6583c47..efcfd8c66b0 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -484,4 +484,50 @@ function(src_separate_compile) endwhile() set(${STUDENT_OBJECT_SIZE} "${OBJECT_COUNT}" PARENT_SCOPE) message("${STUDENT_OBJECT_SIZE} object count is ${OBJECT_COUNT}") -endfunction() \ No newline at end of file +endfunction() + +function(enable_target_when_only_build_plugins target) + if(ONLY_BUILD_DEVICE_PLUGINS) + set_target_properties(${target} PROPERTIES EXCLUDE_FROM_ALL FALSE) + endif() +endfunction() + +function(disable_target_when_only_build_plugins target) + if(ONLY_BUILD_DEVICE_PLUGINS) + get_property(is_set TARGET ${target} PROPERTY EXCLUDE_FROM_ALL) + if(NOT DEFINED is_set) + set_target_properties(${target} PROPERTIES EXCLUDE_FROM_ALL TRUE) + endif() + endif() +endfunction() + +function(enable_directory_when_only_build_plugins dir) + get_property(targets DIRECTORY ${dir} PROPERTY BUILDSYSTEM_TARGETS) + foreach(target ${targets}) + enable_target_when_only_build_plugins(${target}) + endforeach() + get_property(items DIRECTORY ${dir} PROPERTY SUBDIRECTORIES) + foreach(item ${items}) + enable_directory_when_only_build_plugins(${item}) + endforeach() +endfunction() + +function(disable_directory_when_only_build_plugins dir) + get_property(targets DIRECTORY ${dir} PROPERTY BUILDSYSTEM_TARGETS) + foreach(target ${targets}) + disable_target_when_only_build_plugins(${target}) + endforeach() + get_property(items DIRECTORY ${dir} PROPERTY SUBDIRECTORIES) + foreach(item ${items}) + disable_directory_when_only_build_plugins(${item}) + endforeach() +endfunction() + +function(add_subdirectory_with_faster_option dir) + if(ONLY_BUILD_DEVICE_PLUGINS) + add_subdirectory(${dir}) + disable_directory_when_only_build_plugins(${dir}) + else() + add_subdirectory(${dir}) + endif() +endfunction() diff --git a/mindspore/ccsrc/CMakeLists.txt b/mindspore/ccsrc/CMakeLists.txt index e0bb6d67882..e13ef48b73d 100644 --- a/mindspore/ccsrc/CMakeLists.txt +++ b/mindspore/ccsrc/CMakeLists.txt @@ -519,16 +519,19 @@ endif() if(ENABLE_D) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/plugin/device/ascend) add_subdirectory(plugin/device/ascend) + enable_directory_when_only_build_plugins(plugin/device/ascend) endif() if(ENABLE_GPU AND GPU_BACKEND_CUDA) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/plugin/device/gpu) add_subdirectory(plugin/device/gpu) + enable_directory_when_only_build_plugins(plugin/device/gpu) endif() if(ENABLE_GPU AND GPU_BACKEND_ROCM) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/plugin/device/amd) add_subdirectory(plugin/device/amd) + enable_directory_when_only_build_plugins(plugin/device/amd) endif() if(CMAKE_SYSTEM_NAME MATCHES "Windows") target_link_libraries(mindspore PUBLIC mindspore::pybind11_module) @@ -610,3 +613,4 @@ if(NOT ENABLE_TESTCASES AND NOT (ENABLE_D OR ENABLE_CPU OR ENABLE_GPU)) endif() add_subdirectory(transform/graph_ir) add_subdirectory(cxx_api) +enable_directory_when_only_build_plugins(cxx_api) diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/utils/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/utils/CMakeLists.txt index aea65fd992f..31780cc2a4e 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/utils/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/utils/CMakeLists.txt @@ -25,6 +25,7 @@ else() endif() add_library(dvpp_utils SHARED ${DVPP_UTILS_SRC}) +enable_target_when_only_build_plugins(dvpp_utils) if(MSLITE_ENABLE_ACL) find_library(acl_dvpp libacl_dvpp.so ${ASCEND_CANN_RUNTIME_PATH} ${ASCEND_TOOLKIT_RUNTIME_PATH}) diff --git a/mindspore/ccsrc/plugin/device/ascend/CMakeLists.txt b/mindspore/ccsrc/plugin/device/ascend/CMakeLists.txt index e9026a08b18..759eea9902e 100644 --- a/mindspore/ccsrc/plugin/device/ascend/CMakeLists.txt +++ b/mindspore/ccsrc/plugin/device/ascend/CMakeLists.txt @@ -53,8 +53,7 @@ target_link_libraries(mindspore_ascend PRIVATE mindspore_backend mindspore_core target_link_libraries(mindspore_ascend PRIVATE mindspore_shared_lib) target_link_libraries(mindspore_ascend PRIVATE proto_input mindspore::protobuf) target_link_libraries(mindspore_ascend PRIVATE securec d_collective) -target_link_libraries(mindspore_ascend PRIVATE mindspore::dnnl mindspore::mkldnn mindspore::ssl - mindspore::crypto nnacl) +target_link_libraries(mindspore_ascend PRIVATE mindspore::ssl mindspore::crypto) set_target_properties(mindspore_ascend PROPERTIES INSTALL_RPATH ${ORIGIN_PATH}:${ORIGIN_PATH}/ascend:${ORIGIN_PATH}/../:${ASCEND_RPATH}) @@ -69,8 +68,10 @@ if(ENABLE_D) find_library(GE_RUNNER ge_runner ${ASCEND_CANN_RUNTIME_PATH} ${ASCEND_TOOLKIT_RUNTIME_PATH}) find_library(GRAPH graph ${ASCEND_CANN_RUNTIME_PATH} ${ASCEND_TOOLKIT_RUNTIME_PATH}) target_link_libraries(mindspore_ascend PRIVATE ${GE_RUNNER} ${GRAPH}) - target_link_libraries(mindspore_ascend PRIVATE mindspore::event mindspore::event_pthreads + if(ENABLE_CPU) + target_link_libraries(mindspore_ascend PRIVATE mindspore::event mindspore::event_pthreads mindspore::event_openssl -Wl,--no-as-needed mindspore::event_core ps_cache) + endif() endif() if(MODE_ASCEND_ALL) diff --git a/mindspore/ccsrc/plugin/device/gpu/CMakeLists.txt b/mindspore/ccsrc/plugin/device/gpu/CMakeLists.txt index 89ba75e92f3..d2b5cc493a6 100644 --- a/mindspore/ccsrc/plugin/device/gpu/CMakeLists.txt +++ b/mindspore/ccsrc/plugin/device/gpu/CMakeLists.txt @@ -47,13 +47,14 @@ target_link_libraries(mindspore_gpu PRIVATE proto_input mindspore::protobuf) target_link_libraries(mindspore_gpu PRIVATE securec) set_target_properties(mindspore_gpu PROPERTIES INSTALL_RPATH ${ORIGIN_PATH}:${ORIGIN_PATH}/gpu:${ORIGIN_PATH}/gpu${CUDA_VERSION}:${ORIGIN_PATH}/../:${CUDA_PATH}/lib64) -target_link_libraries(mindspore_gpu PRIVATE mindspore::dnnl mindspore::mkldnn nnacl) -if(NOT WIN32) -target_link_libraries(mindspore_gpu PRIVATE mindspore::ssl mindspore::crypto) -target_link_libraries(mindspore_gpu PRIVATE mindspore::event mindspore::event_pthreads - mindspore::event_openssl -Wl,--no-as-needed mindspore::event_core ps_cache) +if(ENABLE_CPU) + target_link_libraries(mindspore_gpu PRIVATE mindspore::dnnl mindspore::mkldnn nnacl) + if(NOT WIN32) + target_link_libraries(mindspore_gpu PRIVATE mindspore::ssl mindspore::crypto) + target_link_libraries(mindspore_gpu PRIVATE mindspore::event mindspore::event_pthreads + mindspore::event_openssl -Wl,--no-as-needed mindspore::event_core ps_cache) + endif() endif() - if(ENABLE_GPU) message("add gpu lib to mindspore_gpu") if(WIN32) diff --git a/mindspore/core/CMakeLists.txt b/mindspore/core/CMakeLists.txt index dfb67cd2d43..6543f7e7188 100644 --- a/mindspore/core/CMakeLists.txt +++ b/mindspore/core/CMakeLists.txt @@ -41,7 +41,9 @@ file(GLOB_RECURSE CORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "expander/*.cc" ) -set(CORE_SRC_LIST ${CORE_SRC_LIST} ${CORE_OPS_LIST}) +if(NOT ONLY_BUILD_DEVICE_PLUGINS) + set(CORE_SRC_LIST ${CORE_SRC_LIST} ${CORE_OPS_LIST}) +endif() if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Linux") LIST(REMOVE_ITEM CORE_SRC_LIST utils/numa_interface.cc) diff --git a/scripts/build/build_mindspore.sh b/scripts/build/build_mindspore.sh index 5a72d43d1a9..2dd8e039dfb 100755 --- a/scripts/build/build_mindspore.sh +++ b/scripts/build/build_mindspore.sh @@ -109,6 +109,9 @@ build_mindspore() else CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_FAST_HASH_TABLE=OFF" fi + if [[ "X$FASTER_BUILD_FOR_PLUGINS" == "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DONLY_BUILD_DEVICE_PLUGINS=ON" + fi echo "${CMAKE_ARGS}" if [[ "X$INC_BUILD" = "Xoff" ]]; then cmake ${CMAKE_ARGS} ${BASEPATH} diff --git a/scripts/build/merge_whl_package.sh b/scripts/build/merge_whl_package.sh index cc68e9d1c8d..d39d6eec07d 100755 --- a/scripts/build/merge_whl_package.sh +++ b/scripts/build/merge_whl_package.sh @@ -30,6 +30,8 @@ done MAX_GPU_VERSION=0 for ((i=1;i<$counter;i=$i+1)) do + echo "Rename $i dirname to mindspore ..." + mv ./$i/mindspore.py* "./$i/mindspore" echo "Copy $i plugin files to 0 ..." if [ -d "./$i/mindspore/lib/plugin" ]; then \cp -rf ./$i/mindspore/lib/plugin/* $BASE_PACKAGE_UNZIP_DIR/mindspore/lib/plugin diff --git a/scripts/build/parse_device.sh b/scripts/build/parse_device.sh index 619aed563e9..d887cc5a603 100755 --- a/scripts/build/parse_device.sh +++ b/scripts/build/parse_device.sh @@ -78,6 +78,7 @@ parse_device() elif [[ "X$D" == "Xcpu" ]]; then export ENABLE_CPU="on" export ENABLE_AKG="on" + export ENABLE_MPI="on" elif [[ "X$D" == "X" ]]; then : else diff --git a/scripts/build/process_options.sh b/scripts/build/process_options.sh index 7b175eb58c7..dd059cfcacc 100755 --- a/scripts/build/process_options.sh +++ b/scripts/build/process_options.sh @@ -20,7 +20,7 @@ set -e process_options() { # Process the options - while getopts 'drvj:c:t:hb:s:a:g:p:ie:l:I:RP:D:zM:V:K:B:En:A:S:k:W:F:H:L:yG:' opt + while getopts 'drvj:c:t:hb:s:a:g:p:ie:l:I:RP:D:zM:V:K:B:En:A:S:k:W:F:H:L:yG:f' opt do CASE_SENSIVE_ARG=${OPTARG} OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]') @@ -107,6 +107,8 @@ process_options() echo "Link Tensor-RT library. Path: ${CASE_SENSIVE_ARG}" ;; G) build_option_proc_upper_g ;; + f) + export FASTER_BUILD_FOR_PLUGINS="on" ;; *) echo "Unknown option ${opt}!" usage diff --git a/scripts/build/usage.sh b/scripts/build/usage.sh index a93200bd8fc..ed94ffe4272 100755 --- a/scripts/build/usage.sh +++ b/scripts/build/usage.sh @@ -65,4 +65,5 @@ usage() echo " -F Use fast hash table in mindspore compiler, default on" echo " -G Select an architecture to build, set 'common' to build with common architectures(eg. gpu: 5.3, 6.0, 6.2, 7.0, 7.2, 7.5),\\" echo " set auto to detect automatically, default: 'auto'. Only effective for GPU currently." + echo " -f Faster build process for device plugins, only build plugin." }