Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

to optimize size of unify package, all gpu plugin use same cuda_ops.so 

Signed-off-by: zhoufeng <zhoufeng54@huawei.com>
This commit is contained in:
zhoufeng 2022-12-26 11:19:47 +08:00
parent 2f3d008c2b
commit c43c16452b
3 changed files with 30 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ccsrc/plugin/device/gpu/CMakeLists.txt
View File

@ -46,7 +46,7 @@ target_link_libraries(mindspore_gpu PRIVATE mindspore_shared_lib)
target_link_libraries(mindspore_gpu PRIVATE proto_input mindspore::protobuf)
target_link_libraries(mindspore_gpu PRIVATE securec)
set_target_properties(mindspore_gpu PROPERTIES INSTALL_RPATH
${ORIGIN_PATH}:${ORIGIN_PATH}/gpu${CUDA_VERSION}:${ORIGIN_PATH}/../:${CUDA_PATH}/lib64)
${ORIGIN_PATH}:${ORIGIN_PATH}/gpu:${ORIGIN_PATH}/gpu${CUDA_VERSION}:${ORIGIN_PATH}/../:${CUDA_PATH}/lib64)
target_link_libraries(mindspore_gpu PRIVATE mindspore::dnnl mindspore::mkldnn nnacl)
if(NOT WIN32)
target_link_libraries(mindspore_gpu PRIVATE mindspore::ssl mindspore::crypto)

21
mindspore/ccsrc/plugin/device/gpu/kernel/cuda_impl/CMakeLists.txt
View File

@ -34,24 +34,9 @@ if(ENABLE_GPU)
target_compile_options(cuda_common_obj PRIVATE "/std:c++17")
endif()
cuda_add_library(cuda_ops SHARED ${CUDA_OPS_SRC_LIST} $<TARGET_OBJECTS:cuda_common_obj>)
message("add gpu lib to cuda_ops")
if(WIN32)
target_link_libraries(cuda_ops mindspore_core
${CUBLAS_LIBRARY_PATH}
${CUDA_PATH}/lib/x64/curand.lib
${CUDNN_LIBRARY_PATH}
${CUDA_PATH}/lib/x64/cudart.lib
${CUDA_PATH}/lib/x64/cuda.lib
${CUDA_PATH}/lib/x64/cusolver.lib
${CUDA_PATH}/lib/x64/cufft.lib)
if(NOT MSVC)
target_link_options(cuda_ops PRIVATE "-Wl,--allow-shlib-undefined")
else()
target_link_libraries(cuda_ops mindspore_core
${CUBLAS_LIBRARY_PATH}
${CUDA_PATH}/lib64/libcurand.so
${CUDNN_LIBRARY_PATH}
${CUDA_PATH}/lib64/libcudart.so
${CUDA_PATH}/lib64/stubs/libcuda.so
${CUDA_PATH}/lib64/libcusolver.so
${CUDA_PATH}/lib64/libcufft.so)
target_link_options(cuda_ops PRIVATE "/FORCE:UNRESOLVED")
endif()
endif()

40
scripts/build/merge_whl_package.sh
View File

@ -17,46 +17,58 @@
set -e
BASEPATH=$(cd "$(dirname $0)"; pwd)
MINDSPORE_ROOT_PATH=`realpath $BASEPATH/../../`
BSAE_PACKAGE_UNZIP_DIR=./0
BASE_PACKAGE_UNZIP_DIR=./0
PACKAGE_FILE_NAME=`basename $1`
counter=0
for whl in "$@"; do
echo "Unzip $whl ..."
unzip $whl -d $counter
unzip -q $whl -d $counter
((++counter))
done
MAX_GPU_VERSION=0
for ((i=1;i<$counter;i=$i+1))
do
echo "Copy $i plugin files to 0 ..."
if [ -d "./$i/mindspore/lib/plugin" ]; then
\cp -rf ./$i/mindspore/lib/plugin/* $BSAE_PACKAGE_UNZIP_DIR/mindspore/lib/plugin
\cp -rf ./$i/mindspore/lib/plugin/* $BASE_PACKAGE_UNZIP_DIR/mindspore/lib/plugin
fi;
if [ -f "./$i/mindspore/lib/libmpi_collective.so" ]; then
\cp -rf ./$i/mindspore/lib/libmpi_collective.so $BSAE_PACKAGE_UNZIP_DIR/mindspore/lib/
\cp -rf ./$i/mindspore/lib/libmpi_collective.so $BASE_PACKAGE_UNZIP_DIR/mindspore/lib/
fi;
if [ -f "./$i/mindspore/lib/libmpi_adapter.so" ]; then
\cp -rf ./$i/mindspore/lib/libmpi_adapter.so $BSAE_PACKAGE_UNZIP_DIR/mindspore/lib/
\cp -rf ./$i/mindspore/lib/libmpi_adapter.so $BASE_PACKAGE_UNZIP_DIR/mindspore/lib/
fi;
if [ -f "./$i/mindspore/lib/libmindspore.so" ]; then
\cp -rf ./$i/mindspore/lib/libmindspore.so $BSAE_PACKAGE_UNZIP_DIR/mindspore/lib/
\cp -rf ./$i/mindspore/lib/libmindspore.so $BASE_PACKAGE_UNZIP_DIR/mindspore/lib/
fi;
if [ -f "./$i/mindspore/lib/libmindspore_shared_lib.so" ]; then
\cp -rf ./$i/mindspore/lib/libmindspore_shared_lib.so $BSAE_PACKAGE_UNZIP_DIR/mindspore/lib/
\cp -rf ./$i/mindspore/lib/libmindspore_shared_lib.so $BASE_PACKAGE_UNZIP_DIR/mindspore/lib/
fi;
CUR_GPU_VERSION=`find "./$i/mindspore/lib/plugin" -name 'gpu*' -exec sh -c 'echo ${0##*gpu}' {} \;`
if [ -n "$CUR_GPU_VERSION" ]; then
if [ "`echo "$CUR_GPU_VERSION > $MAX_GPU_VERSION" | bc`" -eq 1 ]; then
MAX_GPU_VERSION=$CUR_GPU_VERSION
if [ ! -d "$BASE_PACKAGE_UNZIP_DIR/mindspore/lib/plugin/gpu" ]; then
mkdir -p $BASE_PACKAGE_UNZIP_DIR/mindspore/lib/plugin/gpu
fi;
\cp -rf ./$i/mindspore/lib/plugin/gpu$CUR_GPU_VERSION/libcuda_ops.so $BASE_PACKAGE_UNZIP_DIR/mindspore/lib/plugin/gpu
fi;
rm -f $BASE_PACKAGE_UNZIP_DIR/mindspore/lib/plugin/gpu$CUR_GPU_VERSION/libcuda_ops.so
fi;
rm -rf $i
done
export COMMIT_ID=`cat $BSAE_PACKAGE_UNZIP_DIR/mindspore/.commit_id | awk '{print $3}' | sed $'s/\'//g'`
VERSION=`cat $BSAE_PACKAGE_UNZIP_DIR/mindspore/version.py | awk '{print $3}' | sed $'s/\'//g'`
export COMMIT_ID=`cat $BASE_PACKAGE_UNZIP_DIR/mindspore/.commit_id | awk '{print $3}' | sed $'s/\'//g'`
VERSION=`cat $BASE_PACKAGE_UNZIP_DIR/mindspore/version.py | awk '{print $3}' | sed $'s/\'//g'`
echo -n "$VERSION" > $MINDSPORE_ROOT_PATH/version.txt
echo "Delete useless file ..."
rm -f $BSAE_PACKAGE_UNZIP_DIR/mindspore/version.py
rm -f $BSAE_PACKAGE_UNZIP_DIR/mindspore/default_config.py
rm -f $BSAE_PACKAGE_UNZIP_DIR/mindspore/.commit_id
rm -f $BSAE_PACKAGE_UNZIP_DIR/mindspore/lib/libakg.so
rm -f $BASE_PACKAGE_UNZIP_DIR/mindspore/version.py
rm -f $BASE_PACKAGE_UNZIP_DIR/mindspore/default_config.py
rm -f $BASE_PACKAGE_UNZIP_DIR/mindspore/.commit_id
rm -f $BASE_PACKAGE_UNZIP_DIR/mindspore/lib/libakg.so
echo "Repacking new wheel package ..."
PACKAGE_WORK_DIR=$MINDSPORE_ROOT_PATH/build
@ -66,7 +78,7 @@ fi
mkdir -p $MINDSPORE_ROOT_PATH/mindspore/python/mindspore
mkdir -p $PACKAGE_WORK_DIR
PACKAGE_WORK_DIR=`realpath $PACKAGE_WORK_DIR`
mv $BSAE_PACKAGE_UNZIP_DIR $PACKAGE_WORK_DIR/package
mv $BASE_PACKAGE_UNZIP_DIR $PACKAGE_WORK_DIR/package
export MS_PACKAGE_NAME="mindspore"
export BACKEND_POLICY="ms"
export BUILD_PATH=$PACKAGE_WORK_DIR