p85971406
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update ubuntu + gpu with pip/conda install script.

This commit is contained in:
Bowen Ma 2021-12-29 16:50:38 +08:00
parent b94bdc8fdd
commit 7b1b133d1f
2 changed files with 169 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
scripts/install/ubuntu-gpu-conda.sh Normal file
View File

@ -0,0 +1,116 @@
#!/bin/bash
set -ex
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.5.0}
PYTHON_VERSION=${PYTHON_VERSION:-3.7.5}
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.5.0}
CUDA_VERSION=${CUDA_VERSION:-8.0.5.39-1+cuda11.1}
DISTRIBUTED=${DISTRIBUTED:-false}
CUDA_INSTALL_PATH=${CUDA_INSTALL_PATH:-cuda-11}
CUDATOOLKIT_VERSION=${CUDATOOLKIT_VERSION:-11.1}
CUDNN_VERSION=${CUDNN_VERSION:-8.0.5}
#use huaweicloud mirror in China
sudo sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
sudo sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
sudo apt-get update
# install python 3.7 and make it default
sudo apt-get install gcc-7 libgmp-dev curl python3.7 -y
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.7 100
cd /tmp
curl -O https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/Miniconda3-py37_4.10.3-Linux-x86_64.sh
bash Miniconda3-py37_4.10.3-Linux-x86_64.sh
# add conda to PATH
echo -e 'export PATH=~/miniconda3/bin/:$PATH' >> ~/.bash_profile
echo -e '. ~/miniconda3/etc/profile.d/conda.sh' >> ~/.bash_profile
source ~/.bash_profile
conda init bash
# setting up conda mirror
cat >~/.condarc <<END
channels:
- defaults
show_channel_urls: true
default_channels:
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2
custom_channels:
conda-forge: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
msys2: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
bioconda: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
menpo: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
pytorch: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
pytorch-lts: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
simpleitk: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
END
#initialize conda env and install mindspore-cpu
conda create -n py37 python=3.7.5 -y
conda activate py37
# install gmp 6.1.2, downloading gmp is slow
echo "install gmp start"
sudo apt-get install m4 -y
cd /tmp
curl -O https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz
xz -d gmp-6.1.2.tar.xz
tar xvzf gmp-6.1.2.tar && cd gmp-6.1.2
./configure --prefix=/usr/local/gmp-6.1.2
make
sudo make install
echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/gmp-6.1.2/lib' >> ~/.bash_profile
echo "install gmp success"
# install cuda with apt-get
echo "install cuda start"
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-ubuntu1804.pin
sudo mv cuda-ubuntu1804.pin /etc/apt/preferences.d/cuda-repository-pin-600
wget -c https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda-repo-ubuntu1804-11-1-local_11.1.1-455.32.00-1_amd64.deb
sudo dpkg -i cuda-repo-ubuntu1804-11-1-local_11.1.1-455.32.00-1_amd64.deb
sudo apt-key add /var/cuda-repo-ubuntu1804-11-1-local/7fa2af80.pub
sudo apt-get update
sudo apt-get -y install cuda
# add cuda to path
cat >> ~/.bash_profile <<END
export PATH=/usr/local/cuda/bin:\$PATH
export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/cuda/lib64
END
source ~/.bash_profile
echo "cuda install success."
# install cudnn
cd /tmp
wget https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu1804/x86_64/libcudnn8_${CUDA_VERSION}_amd64.deb
wget https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu1804/x86_64/libcudnn8-dev_${CUDA_VERSION}_amd64.deb
sudo dpkg -i libcudnn8_${CUDA_VERSION}_amd64.deb libcudnn8-dev_${CUDA_VERSION}_amd64.deb
# Install CuDNN 8 and NCCL 2
wget https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb
sudo dpkg -i nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb
sudo apt update
sudo apt install -y libcudnn8=${CUDA_VERSION} libcudnn8-dev=${CUDA_VERSION} libnccl2=2.7.8-1+cuda11.1 libnccl2-dev=2.7.8-1+cuda11.1
# optional (tensort for serving, openmpi for distributed training)
if [[ "$DISTRIBUTED" == "false" ]]; then
cd /tmp
curl -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.3.tar.gz
tar xvzf openmpi-4.0.3.tar.gz
cd openmpi-4.0.3
./configure --prefix=/usr/local/openmpi
make
sudo make install
echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/openmpi/lib' >> ~/.bash_profile
echo 'export PATH=$PATH:/usr/local/openmpi/bin' >> ~/.bash_profile
source ~/.bash_profile
fi
# install mindspore-gpu with conda
conda install mindspore-gpu=${MINDSPORE_VERSION} cudatoolkit=${CUDATOOLKIT_VERSION} cudnn=${CUDNN_VERSION} -c mindspore -c conda-forge

71
scripts/install/ubuntu-gpu-pip.sh
View File

@ -1,13 +1,12 @@
#!/bin/bash #!/bin/bash
set -ex set -ex
PYTHON_VERSION=${PYTHON_VERSION:-3.7.5} PYTHON_VERSION=${PYTHON_VERSION:-3.7.5}
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.5.0} MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.5.0}
CUDA_VERSION=${CUDA_VERSION:-8.0.5.39-1+cuda11.1}
declare -A map DISTRIBUTED=${DISTRIBUTED:-false}
CUDA_INSTALL_PATH=${CUDA_INSTALL_PATH:-cuda-11}
ARCH=$(uname -m)
if [[ "${PYTHON_VERSION}" == "3.7.5" ]]; then if [[ "${PYTHON_VERSION}" == "3.7.5" ]]; then
VERSION="${MINDSPORE_VERSION}-cp37-cp37m" VERSION="${MINDSPORE_VERSION}-cp37-cp37m"
@ -21,19 +20,38 @@ sudo sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/
sudo apt-get update sudo apt-get update
# install python 3.7 and make it default # install python 3.7 and make it default
sudo apt-get install gcc-7 libgmp-dev curl python3.7 openssl -y sudo apt-get install gcc-7 libgmp-dev curl python3.7 openssl ubuntu-drivers-common openssl -y
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.7 100 sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.7 100
cd /tmp cd /tmp
curl -O https://bootstrap.pypa.io/get-pip.py curl -O https://bootstrap.pypa.io/get-pip.py
sudo python get-pip.py sudo python get-pip.py
# add pip mirror
# install cuda mkdir ~/.pip
# cd /tmp cat > ~/.pip/pip.conf <<END
# wget https://developer.download.nvidia.com/compute/cuda/11.1.0/local_installers/cuda_11.1.0_455.23.05_linux.run [global]
index-url = https://repo.huaweicloud.com/repository/pypi/simple
trusted-host = repo.huaweicloud.com
timeout = 120
END
# install gmp 6.1.2, downloading gmp is slow
sudo apt-get install m4 -y
cd /tmp
curl -O https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz
xz -d gmp-6.1.2.tar.xz
tar xvzf gmp-6.1.2.tar && cd gmp-6.1.2
./configure --prefix=/usr/local/gmp-6.1.2
make
sudo make install
echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/gmp-6.1.2/lib' >> ~/.bash_profile
# install cuda with linux.run
# another option is to use https://developer.download.nvidia.com/compute/cuda/11.1.0/local_installers/cuda_11.1.0_455.23.05_linux.run
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-ubuntu1804.pin wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-ubuntu1804.pin
sudo mv cuda-ubuntu1804.pin /etc/apt/preferences.d/cuda-repository-pin-600 sudo mv cuda-ubuntu1804.pin /etc/apt/preferences.d/cuda-repository-pin-600
@ -43,24 +61,41 @@ sudo apt-key add /var/cuda-repo-ubuntu1804-11-1-local/7fa2af80.pub
sudo apt-get update sudo apt-get update
sudo apt-get -y install cuda sudo apt-get -y install cuda
# cuda to add path # add cuda to path
cat >> ~/.bash_profile <<END cat >> ~/.bash_profile <<END
export PATH=/usr/local/cuda/bin:\$PATH export PATH=/usr/local/cuda/bin:\$PATH
export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/cuda/lib64 export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/cuda/lib64
END END
source ~/.bash_profile source ~/.bash_profile
[[ nvcc -V ]] && echo "cuda install success." echo "cuda install success."
# install cudnn # install cudnn
cd /tmp cd /tmp
wget https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu1804/x86_64/libcudnn8_8.0.5.39-1+cuda11.1_amd64.deb wget https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu1804/x86_64/libcudnn8_${CUDA_VERSION}_amd64.deb
wget https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu1804/x86_64/libcudnn8-dev_8.0.5.39-1+cuda11.1_amd64.deb wget https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu1804/x86_64/libcudnn8-dev_${CUDA_VERSION}_amd64.deb
sudo dpkg -i libcudnn8_${CUDA_VERSION}_amd64.deb libcudnn8-dev_${CUDA_VERSION}_amd64.deb
sudo dpkg -i libcudnn8_8.0.5.39-1+cuda11.1_amd64.deb libcudnn8-dev_8.0.5.39-1+cuda11.1_amd64.deb # Install CuDNN 8 and NCCL 2
wget https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb
sudo dpkg -i nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb
sudo apt update
sudo apt install -y libcudnn8=${CUDA_VERSION} libcudnn8-dev=${CUDA_VERSION} libnccl2=2.7.8-1+cuda11.1 libnccl2-dev=2.7.8-1+cuda11.1
# optional (tensort for serving, openmpi for distributed training)
if [[ "$DISTRIBUTED" == "false" ]]; then
cd /tmp
curl -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.3.tar.gz
tar xvzf openmpi-4.0.3.tar.gz
cd openmpi-4.0.3
./configure --prefix=/usr/local/openmpi
make
sudo make install
echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/openmpi/lib' >> ~/.bash_profile
echo 'export PATH=$PATH:/usr/local/openmpi/bin' >> ~/.bash_profile
source ~/.bash_profile
fi
# reference # install mindspore gpu
# - https://gist.github.com/bogdan-kulynych/f64eb148eeef9696c70d485a76e42c3a
pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${MINDSPORE_VERSION}/MindSpore/gpu/${CUDA_INSTALL_PATH}/mindspore-${VERSION}-linux_${ARCH}.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.tuna.tsinghua.edu.cn/simple