p60975123
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update auto install scripts

This commit is contained in:
xulei 2022-03-24 10:19:54 +08:00
parent c1a9e313b0
commit 1a7370dfb1
10 changed files with 563 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
scripts/install/euleros-ascend-conda.sh
View File

@ -1,21 +1,66 @@
#!/bin/bash #!/bin/bash
set -ex # Copyright 2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.5.0} # Prepare and Install mindspore ascend by Conda on EulerOS 2.8.
PYTHON_VERSION=${PYTHON_VERSION:-3.7.5} #
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.5.0} # This file will:
# - install mindspore dependencies via apt like gcc, libgmp
# - install conda and set up environment for mindspore
#
# Augments:
# - PYTHON_VERSION: python version to set up. [3.7(default), 3.8, 3.9]
# - MINDSPORE_VERSION: mindspore version to install, default 1.6.0
# - OPENMPI: whether to install optional package Open MPI for distributed training. [on, off(default)]
#
# Usage:
# Run script like `bash ./euleros-ascend-conda.sh`.
# To set augments, run it as `PYTHON_VERSION=3.9 MINDSPORE_VERSION=1.5.0 bash ./euleros-ascend-conda.sh`.
cd /tmp set -e
curl -O https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/Miniconda3-py37_4.10.3-Linux-aarch64.sh
bash Miniconda3-py37_4.10.3-Linux-aarch64.sh -b
# add conda to PATH PYTHON_VERSION=${PYTHON_VERSION:-3.7}
echo -e 'export PATH=~/miniconda3/bin/:$PATH' >> ~/.bash_profile MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0}
echo -e '. ~/miniconda3/etc/profile.d/conda.sh' >> ~/.bash_profile OPENMPI=${OPENMPI:-off}
source ~/.bash_profile
conda init bash available_py_version=(3.7 3.8 3.9)
# setting up conda mirror if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
cat >~/.condarc <<END echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
exit 1
fi
# add value to environment variable if value is not in it
add_env() {
local name=$1
if [[ ":${!name}:" != *":$2:"* ]]; then
echo -e "export $1=$2:\$$1" >> ~/.bashrc
fi
}
sudo yum install gcc gmp-devel -y
install_conda() {
conda_file_name="Miniconda3-py3${PYTHON_VERSION##*.}_4.10.3-Linux-$(arch).sh"
cd /tmp
curl -O https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/$conda_file_name
bash $conda_file_name -b
cd -
. ~/miniconda3/etc/profile.d/conda.sh
conda init bash
# setting up conda mirror with tsinghua source
cat >~/.condarc <<END
channels: channels:
- defaults - defaults
show_channel_urls: true show_channel_urls: true
@ -28,15 +73,23 @@ custom_channels:
msys2: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud msys2: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
bioconda: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud bioconda: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
menpo: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud menpo: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
pytorch: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
pytorch-lts: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
simpleitk: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud simpleitk: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
END END
}
# initialize conda env and install mindspore-ascend # install conda
conda create -n ms_${PYTHON_VERSION} python=${PYTHON_VERSION} -y set +e && type conda &>/dev/null
conda activate ms_${PYTHON_VERSION} if [[ $? -eq 0 ]]; then
conda install mindspore-ascend=${MINDSPORE_VERSION} -c mindspore -c conda-forge echo "conda has been installed, skip."
source "$(conda info --base)"/etc/profile.d/conda.sh
else
install_conda
fi
set -e
# verify installation # set up conda env
python -c "import mindspore;mindspore.run_check()" env_name=mindspore_py3${PYTHON_VERSION##*.}
conda create -n $env_name python=${PYTHON_VERSION} -y
conda activate $env_name
conda install mindspore-ascend -c mindspore -c conda-forge

148
scripts/install/euleros-ascend-pip.sh
View File

@ -1,52 +1,116 @@
#!/bin/bash #!/bin/bash
set -ex # Copyright 2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
# sudo cp -a /etc/apt/sources.list /etc/apt/sources.list.bak 单独执行 # Prepare and Install mindspore ascend by pip on EulerOS 2.8.
PYTHON_VERSION=${PYTHON_VERSION:-3.7.5} #
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.5.0} # This file will:
ARCH=`uname -m` # - install mindspore dependencies via apt like gcc, libgmp
# - install conda and set up environment for mindspore
#
# Augments:
# - PYTHON_VERSION: python version to set up. [3.7(default), 3.8, 3.9]
# - MINDSPORE_VERSION: mindspore version to install, default 1.6.0
# - OPENMPI: whether to install optional package Open MPI for distributed training. [on, off(default)]
#
# Usage:
# Run script like `bash ./euleros-ascend-pip.sh`.
# To set augments, run it as `PYTHON_VERSION=3.9 MINDSPORE_VERSION=1.5.0 bash ./euleros-ascend-pip.sh`.
if [[ "${PYTHON_VERSION}" == "3.7.5" ]]; then set -e
VERSION="${MINDSPORE_VERSION}-cp37-cp37m"
else PYTHON_VERSION=${PYTHON_VERSION:-3.7}
VERSION="${MINDSPORE_VERSION}-cp39-cp39" MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0}
OPENMPI=${OPENMPI:-off}
available_py_version=(3.7 3.8 3.9)
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
exit 1
fi fi
#use huaweicloud mirror in China declare -A version_map=()
repo_path=/etc/yum.repos.d/euleros.repo version_map["3.7"]="${MINDSPORE_VERSION}-cp37-cp37m"
cat > ${repo_path} << END version_map["3.8"]="${MINDSPORE_VERSION}-cp38-cp38"
[base] version_map["3.9"]="${MINDSPORE_VERSION}-cp39-cp39"
name=EulerOS-2.0SP8 base
baseurl=http://repo.huaweicloud.com/euler/2.8/os/${ARCH} # add value to environment variable if value is not in it
enabled=1 add_env() {
gpgcheck=1 local name=$1
gpgkey=http://repo.huaweicloud.com/euler/2.8/os/RPM-GPG-KEY-EulerOS if [[ ":${!name}:" != *":$2:"* ]]; then
echo -e "export $1=$2:\$$1" >> ~/.bashrc
fi
}
sudo yum install gcc gmp-devel -y
install_conda() {
conda_file_name="Miniconda3-py3${PYTHON_VERSION##*.}_4.10.3-Linux-$(arch).sh"
cd /tmp
curl -O https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/$conda_file_name
bash $conda_file_name -b
cd -
. ~/miniconda3/etc/profile.d/conda.sh
conda init bash
# setting up conda mirror with tsinghua source
cat >~/.condarc <<END
channels:
- defaults
show_channel_urls: true
default_channels:
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2
custom_channels:
conda-forge: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
msys2: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
bioconda: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
menpo: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
simpleitk: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
END END
cat ${repo_path} }
yum clean all # install conda
yum makecache set +e && type conda &>/dev/null
if [[ $? -eq 0 ]]; then
echo "conda has been installed, skip."
source "$(conda info --base)"/etc/profile.d/conda.sh
else
install_conda
fi
set -e
yum install gmp-devel # set up conda env
yum install env_name=mindspore_py3${PYTHON_VERSION##*.}
conda create -n $env_name python=${PYTHON_VERSION} -y
conda activate $env_name
# install python 3.7 # optional openmpi for distributed training
cd /tmp if [[ X"$OPENMPI" == "Xon" ]]; then
wget https://github.com/python/cpython/archive/v3.7.5.tar.gz origin_wd=$PWD
tar -xvf v3.7.5.tar.gz cd /tmp
cd /tmp/cpython-3.7.5 curl -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.3.tar.gz
mkdir -p ${PYTHON_ROOT_PATH} tar xzf openmpi-4.0.3.tar.gz
./configure --prefix=${PYTHON_ROOT_PATH} --enable-shared cd openmpi-4.0.3
make -j4 ./configure --prefix=$HOME/openmpi-4.0.3
make install -j4 make
rm -f /usr/local/bin/python sudo make install
rm -f /usr/local/bin/pip add_env PATH $HOME/openmpi-4.0.3/bin
rm -f /usr/local/lib/libpython3.7m.so.1.0 add_env LD_LIBRARY_PATH $HOME/openmpi-4.0.3/lib
ln -s ${PYTHON_ROOT_PATH}/bin/python3.7 /usr/local/bin/python cd $origin_wd
ln -s ${PYTHON_ROOT_PATH}/bin/pip3.7 /usr/local/bin/pip fi
ln -s ${PYTHON_ROOT_PATH}/lib/libpython3.7m.so.1.0 /usr/local/lib/libpython3.7m.so.1.0
ldconfig
rm -rf /tmp/cpython-3.7.5
rm -f /tmp/v3.7.5.tar.gz
pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${MINDSPORE_VERSION}/MindSpore/ascend/${ARCH}/mindspore-${VERSION}-linux_${ARCH}.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.tuna.tsinghua.edu.cn/simple ARCH=`uname -m`
pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${MINDSPORE_VERSION}/MindSpore/ascend/${ARCH}/mindspore_ascend-${version_map["$PYTHON_VERSION"]}-linux_${ARCH}.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.tuna.tsinghua.edu.cn/simple

127
scripts/install/euleros-ascend-source.sh Normal file
View File

@ -0,0 +1,127 @@
#!/bin/bash
# Copyright 2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
# Prepare environment for mindspore ascend compilation on EulerOS 2.8.
#
# This file will:
# - install mindspore dependencies via apt like gcc, cmake
# - install conda and set up environment for mindspore
#
# Augments:
# - PYTHON_VERSION: python version to set up. [3.7(default), 3.8, 3.9]
# - MINDSPORE_VERSION: mindspore version to install, default 1.6.0
# - OPENMPI: whether to install optional package Open MPI for distributed training. [on, off(default)]
#
# Usage:
# Run script like `bash -i ./euleros-ascend-source.sh`.
# To set augments, run it as `PYTHON_VERSION=3.9 MINDSPORE_VERSION=1.5.0 bash -i ./euleros-ascend-source.sh`.
set -e
PYTHON_VERSION=${PYTHON_VERSION:-3.7}
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0}
OPENMPI=${OPENMPI:-off}
available_py_version=(3.7 3.8 3.9)
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
exit 1
fi
# add value to environment variable if value is not in it
add_env() {
local name=$1
if [[ ":${!name}:" != *":$2:"* ]]; then
echo -e "export $1=$2:\$$1" >> ~/.bashrc
fi
}
sudo yum install gcc git gmp-devel tcl patch numactl-devel flex -y
curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo os=el dist=7 bash
sudo yum install git-lfs -y
git lfs install
install_conda() {
conda_file_name="Miniconda3-py3${PYTHON_VERSION##*.}_4.10.3-Linux-$(arch).sh"
cd /tmp
curl -O https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/$conda_file_name
bash $conda_file_name -b
cd -
. ~/miniconda3/etc/profile.d/conda.sh
conda init bash
# setting up conda mirror with tsinghua source
cat >~/.condarc <<END
channels:
- defaults
show_channel_urls: true
default_channels:
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2
custom_channels:
conda-forge: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
msys2: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
bioconda: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
menpo: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
simpleitk: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
END
}
# install conda
set +e && type conda &>/dev/null
if [[ $? -eq 0 ]]; then
echo "conda has been installed, skip."
source "$(conda info --base)"/etc/profile.d/conda.sh
else
install_conda
fi
set -e
# set up conda env
env_name=mindspore_py3${PYTHON_VERSION##*.}
conda create -n $env_name python=${PYTHON_VERSION} -y
conda activate $env_name
pip install wheel
pip install -U setuptools
# cmake
cd /tmp
cmake_file_name="cmake-3.19.8-Linux-$(arch).sh"
curl -O "https://cmake.org/files/v3.19/${cmake_file_name}"
sudo mkdir $HOME/cmake-3.19.8
sudo bash cmake-3.19.8-Linux-*.sh --prefix=$HOME/cmake-3.19.8 --exclude-subdir
add_env PATH $HOME/cmake-3.19.8/bin
source ~/.bashrc
cd -
# optional openmpi for distributed training
if [[ X"$OPENMPI" == "Xon" ]]; then
origin_wd=$PWD
cd /tmp
curl -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.3.tar.gz
tar xzf openmpi-4.0.3.tar.gz
cd openmpi-4.0.3
./configure --prefix=$HOME/openmpi-4.0.3
make
sudo make install
add_env PATH $HOME/openmpi-4.0.3/bin
add_env LD_LIBRARY_PATH $HOME/openmpi-4.0.3/lib
cd $origin_wd
fi
echo "The environment is ready to clone and compile mindspore."

110
scripts/install/euleros-ascend310-pip.sh Normal file
View File

@ -0,0 +1,110 @@
#!/bin/bash
# Copyright 2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
# Prepare and Install mindspore ascend by pip on EulerOS 2.8.
#
# This file will:
# - install mindspore dependencies via apt like gcc, libgmp
# - install conda and set up environment for mindspore
#
# Augments:
# - PYTHON_VERSION: python version to set up. [3.7(default), 3.8, 3.9]
# - MINDSPORE_VERSION: mindspore version to install, default 1.6.0
#
# Usage:
# Run script like `bash ./euleros-ascend310-pip.sh`.
# To set augments, run it as `PYTHON_VERSION=3.9 MINDSPORE_VERSION=1.5.0 bash ./euleros-ascend310-pip.sh`.
set -e
PYTHON_VERSION=${PYTHON_VERSION:-3.7}
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0}
OPENMPI=${OPENMPI:-off}
available_py_version=(3.7 3.8 3.9)
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
exit 1
fi
declare -A version_map=()
version_map["3.7"]="${MINDSPORE_VERSION}-cp37-cp37m"
version_map["3.8"]="${MINDSPORE_VERSION}-cp38-cp38"
version_map["3.9"]="${MINDSPORE_VERSION}-cp39-cp39"
# add value to environment variable if value is not in it
add_env() {
local name=$1
if [[ ":${!name}:" != *":$2:"* ]]; then
echo -e "export $1=$2:\$$1" >> ~/.bashrc
fi
}
sudo yum install gcc gmp-devel -y
install_conda() {
conda_file_name="Miniconda3-py3${PYTHON_VERSION##*.}_4.10.3-Linux-$(arch).sh"
cd /tmp
curl -O https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/$conda_file_name
bash $conda_file_name -b
cd -
. ~/miniconda3/etc/profile.d/conda.sh
conda init bash
# setting up conda mirror with tsinghua source
cat >~/.condarc <<END
channels:
- defaults
show_channel_urls: true
default_channels:
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2
custom_channels:
conda-forge: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
msys2: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
bioconda: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
menpo: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
simpleitk: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
END
}
# install conda
set +e && type conda &>/dev/null
if [[ $? -eq 0 ]]; then
echo "conda has been installed, skip."
source "$(conda info --base)"/etc/profile.d/conda.sh
else
install_conda
fi
set -e
# set up conda env
env_name=mindspore_py3${PYTHON_VERSION##*.}
conda create -n $env_name python=${PYTHON_VERSION} -y
conda activate $env_name
# cmake
cd /tmp
cmake_file_name="cmake-3.19.8-Linux-$(arch).sh"
curl -O "https://cmake.org/files/v3.19/${cmake_file_name}"
sudo mkdir $HOME/cmake-3.19.8
sudo bash cmake-3.19.8-Linux-*.sh --prefix=$HOME/cmake-3.19.8 --exclude-subdir
add_env PATH $HOME/cmake-3.19.8/bin
source ~/.bashrc
cd -
ARCH=`uname -m`
pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${MINDSPORE_VERSION}/MindSpore/ascend/${ARCH}/mindspore_ascend-${version_map["$PYTHON_VERSION"]}-linux_${ARCH}.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.tuna.tsinghua.edu.cn/simple

4
scripts/install/ubuntu-cpu-conda.sh
View File

@ -23,7 +23,7 @@
# - install mindspore-cpu by conda # - install mindspore-cpu by conda
# #
# Augments: # Augments:
# - PYTHON_VERSION: python version to set up. [3.7(default), 3.9] # - PYTHON_VERSION: python version to set up. [3.7(default), 3.8, 3.9]
# - MINDSPORE_VERSION: mindspore version to install, default 1.6.0 # - MINDSPORE_VERSION: mindspore version to install, default 1.6.0
# #
# Usage: # Usage:
@ -35,7 +35,7 @@ set -e
PYTHON_VERSION=${PYTHON_VERSION:-3.7} PYTHON_VERSION=${PYTHON_VERSION:-3.7}
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0} MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0}
available_py_version=(3.7 3.9) available_py_version=(3.7 3.8 3.9)
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]." echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
exit 1 exit 1

35
scripts/install/ubuntu-cpu-pip.sh
View File

@ -23,19 +23,19 @@
# - install mindspore-cpu within new installed python by pip # - install mindspore-cpu within new installed python by pip
# #
# Augments: # Augments:
# - PYTHON_VERSION: python version to install. [3.7(default), 3.9] # - PYTHON_VERSION: python version to install. [3.7(default), 3.8, 3.9]
# - MINDSPORE_VERSION: mindspore version to install, default 1.6.0 # - MINDSPORE_VERSION: mindspore version to install, default 1.6.0
# #
# Usage: # Usage:
# Need root permission to run, like `sudo bash ./ubuntu-cpu-pip.sh`. # Run script like `bash ./ubuntu-cpu-pip.sh`.
# To set augments, run it as `sudo PYTHON_VERSION=3.9 MINDSPORE_VERSION=1.5.0 bash ./ubuntu-cpu-pip.sh`. # To set augments, run it as `PYTHON_VERSION=3.9 MINDSPORE_VERSION=1.5.0 bash ./ubuntu-cpu-pip.sh`.
set -e set -e
PYTHON_VERSION=${PYTHON_VERSION:-3.7} PYTHON_VERSION=${PYTHON_VERSION:-3.7}
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0} MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0}
available_py_version=(3.7 3.9) available_py_version=(3.7 3.8 3.9)
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]." echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
exit 1 exit 1
@ -43,27 +43,28 @@ fi
declare -A version_map=() declare -A version_map=()
version_map["3.7"]="${MINDSPORE_VERSION}-cp37-cp37m" version_map["3.7"]="${MINDSPORE_VERSION}-cp37-cp37m"
version_map["3.8"]="${MINDSPORE_VERSION}-cp38-cp38"
version_map["3.9"]="${MINDSPORE_VERSION}-cp39-cp39" version_map["3.9"]="${MINDSPORE_VERSION}-cp39-cp39"
# use huaweicloud mirror in China # use huaweicloud mirror in China
sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list sudo sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list sudo sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
apt-get update sudo apt-get update
apt-get install gcc-7 libgmp-dev -y sudo apt-get install gcc-7 libgmp-dev -y
# python # python
add-apt-repository -y ppa:deadsnakes/ppa sudo add-apt-repository -y ppa:deadsnakes/ppa
apt-get install python$PYTHON_VERSION python$PYTHON_VERSION-distutils python3-pip -y sudo apt-get install python$PYTHON_VERSION python$PYTHON_VERSION-distutils python3-pip -y
update-alternatives --install /usr/bin/python python /usr/bin/python$PYTHON_VERSION 100 sudo update-alternatives --install /usr/bin/python python /usr/bin/python$PYTHON_VERSION 100
# pip # pip
sudo -u $SUDO_USER python -m pip install pip -i https://pypi.tuna.tsinghua.edu.cn/simple python -m pip install -U pip -i https://pypi.tuna.tsinghua.edu.cn/simple
update-alternatives --install /usr/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100 echo -e "alias pip='python -m pip'" >> ~/.bashrc
update-alternatives --install /usr/local/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100 source ~/.bashrc
sudo -u $SUDO_USER pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
# install mindspore whl # install mindspore whl
arch=`uname -m` arch=`uname -m`
sudo -u $SUDO_USER pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${MINDSPORE_VERSION}/MindSpore/cpu/${arch}/mindspore-${version_map["$PYTHON_VERSION"]}-linux_${arch}.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.tuna.tsinghua.edu.cn/simple pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${MINDSPORE_VERSION}/MindSpore/cpu/${arch}/mindspore-${version_map["$PYTHON_VERSION"]}-linux_${arch}.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.tuna.tsinghua.edu.cn/simple
# check mindspore installation # check mindspore installation
sudo -u $SUDO_USER python -c "import mindspore;mindspore.run_check()" python -c "import mindspore;mindspore.run_check()"

50
scripts/install/ubuntu-cpu-source.sh
View File

@ -23,59 +23,59 @@
# - install LLVM if LLVM is set to on. # - install LLVM if LLVM is set to on.
# #
# Augments: # Augments:
# - PYTHON_VERSION: python version to install. [3.7(default), 3.9] # - PYTHON_VERSION: python version to install. [3.7(default), 3.8, 3.9]
# - LLVM: whether to install optional dependency LLVM for graph kernel fusion. [on, off(default)] # - LLVM: whether to install optional dependency LLVM for graph kernel fusion. [on, off(default)]
# #
# Usage: # Usage:
# Need root permission to run, like `sudo bash ./ubuntu-cpu-source.sh`. # Run script like `bash ./ubuntu-cpu-source.sh`.
# To set augments, run it as `sudo PYTHON_VERSION=3.9 LLVM=on bash ./ubuntu-cpu-source.sh`. # To set augments, run it as `PYTHON_VERSION=3.9 LLVM=on bash ./ubuntu-cpu-source.sh`.
set -e set -e
PYTHON_VERSION=${PYTHON_VERSION:-3.7} PYTHON_VERSION=${PYTHON_VERSION:-3.7}
LLVM=${LLVM:-off} LLVM=${LLVM:-off}
available_py_version=(3.7 3.9) available_py_version=(3.7 3.8 3.9)
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]." echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
exit 1 exit 1
fi fi
# use huaweicloud mirror in China # use huaweicloud mirror in China
sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list sudo sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list sudo sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
# base packages # base packages
apt-get update sudo apt-get update
apt-get install software-properties-common lsb-release -y sudo apt-get install software-properties-common lsb-release -y
apt-get install curl tcl gcc-7 git libgmp-dev patch libnuma-dev -y sudo apt-get install curl tcl gcc-7 git libgmp-dev patch libnuma-dev -y
# cmake # cmake
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add -
apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" sudo apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main"
apt-get install cmake -y sudo apt-get install cmake -y
# optional dependency LLVM for graph-computation fusion # optional dependency LLVM for graph-computation fusion
if [[ X"$LLVM" == "Xon" ]]; then if [[ X"$LLVM" == "Xon" ]]; then
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
add-apt-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" sudo add-apt-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main"
apt-get update sudo apt-get update
apt-get install llvm-12-dev -y sudo apt-get install llvm-12-dev -y
fi fi
# python # python
add-apt-repository -y ppa:deadsnakes/ppa sudo add-apt-repository -y ppa:deadsnakes/ppa
apt-get install python$PYTHON_VERSION python$PYTHON_VERSION-dev python$PYTHON_VERSION-distutils python3-pip -y sudo apt-get install python$PYTHON_VERSION python$PYTHON_VERSION-dev python$PYTHON_VERSION-distutils python3-pip -y
update-alternatives --install /usr/bin/python python /usr/bin/python$PYTHON_VERSION 100 sudo update-alternatives --install /usr/bin/python python /usr/bin/python$PYTHON_VERSION 100
# pip # pip
sudo -u $SUDO_USER python -m pip install pip -i https://pypi.tuna.tsinghua.edu.cn/simple python -m pip install -U pip -i https://pypi.tuna.tsinghua.edu.cn/simple
update-alternatives --install /usr/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100 echo -e "alias pip='python -m pip'" >> ~/.bashrc
update-alternatives --install /usr/local/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100 source ~/.bashrc
sudo -u $SUDO_USER pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
# wheel # wheel
sudo -u $SUDO_USER pip install wheel pip install wheel
# python 3.9 needs setuptools>44.0 # python 3.9 needs setuptools>44.0
sudo -u $SUDO_USER pip install -U setuptools pip install -U setuptools
echo "The environment is ready to clone and compile mindspore." echo "The environment is ready to clone and compile mindspore."

37
scripts/install/ubuntu-gpu-conda.sh
View File

@ -24,23 +24,23 @@
# - compile and install Open MPI if OPENMPI is set to on. # - compile and install Open MPI if OPENMPI is set to on.
# #
# Augments: # Augments:
# - PYTHON_VERSION: python version to install. [3.7(default), 3.9] # - PYTHON_VERSION: python version to install. [3.7(default), 3.8, 3.9]
# - MINDSPORE_VERSION: mindspore version to install, default 1.6.0 # - MINDSPORE_VERSION: mindspore version to install, default 1.6.0
# - CUDA_VERSION: CUDA version to install. [10.1(default), 11.1] # - CUDA_VERSION: CUDA version to install. [10.1, 11.1(default)]
# - OPENMPI: whether to install optional package Open MPI for distributed training. [on, off(default)] # - OPENMPI: whether to install optional package Open MPI for distributed training. [on, off(default)]
# #
# Usage: # Usage:
# Run script like `bash -i ./ubuntu-gpu-conda.sh`. # Run script like `bash -i ./ubuntu-gpu-conda.sh`.
# To set augments, run it as `PYTHON_VERSION=3.9 CUDA_VERSION=11.1 OPENMPI=on bash -i ./ubuntu-gpu-conda.sh`. # To set augments, run it as `PYTHON_VERSION=3.9 CUDA_VERSION=10.1 OPENMPI=on bash -i ./ubuntu-gpu-conda.sh`.
set -e set -e
PYTHON_VERSION=${PYTHON_VERSION:-3.7} PYTHON_VERSION=${PYTHON_VERSION:-3.7}
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0} MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0}
CUDA_VERSION=${CUDA_VERSION:-10.1} CUDA_VERSION=${CUDA_VERSION:-11.1}
OPENMPI=${OPENMPI:-off} OPENMPI=${OPENMPI:-off}
available_py_version=(3.7 3.9) available_py_version=(3.7 3.8 3.9)
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]." echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
exit 1 exit 1
@ -117,6 +117,33 @@ else
fi fi
set -e set -e
# install cuda/cudnn
cd /tmp
declare -A cuda_url_map=()
cuda_url_map["10.1"]=https://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run
cuda_url_map["11.1"]=https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_455.32.00_linux.run
cuda_url=${cuda_url_map[$CUDA_VERSION]}
wget $cuda_url
sudo sh ${cuda_url##*/} --silent --toolkit
cd -
sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /"
sudo add-apt-repository "deb https://developer.download.nvidia.cn/compute/machine-learning/repos/ubuntu1804/x86_64/ /"
sudo apt-get update
declare -A cudnn_name_map=()
cudnn_name_map["10.1"]="libcudnn7=7.6.5.32-1+cuda10.1 libcudnn7-dev=7.6.5.32-1+cuda10.1"
cudnn_name_map["11.1"]="libcudnn8=8.0.4.30-1+cuda11.1 libcudnn8-dev=8.0.4.30-1+cuda11.1"
sudo apt-get install --no-install-recommends ${cudnn_name_map[$CUDA_VERSION]} -y
# add cuda to path
set +e && source ~/.bashrc
set -e
add_env PATH /usr/local/cuda/bin
add_env LD_LIBRARY_PATH /usr/local/cuda/lib64
add_env LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu
set +e && source ~/.bashrc
set -e
# set up conda env and install mindspore-cpu # set up conda env and install mindspore-cpu
env_name=mindspore_py3${PYTHON_VERSION##*.} env_name=mindspore_py3${PYTHON_VERSION##*.}
declare -A cudnn_version_map=() declare -A cudnn_version_map=()

59
scripts/install/ubuntu-gpu-pip.sh
View File

@ -25,23 +25,23 @@
# - compile and install Open MPI if OPENMPI is set to on. # - compile and install Open MPI if OPENMPI is set to on.
# #
# Augments: # Augments:
# - PYTHON_VERSION: python version to install. [3.7(default), 3.9] # - PYTHON_VERSION: python version to install. [3.7(default), 3.8, 3.9]
# - MINDSPORE_VERSION: mindspore version to install, default 1.6.0 # - MINDSPORE_VERSION: mindspore version to install, default 1.6.0
# - CUDA_VERSION: CUDA version to install. [10.1(default), 11.1] # - CUDA_VERSION: CUDA version to install. [10.1, 11.1(default)]
# - OPENMPI: whether to install optional package Open MPI for distributed training. [on, off(default)] # - OPENMPI: whether to install optional package Open MPI for distributed training. [on, off(default)]
# #
# Usage: # Usage:
# Need root permission to run, like `sudo bash -i ./ubuntu-gpu-pip.sh`. # Run script like `bash -i ./ubuntu-gpu-pip.sh`.
# To set augments, run it as `sudo PYTHON_VERSION=3.9 CUDA_VERSION=11.1 OPENMPI=on bash -i ./ubuntu-gpu-pip.sh`. # To set augments, run it as `PYTHON_VERSION=3.9 CUDA_VERSION=10.1 OPENMPI=on bash -i ./ubuntu-gpu-pip.sh`.
set -e set -e
PYTHON_VERSION=${PYTHON_VERSION:-3.7} PYTHON_VERSION=${PYTHON_VERSION:-3.7}
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0} MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0}
CUDA_VERSION=${CUDA_VERSION:-10.1} CUDA_VERSION=${CUDA_VERSION:-11.1}
OPENMPI=${OPENMPI:-off} OPENMPI=${OPENMPI:-off}
available_py_version=(3.7 3.9) available_py_version=(3.7 3.8 3.9)
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]." echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
exit 1 exit 1
@ -64,6 +64,7 @@ cuda_name="cuda-$CUDA_VERSION"
declare -A version_map=() declare -A version_map=()
version_map["3.7"]="${MINDSPORE_VERSION}-cp37-cp37m" version_map["3.7"]="${MINDSPORE_VERSION}-cp37-cp37m"
version_map["3.8"]="${MINDSPORE_VERSION}-cp38-cp38"
version_map["3.9"]="${MINDSPORE_VERSION}-cp39-cp39" version_map["3.9"]="${MINDSPORE_VERSION}-cp39-cp39"
# add value to environment variable if value is not in it # add value to environment variable if value is not in it
@ -75,21 +76,21 @@ add_env() {
} }
# use huaweicloud mirror in China # use huaweicloud mirror in China
sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list sudo sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list sudo sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
apt-get update sudo apt-get update
apt-get install curl make gcc-7 libgmp-dev linux-headers-"$(uname -r)" -y sudo apt-get install curl make gcc-7 libgmp-dev linux-headers-"$(uname -r)" -y
# python # python
add-apt-repository -y ppa:deadsnakes/ppa sudo add-apt-repository -y ppa:deadsnakes/ppa
apt-get install python$PYTHON_VERSION python$PYTHON_VERSION-distutils python3-pip -y sudo apt-get install python$PYTHON_VERSION python$PYTHON_VERSION-distutils python3-pip -y
update-alternatives --install /usr/bin/python python /usr/bin/python$PYTHON_VERSION 100 sudo update-alternatives --install /usr/bin/python python /usr/bin/python$PYTHON_VERSION 100
# pip # pip
sudo -u $SUDO_USER python -m pip install pip -i https://pypi.tuna.tsinghua.edu.cn/simple python -m pip install -U pip -i https://pypi.tuna.tsinghua.edu.cn/simple
update-alternatives --install /usr/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100 echo -e "alias pip='python -m pip'" >> ~/.bashrc
update-alternatives --install /usr/local/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100 source ~/.bashrc
sudo -u $SUDO_USER pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
# install cuda/cudnn # install cuda/cudnn
cd /tmp cd /tmp
@ -98,16 +99,16 @@ cuda_url_map["10.1"]=https://developer.download.nvidia.com/compute/cuda/10.1/Pro
cuda_url_map["11.1"]=https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_455.32.00_linux.run cuda_url_map["11.1"]=https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_455.32.00_linux.run
cuda_url=${cuda_url_map[$CUDA_VERSION]} cuda_url=${cuda_url_map[$CUDA_VERSION]}
wget $cuda_url wget $cuda_url
sh ${cuda_url##*/} --silent --toolkit sudo sh ${cuda_url##*/} --silent --toolkit
cd - cd -
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /" sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /"
add-apt-repository "deb https://developer.download.nvidia.cn/compute/machine-learning/repos/ubuntu1804/x86_64/ /" sudo add-apt-repository "deb https://developer.download.nvidia.cn/compute/machine-learning/repos/ubuntu1804/x86_64/ /"
apt-get update sudo apt-get update
declare -A cudnn_name_map=() declare -A cudnn_name_map=()
cudnn_name_map["10.1"]="libcudnn7=7.6.5.32-1+cuda10.1 libcudnn7-dev=7.6.5.32-1+cuda10.1" cudnn_name_map["10.1"]="libcudnn7=7.6.5.32-1+cuda10.1 libcudnn7-dev=7.6.5.32-1+cuda10.1"
cudnn_name_map["11.1"]="libcudnn8=8.0.4.30-1+cuda11.1 libcudnn8-dev=8.0.4.30-1+cuda11.1" cudnn_name_map["11.1"]="libcudnn8=8.0.4.30-1+cuda11.1 libcudnn8-dev=8.0.4.30-1+cuda11.1"
apt-get install --no-install-recommends ${cudnn_name_map[$CUDA_VERSION]} -y sudo apt-get install --no-install-recommends ${cudnn_name_map[$CUDA_VERSION]} -y
# add cuda to path # add cuda to path
set +e && source ~/.bashrc set +e && source ~/.bashrc
@ -121,18 +122,18 @@ set -e
# optional openmpi for distributed training # optional openmpi for distributed training
if [[ X"$OPENMPI" == "Xon" ]]; then if [[ X"$OPENMPI" == "Xon" ]]; then
cd /tmp cd /tmp
sudo -u $SUDO_USER curl -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.3.tar.gz curl -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.3.tar.gz
sudo -u $SUDO_USER tar xzf openmpi-4.0.3.tar.gz tar xzf openmpi-4.0.3.tar.gz
cd openmpi-4.0.3 cd openmpi-4.0.3
sudo -u $SUDO_USER ./configure --prefix=/usr/local/openmpi-4.0.3 ./configure --prefix=/usr/local/openmpi-4.0.3
sudo -u $SUDO_USER make make
make install sudo make install
add_env PATH /usr/local/openmpi-4.0.3/bin add_env PATH /usr/local/openmpi-4.0.3/bin
add_env LD_LIBRARY_PATH /usr/local/openmpi-4.0.3/lib add_env LD_LIBRARY_PATH /usr/local/openmpi-4.0.3/lib
fi fi
arch=`uname -m` arch=`uname -m`
sudo -u $SUDO_USER pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${MINDSPORE_VERSION}/MindSpore/gpu/${arch}/${cuda_name}/mindspore_gpu-${version_map["$PYTHON_VERSION"]}-linux_${arch}.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.tuna.tsinghua.edu.cn/simple pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${MINDSPORE_VERSION}/MindSpore/gpu/${arch}/${cuda_name}/mindspore_gpu-${version_map["$PYTHON_VERSION"]}-linux_${arch}.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.tuna.tsinghua.edu.cn/simple
# check mindspore installation # check mindspore installation
python -c "import mindspore;mindspore.run_check()" python -c "import mindspore;mindspore.run_check()"

76
scripts/install/ubuntu-gpu-source.sh
View File

@ -25,23 +25,23 @@
# - install LLVM if LLVM is set to on. # - install LLVM if LLVM is set to on.
# #
# Augments: # Augments:
# - PYTHON_VERSION: python version to install. [3.7(default), 3.9] # - PYTHON_VERSION: python version to install. [3.7(default), 3.8, 3.9]
# - CUDA_VERSION: CUDA version to install. [10.1(default), 11.1] # - CUDA_VERSION: CUDA version to install. [10.1, 11.1(default)]
# - OPENMPI: whether to install optional package Open MPI for distributed training. [on, off(default)] # - OPENMPI: whether to install optional package Open MPI for distributed training. [on, off(default)]
# - LLVM: whether to install optional dependency LLVM for graph kernel fusion. [on, off(default)] # - LLVM: whether to install optional dependency LLVM for graph kernel fusion. [on, off(default)]
# #
# Usage: # Usage:
# Need root permission to run, like `sudo bash -i ./ubuntu-gpu-source.sh`. # Run script like `bash -i ./ubuntu-gpu-source.sh`.
# To set augments, run it as `sudo PYTHON_VERSION=3.9 CUDA_VERSION=11.1 OPENMPI=on bash -i ./ubuntu-gpu-source.sh`. # To set augments, run it as `PYTHON_VERSION=3.9 CUDA_VERSION=10.1 OPENMPI=on bash -i ./ubuntu-gpu-source.sh`.
set -e set -e
PYTHON_VERSION=${PYTHON_VERSION:-3.7} PYTHON_VERSION=${PYTHON_VERSION:-3.7}
CUDA_VERSION=${CUDA_VERSION:-10.1} CUDA_VERSION=${CUDA_VERSION:-11.1}
OPENMPI=${OPENMPI:-off} OPENMPI=${OPENMPI:-off}
LLVM=${LLVM:-off} LLVM=${LLVM:-off}
available_py_version=(3.7 3.9) available_py_version=(3.7 3.8 3.9)
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]." echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
exit 1 exit 1
@ -70,51 +70,51 @@ add_env() {
} }
# use huaweicloud mirror in China # use huaweicloud mirror in China
sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list sudo sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list sudo sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
# base packages # base packages
apt-get update sudo apt-get update
apt-get install software-properties-common lsb-release -y sudo apt-get install software-properties-common lsb-release -y
apt-get install curl tcl automake autoconf libtool gcc-7 git libgmp-dev patch libnuma-dev flex -y sudo apt-get install curl tcl automake autoconf libtool gcc-7 git libgmp-dev patch libnuma-dev flex -y
# cmake # cmake
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add -
apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" sudo apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main"
apt-get install cmake -y sudo apt-get install cmake -y
# optional dependency LLVM for graph-computation fusion # optional dependency LLVM for graph-computation fusion
if [[ X"$LLVM" == "Xon" ]]; then if [[ X"$LLVM" == "Xon" ]]; then
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
add-apt-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" sudo add-apt-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main"
apt-get update sudo apt-get update
apt-get install llvm-12-dev -y sudo apt-get install llvm-12-dev -y
fi fi
# optional openmpi for distributed training # optional openmpi for distributed training
if [[ X"$OPENMPI" == "Xon" ]]; then if [[ X"$OPENMPI" == "Xon" ]]; then
origin_wd=$PWD origin_wd=$PWD
cd /tmp cd /tmp
sudo -u $SUDO_USER curl -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.3.tar.gz curl -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.3.tar.gz
sudo -u $SUDO_USER tar xzf openmpi-4.0.3.tar.gz tar xzf openmpi-4.0.3.tar.gz
cd openmpi-4.0.3 cd openmpi-4.0.3
sudo -u $SUDO_USER ./configure --prefix=/usr/local/openmpi-4.0.3 ./configure --prefix=/usr/local/openmpi-4.0.3
sudo -u $SUDO_USER make make
make install sudo make install
add_env PATH /usr/local/openmpi-4.0.3/bin add_env PATH /usr/local/openmpi-4.0.3/bin
add_env LD_LIBRARY_PATH /usr/local/openmpi-4.0.3/lib add_env LD_LIBRARY_PATH /usr/local/openmpi-4.0.3/lib
cd $origin_wd cd $origin_wd
fi fi
# python # python
add-apt-repository -y ppa:deadsnakes/ppa sudo add-apt-repository -y ppa:deadsnakes/ppa
apt-get install python$PYTHON_VERSION python$PYTHON_VERSION-dev python$PYTHON_VERSION-distutils python3-pip -y sudo apt-get install python$PYTHON_VERSION python$PYTHON_VERSION-dev python$PYTHON_VERSION-distutils python3-pip -y
update-alternatives --install /usr/bin/python python /usr/bin/python$PYTHON_VERSION 100 sudo update-alternatives --install /usr/bin/python python /usr/bin/python$PYTHON_VERSION 100
# pip # pip
sudo -u $SUDO_USER python -m pip install pip -i https://pypi.tuna.tsinghua.edu.cn/simple python -m pip install -U pip -i https://pypi.tuna.tsinghua.edu.cn/simple
update-alternatives --install /usr/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100 echo -e "alias pip='python -m pip'" >> ~/.bashrc
update-alternatives --install /usr/local/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100 source ~/.bashrc
sudo -u $SUDO_USER pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
# install cuda/cudnn # install cuda/cudnn
cd /tmp cd /tmp
@ -123,16 +123,16 @@ cuda_url_map["10.1"]=https://developer.download.nvidia.com/compute/cuda/10.1/Pro
cuda_url_map["11.1"]=https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_455.32.00_linux.run cuda_url_map["11.1"]=https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_455.32.00_linux.run
cuda_url=${cuda_url_map[$CUDA_VERSION]} cuda_url=${cuda_url_map[$CUDA_VERSION]}
wget $cuda_url wget $cuda_url
sh ${cuda_url##*/} --silent --toolkit sudo sh ${cuda_url##*/} --silent --toolkit
cd - cd -
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /" sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /"
add-apt-repository "deb https://developer.download.nvidia.cn/compute/machine-learning/repos/ubuntu1804/x86_64/ /" sudo add-apt-repository "deb https://developer.download.nvidia.cn/compute/machine-learning/repos/ubuntu1804/x86_64/ /"
apt-get update sudo apt-get update
declare -A cudnn_name_map=() declare -A cudnn_name_map=()
cudnn_name_map["10.1"]="libcudnn7=7.6.5.32-1+cuda10.1 libcudnn7-dev=7.6.5.32-1+cuda10.1" cudnn_name_map["10.1"]="libcudnn7=7.6.5.32-1+cuda10.1 libcudnn7-dev=7.6.5.32-1+cuda10.1"
cudnn_name_map["11.1"]="libcudnn8=8.0.4.30-1+cuda11.1 libcudnn8-dev=8.0.4.30-1+cuda11.1" cudnn_name_map["11.1"]="libcudnn8=8.0.4.30-1+cuda11.1 libcudnn8-dev=8.0.4.30-1+cuda11.1"
apt-get install --no-install-recommends ${cudnn_name_map[$CUDA_VERSION]} -y sudo apt-get install --no-install-recommends ${cudnn_name_map[$CUDA_VERSION]} -y
# add cuda to path # add cuda to path
set +e && source ~/.bashrc set +e && source ~/.bashrc
@ -144,8 +144,8 @@ set +e && source ~/.bashrc
set -e set -e
# wheel # wheel
sudo -u $SUDO_USER pip install wheel pip install wheel
# python 3.9 needs setuptools>44.0 # python 3.9 needs setuptools>44.0
sudo -u $SUDO_USER pip install -U setuptools pip install -U setuptools
echo "The environment is ready to clone and compile mindspore." echo "The environment is ready to clone and compile mindspore."