forked from mindspore-Ecosystem/mindspore
add auto install scripts
This commit is contained in:
xulei
parent
33c3a9e910
commit
38a1ed4f73
|
|
@ -1,67 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -ex
|
|
||||||
|
|
||||||
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.5.0}
|
|
||||||
PYTHON_VERSION=${PYTHON_VERSION:-3.7.5}
|
|
||||||
|
|
||||||
#use huaweicloud mirror in China
|
|
||||||
sudo sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
|
||||||
sudo sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
|
||||||
sudo apt-get update
|
|
||||||
|
|
||||||
# install python 3.7 and make it default
|
|
||||||
sudo apt-get install gcc-7 libgmp-dev curl python3.7 -y
|
|
||||||
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.7 100
|
|
||||||
|
|
||||||
cd /tmp
|
|
||||||
curl -O https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/Miniconda3-py37_4.10.3-Linux-x86_64.sh
|
|
||||||
bash Miniconda3-py37_4.10.3-Linux-x86_64.sh -b
|
|
||||||
|
|
||||||
# add conda to PATH
|
|
||||||
echo -e 'export PATH=~/miniconda3/bin/:$PATH' >> ~/.bash_profile
|
|
||||||
echo -e '. ~/miniconda3/etc/profile.d/conda.sh' >> ~/.bash_profile
|
|
||||||
source ~/.bash_profile
|
|
||||||
conda init bash
|
|
||||||
# setting up conda mirror with qinghua source
|
|
||||||
cat >~/.condarc <<END
|
|
||||||
channels:
|
|
||||||
- defaults
|
|
||||||
show_channel_urls: true
|
|
||||||
default_channels:
|
|
||||||
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main
|
|
||||||
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r
|
|
||||||
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2
|
|
||||||
custom_channels:
|
|
||||||
conda-forge: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
|
||||||
msys2: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
|
||||||
bioconda: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
|
||||||
menpo: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
|
||||||
pytorch: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
|
||||||
pytorch-lts: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
|
||||||
simpleitk: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
|
||||||
END
|
|
||||||
|
|
||||||
#initialize conda env and install mindspore-cpu
|
|
||||||
|
|
||||||
conda create -n ms_${PYTHON_VERSION} python=${PYTHON_VERSION} -y
|
|
||||||
conda activate ms_${PYTHON_VERSION}
|
|
||||||
conda install mindspore-cpu=${MINDSPORE_VERSION} -c mindspore -c conda-forge -y
|
|
||||||
|
|
||||||
# check if it is the right mindspore version
|
|
||||||
python -c "import mindspore;mindspore.run_check()"
|
|
||||||
|
|
||||||
# check if it can be run with GPU
|
|
||||||
|
|
||||||
cat > example.py <<END
|
|
||||||
import numpy as np
|
|
||||||
from mindspore import Tensor
|
|
||||||
import mindspore.ops as ops
|
|
||||||
import mindspore.context as context
|
|
||||||
|
|
||||||
context.set_context(device_target="GPU")
|
|
||||||
x = Tensor(np.ones([1,3,3,4]).astype(np.float32))
|
|
||||||
y = Tensor(np.ones([1,3,3,4]).astype(np.float32))
|
|
||||||
print(ops.add(x, y))
|
|
||||||
END
|
|
||||||
|
|
||||||
python example.py
|
|
||||||
|
|
@ -0,0 +1,94 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# Copyright 2022 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# Prepare and Install mindspore cpu by conda on Ubuntu 18.04.
|
||||||
|
#
|
||||||
|
# This file will:
|
||||||
|
# - change deb source to huaweicloud mirror
|
||||||
|
# - install mindspore dependencies via apt like gcc, libgmp
|
||||||
|
# - install conda and set up environment for mindspore
|
||||||
|
# - install mindspore-cpu by conda
|
||||||
|
#
|
||||||
|
# Augments:
|
||||||
|
# - PYTHON_VERSION: python version to set up. [3.7(default), 3.9]
|
||||||
|
# - MINDSPORE_VERSION: mindspore version to install, default 1.6.0
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# Run script like `bash ./ubuntu-cpu-conda.sh`.
|
||||||
|
# To set augments, run it as `PYTHON_VERSION=3.9 MINDSPORE_VERSION=1.5.0 bash ./ubuntu-cpu-conda.sh`.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
PYTHON_VERSION=${PYTHON_VERSION:-3.7}
|
||||||
|
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0}
|
||||||
|
|
||||||
|
available_py_version=(3.7 3.9)
|
||||||
|
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
|
||||||
|
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
install_conda() {
|
||||||
|
conda_file_name="Miniconda3-py3${PYTHON_VERSION##*.}_4.10.3-Linux-$(arch).sh"
|
||||||
|
cd /tmp
|
||||||
|
wget https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/$conda_file_name
|
||||||
|
bash $conda_file_name -b
|
||||||
|
cd -
|
||||||
|
. ~/miniconda3/etc/profile.d/conda.sh
|
||||||
|
conda init bash
|
||||||
|
# setting up conda mirror with tsinghua source
|
||||||
|
cat >~/.condarc <<END
|
||||||
|
channels:
|
||||||
|
- defaults
|
||||||
|
show_channel_urls: true
|
||||||
|
default_channels:
|
||||||
|
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main
|
||||||
|
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r
|
||||||
|
- https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2
|
||||||
|
custom_channels:
|
||||||
|
conda-forge: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
||||||
|
msys2: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
||||||
|
bioconda: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
||||||
|
menpo: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
||||||
|
simpleitk: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
||||||
|
END
|
||||||
|
}
|
||||||
|
|
||||||
|
# use huaweicloud mirror in China
|
||||||
|
sudo sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
||||||
|
sudo sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
||||||
|
sudo apt-get update
|
||||||
|
|
||||||
|
sudo apt-get install gcc-7 libgmp-dev -y
|
||||||
|
|
||||||
|
# install conda
|
||||||
|
set +e && type conda &>/dev/null
|
||||||
|
if [[ $? -eq 0 ]]; then
|
||||||
|
echo "conda has been installed, skip."
|
||||||
|
source "$(conda info --base)"/etc/profile.d/conda.sh
|
||||||
|
else
|
||||||
|
install_conda
|
||||||
|
fi
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# set up conda env and install mindspore-cpu
|
||||||
|
env_name=mindspore_py3${PYTHON_VERSION##*.}
|
||||||
|
conda create -n $env_name python=${PYTHON_VERSION} -y
|
||||||
|
conda activate $env_name
|
||||||
|
conda install mindspore-cpu=${MINDSPORE_VERSION} -c mindspore -c conda-forge -y
|
||||||
|
|
||||||
|
# check mindspore installation
|
||||||
|
python -c "import mindspore;mindspore.run_check()"
|
||||||
|
|
@ -0,0 +1,69 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# Copyright 2022 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# Prepare and Install mindspore cpu by pip on Ubuntu 18.04.
|
||||||
|
#
|
||||||
|
# This file will:
|
||||||
|
# - change deb source to huaweicloud mirror
|
||||||
|
# - install mindspore dependencies via apt like gcc, libgmp
|
||||||
|
# - install python3 & pip3 via apt and set it to default
|
||||||
|
# - install mindspore-cpu within new installed python by pip
|
||||||
|
#
|
||||||
|
# Augments:
|
||||||
|
# - PYTHON_VERSION: python version to install. [3.7(default), 3.9]
|
||||||
|
# - MINDSPORE_VERSION: mindspore version to install, default 1.6.0
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# Need root permission to run, like `sudo bash ./ubuntu-cpu-pip.sh`.
|
||||||
|
# To set augments, run it as `sudo PYTHON_VERSION=3.9 MINDSPORE_VERSION=1.5.0 bash ./ubuntu-cpu-pip.sh`.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
PYTHON_VERSION=${PYTHON_VERSION:-3.7}
|
||||||
|
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0}
|
||||||
|
|
||||||
|
available_py_version=(3.7 3.9)
|
||||||
|
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
|
||||||
|
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
declare -A version_map=()
|
||||||
|
version_map["3.7"]="${MINDSPORE_VERSION}-cp37-cp37m"
|
||||||
|
version_map["3.9"]="${MINDSPORE_VERSION}-cp39-cp39"
|
||||||
|
|
||||||
|
# use huaweicloud mirror in China
|
||||||
|
sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
||||||
|
sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
||||||
|
apt-get update
|
||||||
|
apt-get install gcc-7 libgmp-dev -y
|
||||||
|
|
||||||
|
# python
|
||||||
|
add-apt-repository -y ppa:deadsnakes/ppa
|
||||||
|
apt-get install python$PYTHON_VERSION python$PYTHON_VERSION-distutils python3-pip -y
|
||||||
|
update-alternatives --install /usr/bin/python python /usr/bin/python$PYTHON_VERSION 100
|
||||||
|
# pip
|
||||||
|
sudo -u $SUDO_USER python -m pip install pip -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||||
|
update-alternatives --install /usr/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100
|
||||||
|
update-alternatives --install /usr/local/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100
|
||||||
|
sudo -u $SUDO_USER pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||||
|
|
||||||
|
# install mindspore whl
|
||||||
|
arch=`uname -m`
|
||||||
|
sudo -u $SUDO_USER pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${MINDSPORE_VERSION}/MindSpore/cpu/${arch}/mindspore-${version_map["$PYTHON_VERSION"]}-linux_${arch}.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||||
|
|
||||||
|
# check mindspore installation
|
||||||
|
sudo -u $SUDO_USER python -c "import mindspore;mindspore.run_check()"
|
||||||
|
|
@ -0,0 +1,81 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# Copyright 2022 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# Prepare environment for mindspore cpu compilation on Ubuntu 18.04.
|
||||||
|
#
|
||||||
|
# This file will:
|
||||||
|
# - change deb source to huaweicloud mirror
|
||||||
|
# - install compile dependencies via apt like cmake, gcc
|
||||||
|
# - install python3 & pip3 via apt and set it to default
|
||||||
|
# - install LLVM if LLVM is set to on.
|
||||||
|
#
|
||||||
|
# Augments:
|
||||||
|
# - PYTHON_VERSION: python version to install. [3.7(default), 3.9]
|
||||||
|
# - LLVM: whether to install optional dependency LLVM for graph kernel fusion. [on, off(default)]
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# Need root permission to run, like `sudo bash ./ubuntu-cpu-source.sh`.
|
||||||
|
# To set augments, run it as `sudo PYTHON_VERSION=3.9 LLVM=on bash ./ubuntu-cpu-source.sh`.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
PYTHON_VERSION=${PYTHON_VERSION:-3.7}
|
||||||
|
LLVM=${LLVM:-off}
|
||||||
|
|
||||||
|
available_py_version=(3.7 3.9)
|
||||||
|
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
|
||||||
|
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# use huaweicloud mirror in China
|
||||||
|
sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
||||||
|
sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
||||||
|
|
||||||
|
# base packages
|
||||||
|
apt-get update
|
||||||
|
apt-get install software-properties-common lsb-release -y
|
||||||
|
apt-get install curl tcl gcc-7 git libgmp-dev patch libnuma-dev -y
|
||||||
|
|
||||||
|
# cmake
|
||||||
|
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add -
|
||||||
|
apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main"
|
||||||
|
apt-get install cmake -y
|
||||||
|
|
||||||
|
# optional dependency LLVM for graph-computation fusion
|
||||||
|
if [[ X"$LLVM" == "Xon" ]]; then
|
||||||
|
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
|
||||||
|
add-apt-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main"
|
||||||
|
apt-get update
|
||||||
|
apt-get install llvm-12-dev -y
|
||||||
|
fi
|
||||||
|
|
||||||
|
# python
|
||||||
|
add-apt-repository -y ppa:deadsnakes/ppa
|
||||||
|
apt-get install python$PYTHON_VERSION python$PYTHON_VERSION-dev python$PYTHON_VERSION-distutils python3-pip -y
|
||||||
|
update-alternatives --install /usr/bin/python python /usr/bin/python$PYTHON_VERSION 100
|
||||||
|
# pip
|
||||||
|
sudo -u $SUDO_USER python -m pip install pip -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||||
|
update-alternatives --install /usr/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100
|
||||||
|
update-alternatives --install /usr/local/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100
|
||||||
|
sudo -u $SUDO_USER pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||||
|
|
||||||
|
# wheel
|
||||||
|
sudo -u $SUDO_USER pip install wheel
|
||||||
|
# python 3.9 needs setuptools>44.0
|
||||||
|
sudo -u $SUDO_USER pip install -U setuptools
|
||||||
|
|
||||||
|
echo "The environment is ready to clone and compile mindspore."
|
||||||
|
|
@ -1,26 +1,74 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
set -ex
|
# Copyright 2022 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.5.0}
|
# Prepare and Install mindspore gpu by conda on Ubuntu 18.04.
|
||||||
PYTHON_VERSION=${PYTHON_VERSION:-3.7.5}
|
#
|
||||||
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.5.0}
|
# This file will:
|
||||||
CUDA_VERSION=${CUDA_VERSION:-11.1.1-1}
|
# - change deb source to huaweicloud mirror
|
||||||
LIB_CUDA_VERSION=${LIB_CUDA_VERSION:-8.0.5.39-1+cuda11.1}
|
# - install mindspore dependencies via apt like gcc, libgmp
|
||||||
DISTRIBUTED=${DISTRIBUTED:-false}
|
# - install conda and set up environment for mindspore
|
||||||
CUDATOOLKIT_VERSION=${CUDATOOLKIT_VERSION:-11.1}
|
# - install mindspore-gpu by conda
|
||||||
CUDNN_VERSION=${CUDNN_VERSION:-8.0.5}
|
# - compile and install Open MPI if OPENMPI is set to on.
|
||||||
|
#
|
||||||
|
# Augments:
|
||||||
|
# - PYTHON_VERSION: python version to install. [3.7(default), 3.9]
|
||||||
|
# - MINDSPORE_VERSION: mindspore version to install, default 1.6.0
|
||||||
|
# - CUDA_VERSION: CUDA version to install. [10.1(default), 11.1]
|
||||||
|
# - OPENMPI: whether to install optional package Open MPI for distributed training. [on, off(default)]
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# Run script like `bash -i ./ubuntu-gpu-conda.sh`.
|
||||||
|
# To set augments, run it as `PYTHON_VERSION=3.9 CUDA_VERSION=11.1 OPENMPI=on bash -i ./ubuntu-gpu-conda.sh`.
|
||||||
|
|
||||||
cd /tmp
|
set -e
|
||||||
curl -O https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/Miniconda3-py37_4.10.3-Linux-x86_64.sh
|
|
||||||
bash Miniconda3-py37_4.10.3-Linux-x86_64.sh
|
|
||||||
|
|
||||||
# add conda to PATH
|
PYTHON_VERSION=${PYTHON_VERSION:-3.7}
|
||||||
echo -e 'export PATH=~/miniconda3/bin/:$PATH' >> ~/.bash_profile
|
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0}
|
||||||
echo -e '. ~/miniconda3/etc/profile.d/conda.sh' >> ~/.bash_profile
|
CUDA_VERSION=${CUDA_VERSION:-10.1}
|
||||||
source ~/.bash_profile
|
OPENMPI=${OPENMPI:-off}
|
||||||
conda init bash
|
|
||||||
# setting up conda mirror
|
available_py_version=(3.7 3.9)
|
||||||
cat >~/.condarc <<END
|
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
|
||||||
|
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
available_cuda_version=(10.1 11.1)
|
||||||
|
if [[ " ${available_cuda_version[*]} " != *" $CUDA_VERSION "* ]]; then
|
||||||
|
echo "CUDA_VERSION is '$CUDA_VERSION', but available versions are [${available_cuda_version[*]}]."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# add value to environment variable if value is not in it
|
||||||
|
add_env() {
|
||||||
|
local name=$1
|
||||||
|
if [[ ":${!name}:" != *":$2:"* ]]; then
|
||||||
|
echo -e "export $1=$2:\$$1" >> ~/.bashrc
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
install_conda() {
|
||||||
|
conda_file_name="Miniconda3-py3${PYTHON_VERSION##*.}_4.10.3-Linux-$(arch).sh"
|
||||||
|
cd /tmp
|
||||||
|
wget https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/$conda_file_name
|
||||||
|
bash $conda_file_name -b
|
||||||
|
cd -
|
||||||
|
. ~/miniconda3/etc/profile.d/conda.sh
|
||||||
|
conda init bash
|
||||||
|
# setting up conda mirror with tsinghua source
|
||||||
|
cat >~/.condarc <<END
|
||||||
channels:
|
channels:
|
||||||
- defaults
|
- defaults
|
||||||
show_channel_urls: true
|
show_channel_urls: true
|
||||||
|
|
@ -33,36 +81,57 @@ custom_channels:
|
||||||
msys2: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
msys2: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
||||||
bioconda: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
bioconda: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
||||||
menpo: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
menpo: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
||||||
pytorch: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
|
||||||
pytorch-lts: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
|
||||||
simpleitk: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
simpleitk: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
|
||||||
END
|
END
|
||||||
|
}
|
||||||
|
|
||||||
#initialize conda env and install mindspore-cpu
|
# use huaweicloud mirror in China
|
||||||
|
sudo sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
||||||
|
sudo sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
||||||
|
sudo apt-get update
|
||||||
|
|
||||||
conda create -n ms_${PYTHON_VERSION} python=${PYTHON_VERSION} -y
|
sudo apt-get install curl gcc-7 libgmp-dev -y
|
||||||
conda activate ms_${PYTHON_VERSION}
|
|
||||||
|
|
||||||
# install gmp 6.1.2, downloading gmp is slow
|
# optional openmpi for distributed training
|
||||||
# echo "install gmp start"
|
if [[ X"$OPENMPI" == "Xon" ]]; then
|
||||||
# sudo apt-get install m4 -y
|
cd /tmp
|
||||||
# cd /tmp
|
curl -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.3.tar.gz
|
||||||
# curl -O https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz
|
tar xzf openmpi-4.0.3.tar.gz
|
||||||
# xz -d gmp-6.1.2.tar.xz
|
cd openmpi-4.0.3
|
||||||
# tar xvzf gmp-6.1.2.tar && cd gmp-6.1.2
|
./configure --prefix=/usr/local/openmpi-4.0.3
|
||||||
# ./configure --prefix=/usr/local/gmp-6.1.2
|
make
|
||||||
# make
|
sudo make install
|
||||||
# sudo make install
|
set +e && source ~/.bashrc
|
||||||
# echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/gmp-6.1.2/lib' >> ~/.bash_profile
|
set -e
|
||||||
|
add_env PATH /usr/local/openmpi-4.0.3/bin
|
||||||
|
add_env LD_LIBRARY_PATH /usr/local/openmpi-4.0.3/lib
|
||||||
|
fi
|
||||||
|
|
||||||
# install mindspore-gpu with conda
|
# install conda
|
||||||
conda install mindspore-gpu=${MINDSPORE_VERSION} cudatoolkit=${CUDATOOLKIT_VERSION} -c mindspore -c conda-forge -y
|
set +e && type conda &>/dev/null
|
||||||
|
if [[ $? -eq 0 ]]; then
|
||||||
|
echo "conda has been installed, skip."
|
||||||
|
source "$(conda info --base)"/etc/profile.d/conda.sh
|
||||||
|
else
|
||||||
|
install_conda
|
||||||
|
fi
|
||||||
|
set -e
|
||||||
|
|
||||||
# check if it is the right mindspore version
|
# set up conda env and install mindspore-cpu
|
||||||
|
env_name=mindspore_py3${PYTHON_VERSION##*.}
|
||||||
|
declare -A cudnn_version_map=()
|
||||||
|
cudnn_version_map["10.1"]="7.6.5"
|
||||||
|
cudnn_version_map["11.1"]="8.1.0"
|
||||||
|
conda create -n $env_name python=${PYTHON_VERSION} -y
|
||||||
|
conda activate $env_name
|
||||||
|
conda install mindspore-gpu=${MINDSPORE_VERSION} \
|
||||||
|
cudatoolkit=${CUDA_VERSION} cudnn=${cudnn_version_map[$CUDA_VERSION]} -c mindspore -c conda-forge -y
|
||||||
|
|
||||||
|
# check mindspore installation
|
||||||
python -c "import mindspore;mindspore.run_check()"
|
python -c "import mindspore;mindspore.run_check()"
|
||||||
|
|
||||||
# check if it can be run with GPU
|
# check if it can be run with GPU
|
||||||
|
cd /tmp
|
||||||
cat > example.py <<END
|
cat > example.py <<END
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from mindspore import Tensor
|
from mindspore import Tensor
|
||||||
|
|
@ -74,5 +143,5 @@ x = Tensor(np.ones([1,3,3,4]).astype(np.float32))
|
||||||
y = Tensor(np.ones([1,3,3,4]).astype(np.float32))
|
y = Tensor(np.ones([1,3,3,4]).astype(np.float32))
|
||||||
print(ops.add(x, y))
|
print(ops.add(x, y))
|
||||||
END
|
END
|
||||||
|
python example.py
|
||||||
python example.py
|
cd -
|
||||||
|
|
|
||||||
|
|
@ -1,124 +1,144 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
set -ex
|
# Copyright 2022 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
PYTHON_VERSION=${PYTHON_VERSION:-3.7.5}
|
# Prepare and Install mindspore gpu by pip on Ubuntu 18.04.
|
||||||
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.5.0}
|
#
|
||||||
CUDA_VERSION=${CUDA_VERSION:-8.0.5.39-1+cuda11.1}
|
# This file will:
|
||||||
DISTRIBUTED=${DISTRIBUTED:-false}
|
# - change deb source to huaweicloud mirror
|
||||||
CUDA_VERSION=${CUDA_VERSION:-11.1.1-1}
|
# - install mindspore dependencies via apt like gcc, libgmp
|
||||||
CUDA_INSTALL_PATH=${CUDA_INSTALL_PATH:-cuda-11.1}
|
# - install python3 & pip3 via apt and set it to default
|
||||||
LIBNCCL2_VERSION=${LIBNCCL2_VERSION:-2.7.8-1+cuda11.1}
|
# - install CUDA by run file and cudnn via apt.
|
||||||
ARCH=$(uname -m)
|
# - install mindspore-cpu within new installed python by pip
|
||||||
|
# - compile and install Open MPI if OPENMPI is set to on.
|
||||||
|
#
|
||||||
|
# Augments:
|
||||||
|
# - PYTHON_VERSION: python version to install. [3.7(default), 3.9]
|
||||||
|
# - MINDSPORE_VERSION: mindspore version to install, default 1.6.0
|
||||||
|
# - CUDA_VERSION: CUDA version to install. [10.1(default), 11.1]
|
||||||
|
# - OPENMPI: whether to install optional package Open MPI for distributed training. [on, off(default)]
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# Need root permission to run, like `sudo bash -i ./ubuntu-gpu-pip.sh`.
|
||||||
|
# To set augments, run it as `sudo PYTHON_VERSION=3.9 CUDA_VERSION=11.1 OPENMPI=on bash -i ./ubuntu-gpu-pip.sh`.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
PYTHON_VERSION=${PYTHON_VERSION:-3.7}
|
||||||
|
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.6.0}
|
||||||
|
CUDA_VERSION=${CUDA_VERSION:-10.1}
|
||||||
|
OPENMPI=${OPENMPI:-off}
|
||||||
|
|
||||||
|
available_py_version=(3.7 3.9)
|
||||||
|
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
|
||||||
|
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
available_cuda_version=(10.1 11.1)
|
||||||
|
if [[ " ${available_cuda_version[*]} " != *" $CUDA_VERSION "* ]]; then
|
||||||
|
echo "CUDA_VERSION is '$CUDA_VERSION', but available versions are [${available_cuda_version[*]}]."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
declare -A minimum_driver_version_map=()
|
||||||
|
minimum_driver_version_map["10.1"]="418.39"
|
||||||
|
minimum_driver_version_map["11.1"]="450.80.02"
|
||||||
|
driver_version=$(modinfo nvidia | grep ^version | awk '{printf $2}')
|
||||||
|
if [[ $driver_version < ${minimum_driver_version_map[$CUDA_VERSION]} ]]; then
|
||||||
|
echo "CUDA $CUDA_VERSION minimum required driver version is ${minimum_driver_version_map[$CUDA_VERSION]}, \
|
||||||
|
but current nvidia driver version is $driver_version, please upgrade your driver manually."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
cuda_name="cuda-$CUDA_VERSION"
|
||||||
|
|
||||||
declare -A version_map=()
|
declare -A version_map=()
|
||||||
version_map["3.7.5"]="${MINDSPORE_VERSION}-cp37-cp37m"
|
version_map["3.7"]="${MINDSPORE_VERSION}-cp37-cp37m"
|
||||||
version_map["3.9.0"]="${MINDSPORE_VERSION}-cp39-cp39m"
|
version_map["3.9"]="${MINDSPORE_VERSION}-cp39-cp39"
|
||||||
|
|
||||||
#use huaweicloud mirror in China
|
# add value to environment variable if value is not in it
|
||||||
sudo sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
add_env() {
|
||||||
sudo sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
local name=$1
|
||||||
sudo apt-get update
|
if [[ ":${!name}:" != *":$2:"* ]]; then
|
||||||
|
echo -e "export $1=$2:\$$1" >> ~/.bashrc
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# install python 3.7 and make it default
|
# use huaweicloud mirror in China
|
||||||
sudo apt-get install gcc-7 libgmp-dev curl python3.7 openssl ubuntu-drivers-common openssl software-properties-common -y
|
sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
||||||
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.7 100
|
sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
||||||
|
apt-get update
|
||||||
|
|
||||||
|
apt-get install curl make gcc-7 libgmp-dev linux-headers-"$(uname -r)" -y
|
||||||
|
|
||||||
|
# python
|
||||||
|
add-apt-repository -y ppa:deadsnakes/ppa
|
||||||
|
apt-get install python$PYTHON_VERSION python$PYTHON_VERSION-distutils python3-pip -y
|
||||||
|
update-alternatives --install /usr/bin/python python /usr/bin/python$PYTHON_VERSION 100
|
||||||
|
# pip
|
||||||
|
sudo -u $SUDO_USER python -m pip install pip -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||||
|
update-alternatives --install /usr/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100
|
||||||
|
update-alternatives --install /usr/local/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100
|
||||||
|
sudo -u $SUDO_USER pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||||
|
|
||||||
|
# install cuda/cudnn
|
||||||
cd /tmp
|
cd /tmp
|
||||||
curl -O https://bootstrap.pypa.io/get-pip.py
|
declare -A cuda_url_map=()
|
||||||
sudo python get-pip.py
|
cuda_url_map["10.1"]=https://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run
|
||||||
|
cuda_url_map["11.1"]=https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_455.32.00_linux.run
|
||||||
|
cuda_url=${cuda_url_map[$CUDA_VERSION]}
|
||||||
|
wget $cuda_url
|
||||||
|
sh ${cuda_url##*/} --silent --toolkit
|
||||||
|
cd -
|
||||||
|
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
|
||||||
|
add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /"
|
||||||
|
add-apt-repository "deb https://developer.download.nvidia.cn/compute/machine-learning/repos/ubuntu1804/x86_64/ /"
|
||||||
|
apt-get update
|
||||||
|
declare -A cudnn_name_map=()
|
||||||
|
cudnn_name_map["10.1"]="libcudnn7=7.6.5.32-1+cuda10.1 libcudnn7-dev=7.6.5.32-1+cuda10.1"
|
||||||
|
cudnn_name_map["11.1"]="libcudnn8=8.0.4.30-1+cuda11.1 libcudnn8-dev=8.0.4.30-1+cuda11.1"
|
||||||
|
apt-get install --no-install-recommends ${cudnn_name_map[$CUDA_VERSION]} -y
|
||||||
|
|
||||||
# add pip mirror
|
# add cuda to path
|
||||||
mkdir -p ~/.pip
|
set +e && source ~/.bashrc
|
||||||
cat > ~/.pip/pip.conf <<END
|
set -e
|
||||||
[global]
|
add_env PATH /usr/local/cuda/bin
|
||||||
index-url = https://repo.huaweicloud.com/repository/pypi/simple
|
add_env LD_LIBRARY_PATH /usr/local/cuda/lib64
|
||||||
trusted-host = repo.huaweicloud.com
|
add_env LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu
|
||||||
timeout = 120
|
set +e && source ~/.bashrc
|
||||||
END
|
set -e
|
||||||
|
|
||||||
# install nvidia driver if not presented
|
# optional openmpi for distributed training
|
||||||
# root@ecs-gpu-testing:~# ubuntu-drivers devices
|
if [[ X"$OPENMPI" == "Xon" ]]; then
|
||||||
# == /sys/devices/pci0000:20/0000:20:00.0/0000:21:01.0 ==
|
cd /tmp
|
||||||
# modalias : pci:v000010DEd00001EB8sv000010DEsd000012A2bc03sc02i00
|
sudo -u $SUDO_USER curl -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.3.tar.gz
|
||||||
# vendor : NVIDIA Corporation
|
sudo -u $SUDO_USER tar xzf openmpi-4.0.3.tar.gz
|
||||||
# driver : nvidia-driver-418 - third-party non-free
|
cd openmpi-4.0.3
|
||||||
# driver : nvidia-driver-450 - third-party non-free
|
sudo -u $SUDO_USER ./configure --prefix=/usr/local/openmpi-4.0.3
|
||||||
# driver : nvidia-driver-460 - third-party non-free
|
sudo -u $SUDO_USER make
|
||||||
# driver : nvidia-driver-450-server - distro non-free
|
make install
|
||||||
# driver : nvidia-driver-460-server - distro non-free
|
add_env PATH /usr/local/openmpi-4.0.3/bin
|
||||||
# driver : nvidia-driver-440 - third-party non-free
|
add_env LD_LIBRARY_PATH /usr/local/openmpi-4.0.3/lib
|
||||||
# driver : nvidia-driver-418-server - distro non-free
|
fi
|
||||||
# driver : nvidia-driver-465 - third-party non-free
|
|
||||||
# driver : nvidia-driver-470 - third-party non-free recommended #pick the latest one
|
|
||||||
# driver : nvidia-driver-410 - third-party non-free
|
|
||||||
# driver : nvidia-driver-470-server - distro non-free
|
|
||||||
# driver : nvidia-driver-455 - third-party non-free
|
|
||||||
# driver : xserver-xorg-video-nouveau - distro free builtin
|
|
||||||
# sudo apt-get install nvidia-driver-470 -y
|
|
||||||
# nvidia-smi # run this to check the driver is working
|
|
||||||
#root@ecs-testing:~# nvidia-smi
|
|
||||||
#Thu Dec 30 21:06:13 2021
|
|
||||||
#+-----------------------------------------------------------------------------+
|
|
||||||
#| NVIDIA-SMI 460.73.01 Driver Version: 460.73.01 CUDA Version: 11.2 |
|
|
||||||
#|-------------------------------+----------------------+----------------------+
|
|
||||||
#| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
|
|
||||||
#| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
|
|
||||||
#| | | MIG M. |
|
|
||||||
#|===============================+======================+======================|
|
|
||||||
#| 0 Tesla T4 Off | 00000000:21:01.0 Off | 0 |
|
|
||||||
#| N/A 61C P0 29W / 70W | 0MiB / 15109MiB | 0% Default |
|
|
||||||
#| | | N/A |
|
|
||||||
#+-------------------------------+----------------------+----------------------+
|
|
||||||
#
|
|
||||||
#+-----------------------------------------------------------------------------+
|
|
||||||
#| Processes: |
|
|
||||||
#| GPU GI CI PID Type Process name GPU Memory |
|
|
||||||
#| ID ID Usage |
|
|
||||||
#|=============================================================================|
|
|
||||||
#| No running processes found |
|
|
||||||
#+-----------------------------------------------------------------------------+
|
|
||||||
|
|
||||||
# install cuda/cudnn/nccl2 with apt-get
|
arch=`uname -m`
|
||||||
# another option is to use linux.run https://developer.download.nvidia.com/compute/cuda/11.1.0/local_installers/cuda_11.1.0_455.23.05_linux.run
|
sudo -u $SUDO_USER pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${MINDSPORE_VERSION}/MindSpore/gpu/${arch}/${cuda_name}/mindspore_gpu-${version_map["$PYTHON_VERSION"]}-linux_${arch}.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||||
sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
|
|
||||||
sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /"
|
|
||||||
sudo add-apt-repository "deb https://developer.download.nvidia.cn/compute/machine-learning/repos/ubuntu1804/x86_64/ /"
|
|
||||||
sudo apt-get update
|
|
||||||
sudo apt-get -y install cuda=${CUDA_VERSION}
|
|
||||||
|
|
||||||
# add cuda to path
|
# check mindspore installation
|
||||||
cat >> ~/.bash_profile <<END
|
|
||||||
export PATH=/usr/local/cuda/bin:\$PATH
|
|
||||||
export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/cuda/lib64
|
|
||||||
END
|
|
||||||
source ~/.bash_profile
|
|
||||||
echo "cuda install success."
|
|
||||||
|
|
||||||
sudo apt-get install -y libcudnn8=${CUDA_VERSION} libcudnn8-dev=${CUDA_VERSION} libnccl2=${LIBNCCL2_VERSION} libnccl-dev=${LIBNCCL2_VERSION}
|
|
||||||
|
|
||||||
# optional (tensort for serving, openmpi for distributed training)
|
|
||||||
# uncomment this to compile openmpi
|
|
||||||
# cd /tmp
|
|
||||||
# curl -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.3.tar.gz
|
|
||||||
# tar xvzf openmpi-4.0.3.tar.gz
|
|
||||||
# cd openmpi-4.0.3
|
|
||||||
# ./configure --prefix=/usr/local/openmpi
|
|
||||||
# make
|
|
||||||
# sudo make install
|
|
||||||
# echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/openmpi/lib' >> ~/.bash_profile
|
|
||||||
# echo 'export PATH=$PATH:/usr/local/openmpi/bin' >> ~/.bash_profile
|
|
||||||
# source ~/.bash_profile
|
|
||||||
# reference this to install tensorrt https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html#downloading
|
|
||||||
|
|
||||||
echo "install mindspore gpu ${MINDSPORE_VERSION}"
|
|
||||||
pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${MINDSPORE_VERSION}/MindSpore/gpu/${ARCH}/${CUDA_INSTALL_PATH}/mindspore_gpu-${version_map["$PYTHON_VERSION"]}-linux_${ARCH}.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.tuna.tsinghua.edu.cn/simple
|
|
||||||
|
|
||||||
|
|
||||||
# check if it is the right mindspore version
|
|
||||||
python -c "import mindspore;mindspore.run_check()"
|
python -c "import mindspore;mindspore.run_check()"
|
||||||
|
|
||||||
# check if it can be run with GPU
|
# check if it can be run with GPU
|
||||||
|
cd /tmp
|
||||||
cat > example.py <<END
|
cat > example.py <<END
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from mindspore import Tensor
|
from mindspore import Tensor
|
||||||
|
|
@ -130,5 +150,5 @@ x = Tensor(np.ones([1,3,3,4]).astype(np.float32))
|
||||||
y = Tensor(np.ones([1,3,3,4]).astype(np.float32))
|
y = Tensor(np.ones([1,3,3,4]).astype(np.float32))
|
||||||
print(ops.add(x, y))
|
print(ops.add(x, y))
|
||||||
END
|
END
|
||||||
|
python example.py
|
||||||
python example.py
|
cd -
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,151 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# Copyright 2022 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# Prepare environment for mindspore gpu compilation on Ubuntu 18.04.
|
||||||
|
#
|
||||||
|
# This file will:
|
||||||
|
# - change deb source to huaweicloud mirror
|
||||||
|
# - install compile dependencies via apt like cmake, gcc
|
||||||
|
# - install python3 & pip3 via apt and set it to default
|
||||||
|
# - install CUDA by run file and cudnn via apt.
|
||||||
|
# - compile and install Open MPI if OPENMPI is set to on.
|
||||||
|
# - install LLVM if LLVM is set to on.
|
||||||
|
#
|
||||||
|
# Augments:
|
||||||
|
# - PYTHON_VERSION: python version to install. [3.7(default), 3.9]
|
||||||
|
# - CUDA_VERSION: CUDA version to install. [10.1(default), 11.1]
|
||||||
|
# - OPENMPI: whether to install optional package Open MPI for distributed training. [on, off(default)]
|
||||||
|
# - LLVM: whether to install optional dependency LLVM for graph kernel fusion. [on, off(default)]
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# Need root permission to run, like `sudo bash -i ./ubuntu-gpu-source.sh`.
|
||||||
|
# To set augments, run it as `sudo PYTHON_VERSION=3.9 CUDA_VERSION=11.1 OPENMPI=on bash -i ./ubuntu-gpu-source.sh`.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
PYTHON_VERSION=${PYTHON_VERSION:-3.7}
|
||||||
|
CUDA_VERSION=${CUDA_VERSION:-10.1}
|
||||||
|
OPENMPI=${OPENMPI:-off}
|
||||||
|
LLVM=${LLVM:-off}
|
||||||
|
|
||||||
|
available_py_version=(3.7 3.9)
|
||||||
|
if [[ " ${available_py_version[*]} " != *" $PYTHON_VERSION "* ]]; then
|
||||||
|
echo "PYTHON_VERSION is '$PYTHON_VERSION', but available versions are [${available_py_version[*]}]."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
available_cuda_version=(10.1 11.1)
|
||||||
|
if [[ " ${available_cuda_version[*]} " != *" $CUDA_VERSION "* ]]; then
|
||||||
|
echo "CUDA_VERSION is '$CUDA_VERSION', but available versions are [${available_cuda_version[*]}]."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
declare -A minimum_driver_version_map=()
|
||||||
|
minimum_driver_version_map["10.1"]="418.39"
|
||||||
|
minimum_driver_version_map["11.1"]="450.80.02"
|
||||||
|
driver_version=$(modinfo nvidia | grep ^version | awk '{printf $2}')
|
||||||
|
if [[ $driver_version < ${minimum_driver_version_map[$CUDA_VERSION]} ]]; then
|
||||||
|
echo "CUDA $CUDA_VERSION minimum required driver version is ${minimum_driver_version_map[$CUDA_VERSION]}, \
|
||||||
|
but current nvidia driver version is $driver_version, please upgrade your driver manually."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# add value to environment variable if value is not in it
|
||||||
|
add_env() {
|
||||||
|
local name=$1
|
||||||
|
if [[ ":${!name}:" != *":$2:"* ]]; then
|
||||||
|
echo -e "export $1=$2:\$$1" >> ~/.bashrc
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# use huaweicloud mirror in China
|
||||||
|
sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
||||||
|
sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
||||||
|
|
||||||
|
# base packages
|
||||||
|
apt-get update
|
||||||
|
apt-get install software-properties-common lsb-release -y
|
||||||
|
apt-get install curl tcl automake autoconf libtool gcc-7 git libgmp-dev patch libnuma-dev flex -y
|
||||||
|
|
||||||
|
# cmake
|
||||||
|
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add -
|
||||||
|
apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main"
|
||||||
|
apt-get install cmake -y
|
||||||
|
|
||||||
|
# optional dependency LLVM for graph-computation fusion
|
||||||
|
if [[ X"$LLVM" == "Xon" ]]; then
|
||||||
|
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
|
||||||
|
add-apt-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main"
|
||||||
|
apt-get update
|
||||||
|
apt-get install llvm-12-dev -y
|
||||||
|
fi
|
||||||
|
|
||||||
|
# optional openmpi for distributed training
|
||||||
|
if [[ X"$OPENMPI" == "Xon" ]]; then
|
||||||
|
origin_wd=$PWD
|
||||||
|
cd /tmp
|
||||||
|
sudo -u $SUDO_USER curl -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.3.tar.gz
|
||||||
|
sudo -u $SUDO_USER tar xzf openmpi-4.0.3.tar.gz
|
||||||
|
cd openmpi-4.0.3
|
||||||
|
sudo -u $SUDO_USER ./configure --prefix=/usr/local/openmpi-4.0.3
|
||||||
|
sudo -u $SUDO_USER make
|
||||||
|
make install
|
||||||
|
add_env PATH /usr/local/openmpi-4.0.3/bin
|
||||||
|
add_env LD_LIBRARY_PATH /usr/local/openmpi-4.0.3/lib
|
||||||
|
cd $origin_wd
|
||||||
|
fi
|
||||||
|
|
||||||
|
# python
|
||||||
|
add-apt-repository -y ppa:deadsnakes/ppa
|
||||||
|
apt-get install python$PYTHON_VERSION python$PYTHON_VERSION-dev python$PYTHON_VERSION-distutils python3-pip -y
|
||||||
|
update-alternatives --install /usr/bin/python python /usr/bin/python$PYTHON_VERSION 100
|
||||||
|
# pip
|
||||||
|
sudo -u $SUDO_USER python -m pip install pip -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||||
|
update-alternatives --install /usr/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100
|
||||||
|
update-alternatives --install /usr/local/bin/pip pip ~/.local/bin/pip$PYTHON_VERSION 100
|
||||||
|
sudo -u $SUDO_USER pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||||
|
|
||||||
|
# install cuda/cudnn
|
||||||
|
cd /tmp
|
||||||
|
declare -A cuda_url_map=()
|
||||||
|
cuda_url_map["10.1"]=https://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run
|
||||||
|
cuda_url_map["11.1"]=https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_455.32.00_linux.run
|
||||||
|
cuda_url=${cuda_url_map[$CUDA_VERSION]}
|
||||||
|
wget $cuda_url
|
||||||
|
sh ${cuda_url##*/} --silent --toolkit
|
||||||
|
cd -
|
||||||
|
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
|
||||||
|
add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /"
|
||||||
|
add-apt-repository "deb https://developer.download.nvidia.cn/compute/machine-learning/repos/ubuntu1804/x86_64/ /"
|
||||||
|
apt-get update
|
||||||
|
declare -A cudnn_name_map=()
|
||||||
|
cudnn_name_map["10.1"]="libcudnn7=7.6.5.32-1+cuda10.1 libcudnn7-dev=7.6.5.32-1+cuda10.1"
|
||||||
|
cudnn_name_map["11.1"]="libcudnn8=8.0.4.30-1+cuda11.1 libcudnn8-dev=8.0.4.30-1+cuda11.1"
|
||||||
|
apt-get install --no-install-recommends ${cudnn_name_map[$CUDA_VERSION]} -y
|
||||||
|
|
||||||
|
# add cuda to path
|
||||||
|
set +e && source ~/.bashrc
|
||||||
|
set -e
|
||||||
|
add_env PATH /usr/local/cuda/bin
|
||||||
|
add_env LD_LIBRARY_PATH /usr/local/cuda/lib64
|
||||||
|
add_env LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu
|
||||||
|
set +e && source ~/.bashrc
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# wheel
|
||||||
|
sudo -u $SUDO_USER pip install wheel
|
||||||
|
# python 3.9 needs setuptools>44.0
|
||||||
|
sudo -u $SUDO_USER pip install -U setuptools
|
||||||
|
|
||||||
|
echo "The environment is ready to clone and compile mindspore."
|
||||||
|
|
@ -1,44 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -ex
|
|
||||||
|
|
||||||
# sudo cp -a /etc/apt/sources.list /etc/apt/sources.list.bak 单独执行
|
|
||||||
PYTHON_VERSION=${PYTHON_VERSION:-3.7.5}
|
|
||||||
MINDSPORE_VERSION=${MINDSPORE_VERSION:-1.5.0}
|
|
||||||
ARCH=`uname -m`
|
|
||||||
|
|
||||||
declare -A version_map=()
|
|
||||||
version_map["3.7.5"]="${MINDSPORE_VERSION}-cp37-cp37m"
|
|
||||||
|
|
||||||
#use huaweicloud mirror in China
|
|
||||||
sudo sed -i "s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
|
||||||
sudo sed -i "s@http://.*security.ubuntu.com@http://repo.huaweicloud.com@g" /etc/apt/sources.list
|
|
||||||
sudo apt-get update
|
|
||||||
|
|
||||||
# install python 3.7 and make it default
|
|
||||||
sudo apt-get install gcc-7 libgmp-dev curl python3.7 -y
|
|
||||||
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.7 100
|
|
||||||
|
|
||||||
cd /tmp
|
|
||||||
curl -O https://bootstrap.pypa.io/get-pip.py
|
|
||||||
sudo python get-pip.py
|
|
||||||
|
|
||||||
pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${MINDSPORE_VERSION}/MindSpore/cpu/${ARCH}/mindspore-${version_map["$PYTHON_VERSION"]}-linux_${ARCH}.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.tuna.tsinghua.edu.cn/simple
|
|
||||||
|
|
||||||
# check if it is the right mindspore version
|
|
||||||
python -c "import mindspore;mindspore.run_check()"
|
|
||||||
|
|
||||||
# check if it can be run with GPU
|
|
||||||
|
|
||||||
cat > example.py <<END
|
|
||||||
import numpy as np
|
|
||||||
from mindspore import Tensor
|
|
||||||
import mindspore.ops as ops
|
|
||||||
import mindspore.context as context
|
|
||||||
|
|
||||||
context.set_context(device_target="GPU")
|
|
||||||
x = Tensor(np.ones([1,3,3,4]).astype(np.float32))
|
|
||||||
y = Tensor(np.ones([1,3,3,4]).astype(np.float32))
|
|
||||||
print(ops.add(x, y))
|
|
||||||
END
|
|
||||||
|
|
||||||
python example.py
|
|
||||||
Loading…
Reference in New Issue