Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

replace tab with spaces

This commit is contained in:
xsmq 2021-02-23 09:41:47 +08:00
parent 57f6c17933
commit 923b53d2e6
30 changed files with 168 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ops/operations/image_ops.py
View File

@ -29,7 +29,7 @@ class CropAndResize(PrimitiveWithInfer):
In case that the output shape depends on crop_size, the crop_size must be constant. In case that the output shape depends on crop_size, the crop_size must be constant.
Args: Args:
method (str): An optional string that specifies the sampling method for resizing. method (str): An optional string that specifies the sampling method for resizing.
It can be "bilinear", "nearest" or "bilinear_v2". The option "bilinear" stands for standard bilinear It can be "bilinear", "nearest" or "bilinear_v2". The option "bilinear" stands for standard bilinear
interpolation algorithm, while "bilinear_v2" may result in better result in some cases. Default: "bilinear" interpolation algorithm, while "bilinear_v2" may result in better result in some cases. Default: "bilinear"
extrapolation_value (float): An optional float value used extrapolation, if applicable. Default: 0. extrapolation_value (float): An optional float value used extrapolation, if applicable. Default: 0.

16
model_zoo/official/cv/maskrcnn_mobilenetv1/scripts/run_distribute_train.sh
View File

@ -16,16 +16,16 @@
if [ $# != 2 ] && [ $# != 1 ] if [ $# != 2 ] && [ $# != 1 ]
then then
echo "Usage: sh run_distribute_train.sh [RANK_TABLE_FILE] [PRETRAINED_PATH](optional)" echo "Usage: sh run_distribute_train.sh [RANK_TABLE_FILE] [PRETRAINED_PATH](optional)"
exit 1 exit 1
fi fi
get_real_path(){ get_real_path(){
if [ "${1:0:1}" == "/" ]; then if [ "${1:0:1}" == "/" ]; then
echo "$1" echo "$1"
else else
echo "$(realpath -m $PWD/$1)" echo "$(realpath -m $PWD/$1)"
fi fi
} }
PATH1=$(get_real_path $1) PATH1=$(get_real_path $1)
PATH2=$2 PATH2=$2
@ -40,7 +40,7 @@ fi
if [ ! -f $PATH1 ] if [ ! -f $PATH1 ]
then then
echo "error: RANK_TABLE_FILE=$PATH1 is not a file" echo "error: RANK_TABLE_FILE=$PATH1 is not a file"
exit 1 exit 1
fi fi
ulimit -u unlimited ulimit -u unlimited

14
model_zoo/official/cv/maskrcnn_mobilenetv1/scripts/run_standalone_train.sh
View File

@ -16,16 +16,16 @@
if [ $# != 1 ] && [ $# != 0 ] if [ $# != 1 ] && [ $# != 0 ]
then then
echo "Usage: sh run_standalone_train.sh [PRETRAINED_PATH](optional)" echo "Usage: sh run_standalone_train.sh [PRETRAINED_PATH](optional)"
exit 1 exit 1
fi fi
get_real_path(){ get_real_path(){
if [ "${1:0:1}" == "/" ]; then if [ "${1:0:1}" == "/" ]; then
echo "$1" echo "$1"
else else
echo "$(realpath -m $PWD/$1)" echo "$(realpath -m $PWD/$1)"
fi fi
} }
if [ $# == 1 ] if [ $# == 1 ]

10
model_zoo/official/cv/mobilenetv1/scripts/run_distribute_train.sh
View File

@ -17,13 +17,13 @@
if [ $# != 3 ] && [ $# != 4 ] if [ $# != 3 ] && [ $# != 4 ]
then then
echo "Usage: sh run_distribute_train.sh [cifar10|imagenet2012] [RANK_TABLE_FILE] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" echo "Usage: sh run_distribute_train.sh [cifar10|imagenet2012] [RANK_TABLE_FILE] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)"
exit 1 exit 1
fi fi
if [ $1 != "cifar10" ] && [ $1 != "imagenet2012" ] if [ $1 != "cifar10" ] && [ $1 != "imagenet2012" ]
then then
echo "error: the selected dataset is neither cifar10 nor imagenet2012" echo "error: the selected dataset is neither cifar10 nor imagenet2012"
exit 1 exit 1
fi fi
get_real_path(){ get_real_path(){
@ -45,19 +45,19 @@ fi
if [ ! -f $PATH1 ] if [ ! -f $PATH1 ]
then then
echo "error: RANK_TABLE_FILE=$PATH1 is not a file" echo "error: RANK_TABLE_FILE=$PATH1 is not a file"
exit 1 exit 1
fi fi
if [ ! -d $PATH2 ] if [ ! -d $PATH2 ]
then then
echo "error: DATASET_PATH=$PATH2 is not a directory" echo "error: DATASET_PATH=$PATH2 is not a directory"
exit 1 exit 1
fi fi
if [ $# == 4 ] && [ ! -f $PATH3 ] if [ $# == 4 ] && [ ! -f $PATH3 ]
then then
echo "error: PRETRAINED_CKPT_PATH=$PATH3 is not a file" echo "error: PRETRAINED_CKPT_PATH=$PATH3 is not a file"
exit 1 exit 1
fi fi
ulimit -u unlimited ulimit -u unlimited

4
model_zoo/official/cv/resnet/scripts/run_distribute_train.sh
View File

@ -16,8 +16,8 @@
if [ $# != 4 ] && [ $# != 5 ] if [ $# != 4 ] && [ $# != 5 ]
then then
echo "Usage: sh run_distribute_train.sh [resnet50|resnet101|se-resnet50] [cifar10|imagenet2012] [RANK_TABLE_FILE] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" echo "Usage: sh run_distribute_train.sh [resnet50|resnet101|se-resnet50] [cifar10|imagenet2012] [RANK_TABLE_FILE] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)"
exit 1 exit 1
fi fi
if [ $1 != "resnet50" ] && [ $1 != "resnet101" ] && [ $1 != "se-resnet50" ] if [ $1 != "resnet50" ] && [ $1 != "resnet101" ] && [ $1 != "se-resnet50" ]

26
model_zoo/official/cv/resnet/scripts/run_distribute_train_gpu.sh
View File

@ -16,26 +16,26 @@
if [ $# != 3 ] && [ $# != 4 ] if [ $# != 3 ] && [ $# != 4 ]
then then
echo "Usage: sh run_distribute_train_gpu.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" echo "Usage: sh run_distribute_train_gpu.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)"
exit 1 exit 1
fi fi
if [ $1 != "resnet50" ] && [ $1 != "resnet101" ] if [ $1 != "resnet50" ] && [ $1 != "resnet101" ]
then then
echo "error: the selected net is neither resnet50 nor resnet101" echo "error: the selected net is neither resnet50 nor resnet101"
exit 1 exit 1
fi fi
if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ] if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ]
then then
echo "error: the selected dataset is neither cifar10 nor imagenet2012" echo "error: the selected dataset is neither cifar10 nor imagenet2012"
exit 1 exit 1
fi fi
if [ $1 == "resnet101" ] && [ $2 == "cifar10" ] if [ $1 == "resnet101" ] && [ $2 == "cifar10" ]
then then
echo "error: training resnet101 with cifar10 dataset is unsupported now!" echo "error: training resnet101 with cifar10 dataset is unsupported now!"
exit 1 exit 1
fi fi
@ -58,13 +58,13 @@ fi
if [ ! -d $PATH1 ] if [ ! -d $PATH1 ]
then then
echo "error: DATASET_PATH=$PATH1 is not a directory" echo "error: DATASET_PATH=$PATH1 is not a directory"
exit 1 exit 1
fi fi
if [ $# == 5 ] && [ ! -f $PATH2 ] if [ $# == 5 ] && [ ! -f $PATH2 ]
then then
echo "error: PRETRAINED_CKPT_PATH=$PATH2 is not a file" echo "error: PRETRAINED_CKPT_PATH=$PATH2 is not a file"
exit 1 exit 1
fi fi
ulimit -u unlimited ulimit -u unlimited
@ -80,14 +80,14 @@ cd ./train_parallel || exit
if [ $# == 3 ] if [ $# == 3 ]
then then
mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \ mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \
python train.py --net=$1 --dataset=$2 --run_distribute=True \ python train.py --net=$1 --dataset=$2 --run_distribute=True \
--device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 &> log & --device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 &> log &
fi fi
if [ $# == 4 ] if [ $# == 4 ]
then then
mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \ mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \
python train.py --net=$1 --dataset=$2 --run_distribute=True \ python train.py --net=$1 --dataset=$2 --run_distribute=True \
--device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --pre_trained=$PATH2 &> log & --device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --pre_trained=$PATH2 &> log &
fi fi

8
model_zoo/official/cv/resnet/scripts/run_eval_gpu_resnet_benckmark.sh
View File

@ -16,10 +16,10 @@
if [ $# != 1 ] && [ $# != 2 ] && [ $# != 3 ] && [ $# != 4 ] && [ $# != 5 ] if [ $# != 1 ] && [ $# != 2 ] && [ $# != 3 ] && [ $# != 4 ] && [ $# != 5 ]
then then
echo "Usage: sh run_eval_gpu_resnet_benchmark.sh [DATASET_PATH] [CKPT_PATH] [BATCH_SIZE](optional) \ echo "Usage: sh run_eval_gpu_resnet_benchmark.sh [DATASET_PATH] [CKPT_PATH] [BATCH_SIZE](optional) \
[DTYPE](optional)" [DTYPE](optional)"
echo "Example: sh run_eval_gpu_resnet_benchmark.sh /path/imagenet/train /path/ckpt 256 FP16" echo "Example: sh run_eval_gpu_resnet_benchmark.sh /path/imagenet/train /path/ckpt 256 FP16"
exit 1 exit 1
fi fi
get_real_path(){ get_real_path(){

8
model_zoo/official/cv/resnet/scripts/run_gpu_resnet_benchmark.sh
View File

@ -16,10 +16,10 @@
if [ $# != 1 ] && [ $# != 2 ] && [ $# != 3 ] && [ $# != 4 ] && [ $# != 5 ] if [ $# != 1 ] && [ $# != 2 ] && [ $# != 3 ] && [ $# != 4 ] && [ $# != 5 ]
then then
echo "Usage: sh run_gpu_resnet_benchmark.sh [DATASET_PATH] [BATCH_SIZE](optional) [DTYPE](optional)\ echo "Usage: sh run_gpu_resnet_benchmark.sh [DATASET_PATH] [BATCH_SIZE](optional) [DTYPE](optional)\
[DEVICE_NUM](optional) [SAVE_CKPT](optional) [SAVE_PATH](optional)" [DEVICE_NUM](optional) [SAVE_CKPT](optional) [SAVE_PATH](optional)"
echo "Example: sh run_gpu_resnet_benchmark.sh /path/imagenet/train 256 FP16 8 true /path/ckpt" echo "Example: sh run_gpu_resnet_benchmark.sh /path/imagenet/train 256 FP16 8 true /path/ckpt"
exit 1 exit 1
fi fi
get_real_path(){ get_real_path(){

16
model_zoo/official/cv/resnet/scripts/run_parameter_server_train.sh
View File

@ -16,26 +16,26 @@
if [ $# != 4 ] && [ $# != 5 ] if [ $# != 4 ] && [ $# != 5 ]
then then
echo "Usage: sh run_distribute_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [RANK_TABLE_FILE] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" echo "Usage: sh run_distribute_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [RANK_TABLE_FILE] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)"
exit 1 exit 1
fi fi
if [ $1 != "resnet50" ] && [ $1 != "resnet101" ] if [ $1 != "resnet50" ] && [ $1 != "resnet101" ]
then then
echo "error: the selected net is neither resnet50 nor resnet101" echo "error: the selected net is neither resnet50 nor resnet101"
exit 1 exit 1
fi fi
if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ] if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ]
then then
echo "error: the selected dataset is neither cifar10 nor imagenet2012" echo "error: the selected dataset is neither cifar10 nor imagenet2012"
exit 1 exit 1
fi fi
if [ $1 == "resnet101" ] && [ $2 == "cifar10" ] if [ $1 == "resnet101" ] && [ $2 == "cifar10" ]
then then
echo "error: training resnet101 with cifar10 dataset is unsupported now!" echo "error: training resnet101 with cifar10 dataset is unsupported now!"
exit 1 exit 1
fi fi
@ -58,19 +58,19 @@ fi
if [ ! -f $PATH1 ] if [ ! -f $PATH1 ]
then then
echo "error: RANK_TABLE_FILE=$PATH1 is not a file" echo "error: RANK_TABLE_FILE=$PATH1 is not a file"
exit 1 exit 1
fi fi
if [ ! -d $PATH2 ] if [ ! -d $PATH2 ]
then then
echo "error: DATASET_PATH=$PATH2 is not a directory" echo "error: DATASET_PATH=$PATH2 is not a directory"
exit 1 exit 1
fi fi
if [ $# == 5 ] && [ ! -f $PATH3 ] if [ $# == 5 ] && [ ! -f $PATH3 ]
then then
echo "error: PRETRAINED_CKPT_PATH=$PATH3 is not a file" echo "error: PRETRAINED_CKPT_PATH=$PATH3 is not a file"
exit 1 exit 1
fi fi
ulimit -u unlimited ulimit -u unlimited

77
model_zoo/official/cv/resnet/scripts/run_parameter_server_train_gpu.sh
View File

@ -16,26 +16,26 @@
if [ $# != 3 ] && [ $# != 4 ] if [ $# != 3 ] && [ $# != 4 ]
then then
echo "Usage: sh run_distribute_train_gpu.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" echo "Usage: sh run_distribute_train_gpu.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)"
exit 1 exit 1
fi fi
if [ $1 != "resnet50" ] && [ $1 != "resnet101" ] if [ $1 != "resnet50" ] && [ $1 != "resnet101" ]
then then
echo "error: the selected net is neither resnet50 nor resnet101" echo "error: the selected net is neither resnet50 nor resnet101"
exit 1 exit 1
fi fi
if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ] if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ]
then then
echo "error: the selected dataset is neither cifar10 nor imagenet2012" echo "error: the selected dataset is neither cifar10 nor imagenet2012"
exit 1 exit 1
fi fi
if [ $1 == "resnet101" ] && [ $2 == "cifar10" ] if [ $1 == "resnet101" ] && [ $2 == "cifar10" ]
then then
echo "error: training resnet101 with cifar10 dataset is unsupported now!" echo "error: training resnet101 with cifar10 dataset is unsupported now!"
exit 1 exit 1
fi fi
@ -58,13 +58,13 @@ fi
if [ ! -d $PATH2 ] if [ ! -d $PATH2 ]
then then
echo "error: DATASET_PATH=$PATH1 is not a directory" echo "error: DATASET_PATH=$PATH1 is not a directory"
exit 1 exit 1
fi fi
if [ $# == 5 ] && [ ! -f $PATH2 ] if [ $# == 5 ] && [ ! -f $PATH2 ]
then then
echo "error: PRETRAINED_CKPT_PATH=$PATH2 is not a file" echo "error: PRETRAINED_CKPT_PATH=$PATH2 is not a file"
exit 1 exit 1
fi fi
export DEVICE_NUM=8 export DEVICE_NUM=8
@ -86,43 +86,42 @@ cp -r ../src ./sched
cd ./sched || exit cd ./sched || exit
if [ $# == 3 ] if [ $# == 3 ]
then then
mpirun --allow-run-as-root -n 1 --output-filename log_output --merge-stderr-to-stdout \ mpirun --allow-run-as-root -n 1 --output-filename log_output --merge-stderr-to-stdout \
python train.py --net=$1 --dataset=$2 --run_distribute=True \ python train.py --net=$1 --dataset=$2 --run_distribute=True \
--device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --parameter_server=True &> sched.log & --device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --parameter_server=True &> sched.log &
fi fi
if [ $# == 4 ] if [ $# == 4 ]
then then
mpirun --allow-run-as-root -n 1 --output-filename log_output --merge-stderr-to-stdout \ mpirun --allow-run-as-root -n 1 --output-filename log_output --merge-stderr-to-stdout \
python train.py --net=$1 --dataset=$2 --run_distribute=True \ python train.py --net=$1 --dataset=$2 --run_distribute=True \
--device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --parameter_server=True --pre_trained=$PATH2 &> sched.log & --device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --parameter_server=True --pre_trained=$PATH2 &> sched.log &
fi fi
cd .. cd ..
export MS_ROLE=MS_PSERVER export MS_ROLE=MS_PSERVER
for((i=0;i<$MS_SERVER_NUM;i++)); for((i=0;i<$MS_SERVER_NUM;i++));
do do
rm -rf ./server_$i rm -rf ./server_$i
mkdir ./server_$i mkdir ./server_$i
cp ../*.py ./server_$i cp ../*.py ./server_$i
cp *.sh ./server_$i cp *.sh ./server_$i
cp -r ../src ./server_$i cp -r ../src ./server_$i
cd ./server_$i || exit cd ./server_$i || exit
if [ $# == 3 ] if [ $# == 3 ]
then then
mpirun --allow-run-as-root -n 1 --output-filename log_output --merge-stderr-to-stdout \ mpirun --allow-run-as-root -n 1 --output-filename log_output --merge-stderr-to-stdout \
python train.py --net=$1 --dataset=$2 --run_distribute=True \ python train.py --net=$1 --dataset=$2 --run_distribute=True \
--device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --parameter_server=True &> server_$i.log & --device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --parameter_server=True &> server_$i.log &
fi fi
if [ $# == 4 ] if [ $# == 4 ]
then then
mpirun --allow-run-as-root -n 1 --output-filename log_output --merge-stderr-to-stdout \ mpirun --allow-run-as-root -n 1 --output-filename log_output --merge-stderr-to-stdout \
python train.py --net=$1 --dataset=$2 --run_distribute=True \ python train.py --net=$1 --dataset=$2 --run_distribute=True \
--device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --parameter_server=True --pre_trained=$PATH2 &> server_$i.log & --device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --parameter_server=True --pre_trained=$PATH2 &> server_$i.log &
fi fi
cd .. cd ..
done done
export MS_ROLE=MS_WORKER export MS_ROLE=MS_WORKER
@ -134,15 +133,15 @@ cp -r ../src ./worker
cd ./worker || exit cd ./worker || exit
if [ $# == 3 ] if [ $# == 3 ]
then then
mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \ mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \
python train.py --net=$1 --dataset=$2 --run_distribute=True \ python train.py --net=$1 --dataset=$2 --run_distribute=True \
--device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --parameter_server=True &> worker.log & --device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --parameter_server=True &> worker.log &
fi fi
if [ $# == 4 ] if [ $# == 4 ]
then then
mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \ mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \
python train.py --net=$1 --dataset=$2 --run_distribute=True \ python train.py --net=$1 --dataset=$2 --run_distribute=True \
--device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --parameter_server=True --pre_trained=$PATH2 &> worker.log & --device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --parameter_server=True --pre_trained=$PATH2 &> worker.log &
fi fi
cd .. cd ..

4
model_zoo/official/cv/resnet_thor/scripts/run_distribute_train_gpu.sh
View File

@ -16,8 +16,8 @@
if [ $# != 2 ] if [ $# != 2 ]
then then
echo "Usage: sh run_distribute_train_gpu.sh [DATASET_PATH] [DEVICE_NUM]" echo "Usage: sh run_distribute_train_gpu.sh [DATASET_PATH] [DEVICE_NUM]"
exit 1 exit 1
fi fi
get_real_path(){ get_real_path(){

2
model_zoo/official/cv/resnext50/scripts/run_distribute_train.sh
View File

@ -22,7 +22,7 @@ echo "hccl connect time out has changed to 600 second"
PATH_CHECKPOINT="" PATH_CHECKPOINT=""
if [ $# == 3 ] if [ $# == 3 ]
then then
PATH_CHECKPOINT=$3 PATH_CHECKPOINT=$3
fi fi
cores=`cat /proc/cpuinfo|grep "processor" |wc -l` cores=`cat /proc/cpuinfo|grep "processor" |wc -l`

4
model_zoo/official/cv/resnext50/scripts/run_distribute_train_for_gpu.sh
View File

@ -19,11 +19,11 @@ export RANK_SIZE=8
PATH_CHECKPOINT="" PATH_CHECKPOINT=""
if [ $# == 2 ] if [ $# == 2 ]
then then
PATH_CHECKPOINT=$2 PATH_CHECKPOINT=$2
fi fi
mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \ mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \
python train.py \ python train.py \
--is_distribute=1 \ --is_distribute=1 \
--platform="GPU" \ --platform="GPU" \
--pretrained=$PATH_CHECKPOINT \ --pretrained=$PATH_CHECKPOINT \

2
model_zoo/official/cv/retinaface_resnet50/scripts/run_distribute_gpu_train.sh
View File

@ -24,4 +24,4 @@ RANK_SIZE=$1
export CUDA_VISIBLE_DEVICES="$2" export CUDA_VISIBLE_DEVICES="$2"
mpirun --allow-run-as-root -n $RANK_SIZE \ mpirun --allow-run-as-root -n $RANK_SIZE \
python train.py > train.log 2>&1 & python train.py > train.log 2>&1 &

14
model_zoo/official/cv/squeezenet/scripts/run_distribute_train.sh
View File

@ -16,20 +16,20 @@
if [ $# != 4 ] && [ $# != 5 ] if [ $# != 4 ] && [ $# != 5 ]
then then
echo "Usage: sh scripts/run_distribute_train.sh [squeezenet|squeezenet_residual] [cifar10|imagenet] [RANK_TABLE_FILE] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" echo "Usage: sh scripts/run_distribute_train.sh [squeezenet|squeezenet_residual] [cifar10|imagenet] [RANK_TABLE_FILE] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)"
exit 1 exit 1
fi fi
if [ $1 != "squeezenet" ] && [ $1 != "squeezenet_residual" ] if [ $1 != "squeezenet" ] && [ $1 != "squeezenet_residual" ]
then then
echo "error: the selected net is neither squeezenet nor squeezenet_residual" echo "error: the selected net is neither squeezenet nor squeezenet_residual"
exit 1 exit 1
fi fi
if [ $2 != "cifar10" ] && [ $2 != "imagenet" ] if [ $2 != "cifar10" ] && [ $2 != "imagenet" ]
then then
echo "error: the selected dataset is neither cifar10 nor imagenet" echo "error: the selected dataset is neither cifar10 nor imagenet"
exit 1 exit 1
fi fi
get_real_path(){ get_real_path(){
@ -51,19 +51,19 @@ fi
if [ ! -f $PATH1 ] if [ ! -f $PATH1 ]
then then
echo "error: RANK_TABLE_FILE=$PATH1 is not a file" echo "error: RANK_TABLE_FILE=$PATH1 is not a file"
exit 1 exit 1
fi fi
if [ ! -d $PATH2 ] if [ ! -d $PATH2 ]
then then
echo "error: DATASET_PATH=$PATH2 is not a directory" echo "error: DATASET_PATH=$PATH2 is not a directory"
exit 1 exit 1
fi fi
if [ $# == 5 ] && [ ! -f $PATH3 ] if [ $# == 5 ] && [ ! -f $PATH3 ]
then then
echo "error: PRETRAINED_CKPT_PATH=$PATH3 is not a file" echo "error: PRETRAINED_CKPT_PATH=$PATH3 is not a file"
exit 1 exit 1
fi fi
ulimit -u unlimited ulimit -u unlimited

2
model_zoo/official/cv/unet/README.md
View File

@ -27,7 +27,7 @@
Unet Medical model for 2D image segmentation. This implementation is as described in the original paper [UNet: Convolutional Networks for Biomedical Image Segmentation](https://arxiv.org/abs/1505.04597). Unet, in the 2015 ISBI cell tracking competition, many of the best are obtained. In this paper, a network model for medical image segmentation is proposed, and a data enhancement method is proposed to effectively use the annotation data to solve the problem of insufficient annotation data in the medical field. A U-shaped network structure is also used to extract the context and location information. Unet Medical model for 2D image segmentation. This implementation is as described in the original paper [UNet: Convolutional Networks for Biomedical Image Segmentation](https://arxiv.org/abs/1505.04597). Unet, in the 2015 ISBI cell tracking competition, many of the best are obtained. In this paper, a network model for medical image segmentation is proposed, and a data enhancement method is proposed to effectively use the annotation data to solve the problem of insufficient annotation data in the medical field. A U-shaped network structure is also used to extract the context and location information.
[Paper](https://arxiv.org/abs/1505.04597): Olaf Ronneberger, Philipp Fischer, Thomas Brox. "U-Net: Convolutional Networks for Biomedical Image Segmentation." * conditionally accepted at MICCAI 2015*. 2015. [Paper](https://arxiv.org/abs/1505.04597): Olaf Ronneberger, Philipp Fischer, Thomas Brox. "U-Net: Convolutional Networks for Biomedical Image Segmentation." * conditionally accepted at MICCAI 2015*. 2015.
# [Model Architecture](#contents) # [Model Architecture](#contents)

4
model_zoo/official/cv/vgg16/README.md
View File

@ -47,8 +47,8 @@ Note that you can run the scripts based on the dataset mentioned in original pap
#### Dataset used: [ImageNet2012](http://www.image-net.org/) #### Dataset used: [ImageNet2012](http://www.image-net.org/)
- Dataset size: ~146G, 1.28 million colorful images in 1000 classes - Dataset size: ~146G, 1.28 million colorful images in 1000 classes
- Train: 140G, 1,281,167 images - Train: 140G, 1,281,167 images
- Test: 6.4G, 50, 000 images - Test: 6.4G, 50, 000 images
- Data format: RGB images - Data format: RGB images
- Note: Data will be processed in src/dataset.py - Note: Data will be processed in src/dataset.py

20
model_zoo/official/nlp/tinybert/scripts/run_distributed_gd_gpu.sh
View File

@ -30,13 +30,13 @@ TEACHER_CKPT_PATH=$5
PROJECT_DIR=$(cd "$(dirname "$0")" || exit; pwd) PROJECT_DIR=$(cd "$(dirname "$0")" || exit; pwd)
mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \ mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \
python ${PROJECT_DIR}/../run_general_distill.py \ python ${PROJECT_DIR}/../run_general_distill.py \
--distribute="true" \ --distribute="true" \
--device_target="GPU" \ --device_target="GPU" \
--epoch_size=$EPOCH_SIZE \ --epoch_size=$EPOCH_SIZE \
--save_ckpt_path="" \ --save_ckpt_path="" \
--data_dir=$DATA_DIR \ --data_dir=$DATA_DIR \
--schema_dir=$SCHEMA_DIR \ --schema_dir=$SCHEMA_DIR \
--dataset_type="tfrecord" \ --dataset_type="tfrecord" \
--enable_data_sink="false" \ --enable_data_sink="false" \
--load_teacher_ckpt_path=$TEACHER_CKPT_PATH > log.txt 2>&1 & --load_teacher_ckpt_path=$TEACHER_CKPT_PATH > log.txt 2>&1 &

2
model_zoo/research/cv/ghostnet/Readme.md
View File

@ -36,7 +36,7 @@ Dataset used: [Oxford-IIIT Pet](https://www.robots.ox.ac.uk/~vgg/data/pets/)
- Train: 3680 images - Train: 3680 images
- Test: 3369 images - Test: 3369 images
- Data format: RGB images. - Data format: RGB images.
- Note: Data will be processed in src/dataset.py - Note: Data will be processed in src/dataset.py
# [Environment Requirements](#contents) # [Environment Requirements](#contents)

2
model_zoo/research/cv/ghostnet_quant/Readme.md
View File

@ -41,7 +41,7 @@ Dataset used: [Oxford-IIIT Pet](https://www.robots.ox.ac.uk/~vgg/data/pets/)
- Train: 3680 images - Train: 3680 images
- Test: 3369 images - Test: 3369 images
- Data format: RGB images. - Data format: RGB images.
- Note: Data will be processed in src/dataset.py - Note: Data will be processed in src/dataset.py
# [Environment Requirements](#contents) # [Environment Requirements](#contents)

14
model_zoo/research/cv/squeezenet/scripts/run_distribute_train.sh
View File

@ -16,20 +16,20 @@
if [ $# != 4 ] && [ $# != 5 ] if [ $# != 4 ] && [ $# != 5 ]
then then
echo "Usage: sh scripts/run_distribute_train.sh [squeezenet|squeezenet_residual] [cifar10|imagenet] [RANK_TABLE_FILE] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" echo "Usage: sh scripts/run_distribute_train.sh [squeezenet|squeezenet_residual] [cifar10|imagenet] [RANK_TABLE_FILE] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)"
exit 1 exit 1
fi fi
if [ $1 != "squeezenet" ] && [ $1 != "squeezenet_residual" ] if [ $1 != "squeezenet" ] && [ $1 != "squeezenet_residual" ]
then then
echo "error: the selected net is neither squeezenet nor squeezenet_residual" echo "error: the selected net is neither squeezenet nor squeezenet_residual"
exit 1 exit 1
fi fi
if [ $2 != "cifar10" ] && [ $2 != "imagenet" ] if [ $2 != "cifar10" ] && [ $2 != "imagenet" ]
then then
echo "error: the selected dataset is neither cifar10 nor imagenet" echo "error: the selected dataset is neither cifar10 nor imagenet"
exit 1 exit 1
fi fi
get_real_path(){ get_real_path(){
@ -51,19 +51,19 @@ fi
if [ ! -f $PATH1 ] if [ ! -f $PATH1 ]
then then
echo "error: RANK_TABLE_FILE=$PATH1 is not a file" echo "error: RANK_TABLE_FILE=$PATH1 is not a file"
exit 1 exit 1
fi fi
if [ ! -d $PATH2 ] if [ ! -d $PATH2 ]
then then
echo "error: DATASET_PATH=$PATH2 is not a directory" echo "error: DATASET_PATH=$PATH2 is not a directory"
exit 1 exit 1
fi fi
if [ $# == 5 ] && [ ! -f $PATH3 ] if [ $# == 5 ] && [ ! -f $PATH3 ]
then then
echo "error: PRETRAINED_CKPT_PATH=$PATH3 is not a file" echo "error: PRETRAINED_CKPT_PATH=$PATH3 is not a file"
exit 1 exit 1
fi fi
ulimit -u unlimited ulimit -u unlimited

24
model_zoo/research/cv/squeezenet/scripts/run_distribute_train_gpu.sh
View File

@ -16,20 +16,20 @@
if [ $# != 3 ] && [ $# != 4 ] if [ $# != 3 ] && [ $# != 4 ]
then then
echo "Usage: sh scripts/run_distribute_train_gpu.sh [squeezenet|squeezenet_residual] [cifar10|imagenet] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" echo "Usage: sh scripts/run_distribute_train_gpu.sh [squeezenet|squeezenet_residual] [cifar10|imagenet] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)"
exit 1 exit 1
fi fi
if [ $1 != "squeezenet" ] && [ $1 != "squeezenet_residual" ] if [ $1 != "squeezenet" ] && [ $1 != "squeezenet_residual" ]
then then
echo "error: the selected net is neither squeezenet nor squeezenet_residual" echo "error: the selected net is neither squeezenet nor squeezenet_residual"
exit 1 exit 1
fi fi
if [ $2 != "cifar10" ] && [ $2 != "imagenet" ] if [ $2 != "cifar10" ] && [ $2 != "imagenet" ]
then then
echo "error: the selected dataset is neither cifar10 nor imagenet" echo "error: the selected dataset is neither cifar10 nor imagenet"
exit 1 exit 1
fi fi
get_real_path(){ get_real_path(){
@ -51,13 +51,13 @@ fi
if [ ! -d $PATH1 ] if [ ! -d $PATH1 ]
then then
echo "error: DATASET_PATH=$PATH1 is not a directory" echo "error: DATASET_PATH=$PATH1 is not a directory"
exit 1 exit 1
fi fi
if [ $# == 5 ] && [ ! -f $PATH2 ] if [ $# == 5 ] && [ ! -f $PATH2 ]
then then
echo "error: PRETRAINED_CKPT_PATH=$PATH2 is not a file" echo "error: PRETRAINED_CKPT_PATH=$PATH2 is not a file"
exit 1 exit 1
fi fi
ulimit -u unlimited ulimit -u unlimited
@ -72,14 +72,14 @@ cd ./train_parallel || exit
if [ $# == 3 ] if [ $# == 3 ]
then then
mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \ mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \
python train.py --net=$1 --dataset=$2 --run_distribute=True \ python train.py --net=$1 --dataset=$2 --run_distribute=True \
--device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 &> log & --device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 &> log &
fi fi
if [ $# == 4 ] if [ $# == 4 ]
then then
mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \ mpirun --allow-run-as-root -n $RANK_SIZE --output-filename log_output --merge-stderr-to-stdout \
python train.py --net=$1 --dataset=$2 --run_distribute=True \ python train.py --net=$1 --dataset=$2 --run_distribute=True \
--device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --pre_trained=$PATH2 &> log & --device_num=$DEVICE_NUM --device_target="GPU" --dataset_path=$PATH1 --pre_trained=$PATH2 &> log &
fi fi

2
model_zoo/research/cv/tinynet/script/train_distributed_gpu.sh
View File

@ -78,5 +78,5 @@ time mpirun -n $RANK_SIZE --allow-run-as-root python3 ${current_exec_path}/train
--distributed \ --distributed \
--data_path /path_to_ImageNet/ \ --data_path /path_to_ImageNet/ \
--GPU \ --GPU \
--dataset_sink > tinynet_c.log 2>&1 & --dataset_sink > tinynet_c.log 2>&1 &

2
scripts/dbg_dump_parser.sh
View File

@ -64,7 +64,7 @@ function checkopts()
usage usage
exit 1 exit 1
fi fi
;; ;;
h) h)
usage usage
exit 0 exit 0

4
scripts/run_perf_test.sh
View File

@ -22,10 +22,10 @@ PYTHONTEST_DIR="${PROJECT_PATH}/tests/perf_test"
PERF_RESULT_DIR="${CURRPATH}/" PERF_RESULT_DIR="${CURRPATH}/"
PERF_SUFFIX=".perf" PERF_SUFFIX=".perf"
if [[ "${BUILD_PATH}" ]];then if [[ "${BUILD_PATH}" ]];then
echo "BUILD_PATH = ${BUILD_PATH}" echo "BUILD_PATH = ${BUILD_PATH}"
else else
BUILD_PATH="${PROJECT_PATH}/build" BUILD_PATH="${PROJECT_PATH}/build"
echo "BUILD_PATH = ${BUILD_PATH}" echo "BUILD_PATH = ${BUILD_PATH}"
fi fi
cd "${PROJECT_PATH}"; sh build.sh -t off -l none -r -p on -j 20; cd - cd "${PROJECT_PATH}"; sh build.sh -t off -l none -r -p on -j 20; cd -

6
tests/ut/cpp/runtest.sh
View File

@ -18,10 +18,10 @@ set -e
BASEPATH=$(cd "$(dirname "$0")"; pwd) BASEPATH=$(cd "$(dirname "$0")"; pwd)
PROJECT_PATH=${BASEPATH}/../../.. PROJECT_PATH=${BASEPATH}/../../..
if [ $BUILD_PATH ];then if [ $BUILD_PATH ];then
echo "BUILD_PATH = $BUILD_PATH" echo "BUILD_PATH = $BUILD_PATH"
else else
BUILD_PATH=${PROJECT_PATH}/build BUILD_PATH=${PROJECT_PATH}/build
echo "BUILD_PATH = $BUILD_PATH" echo "BUILD_PATH = $BUILD_PATH"
fi fi
cd ${BUILD_PATH}/mindspore/tests/ut/cpp cd ${BUILD_PATH}/mindspore/tests/ut/cpp

4
tests/ut/python/runtest.sh
View File

@ -18,10 +18,10 @@ IGNORE_EXEC="--ignore=$CURRPATH/exec"
PROJECT_PATH=$(cd ${CURRPATH}/../../..; pwd) PROJECT_PATH=$(cd ${CURRPATH}/../../..; pwd)
if [ $BUILD_PATH ];then if [ $BUILD_PATH ];then
echo "BUILD_PATH = $BUILD_PATH" echo "BUILD_PATH = $BUILD_PATH"
else else
BUILD_PATH=${PROJECT_PATH}/build BUILD_PATH=${PROJECT_PATH}/build
echo "BUILD_PATH = $BUILD_PATH" echo "BUILD_PATH = $BUILD_PATH"
fi fi
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${BUILD_PATH}/third_party/gtest/lib export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${BUILD_PATH}/third_party/gtest/lib

4
tests/ut/python/runtest_opensrc.sh
View File

@ -17,10 +17,10 @@
CURRPATH=$(cd $(dirname $0); pwd) CURRPATH=$(cd $(dirname $0); pwd)
PROJECT_PATH=${CURRPATH}/../.. PROJECT_PATH=${CURRPATH}/../..
if [ $BUILD_PATH ];then if [ $BUILD_PATH ];then
echo "BUILD_PATH = $BUILD_PATH" echo "BUILD_PATH = $BUILD_PATH"
else else
BUILD_PATH=${PROJECT_PATH}/build BUILD_PATH=${PROJECT_PATH}/build
echo "BUILD_PATH = $BUILD_PATH" echo "BUILD_PATH = $BUILD_PATH"
fi fi
cd ${BUILD_PATH}/mindspore/tests/ut cd ${BUILD_PATH}/mindspore/tests/ut