p60975123
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

ascend ci smoke support to multi user and multi device id

This commit is contained in:
zhengyuanhua 2022-01-12 15:16:48 +08:00
parent cd81f15dc5
commit 5c46b87f64
6 changed files with 131 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mindspore/lite/test/st/run_benchmark_nets.sh
View File

@ -153,11 +153,11 @@ if [[ $backend == "all" || $backend == "x86_gpu" ]]; then
fi fi
fi fi
if [[ $backend == "all" || $backend == "Ascend310" || $backend == "Ascend710" ]]; then if [[ $backend == "all" || $backend == "x86_ascend310" || $backend == "x86_ascend710" ]]; then
sh $cur_path/scripts/ascend/run_ascend.sh -r $release_path -m $models_path -d $device_id -e $backend sh $cur_path/scripts/ascend/run_ascend.sh -r $release_path -m $models_path -d $device_id -e $backend
ascend_status=$? ascend_status=$?
if [[ ascend_status -ne 0 ]]; then if [[ ascend_status -ne 0 ]]; then
echo "Run ascend failed" echo "Run ${backend} failed"
exit 1 exit 1
fi fi
fi fi

91
mindspore/lite/test/st/scripts/ascend/run_ascend.sh
View File

@ -1,13 +1,38 @@
#!/bin/bash #!/bin/bash
source ./scripts/base_functions.sh source ./scripts/base_functions.sh
function Run_Ascend() { function PrePareLocal() {
# copy converter files to scripts echo "Start to copy local file"
scp ./scripts/ascend/run_remote_ascend.sh root@${device_ip}:${device_scripts_path} || exit 1 rm -rf ${benchmark_test_path}
# copy benchmark files to sc mkdir -p ${benchmark_test_path}
scp ./scripts/ascend/run_benchmark_ascend.sh root@${device_ip}:${device_scripts_path} || exit 1
ssh root@${device_ip} "cd ${device_scripts_path}; sh run_remote_ascend.sh ${backend}" cp ./scripts/base_functions.sh ${benchmark_test_path} || exit 1
cp ./scripts/ascend/run_converter_ascend.sh ${benchmark_test_path} || exit 1
cp ./scripts/ascend/run_benchmark_ascend.sh ${benchmark_test_path} || exit 1
cp ./../config/models_ascend.cfg ${benchmark_test_path} || exit 1
cp ${release_path}/centos_x86/ascend/*-linux-x64.tar.gz ${benchmark_test_path} || exit 1
echo "Copy file success"
}
function PrePareRemote() {
echo "Start to copy remote file"
ssh ${user_name}@${device_ip} "rm -rf ${benchmark_test_path}; mkdir -p ${benchmark_test_path}" || exit 1
scp ./scripts/ascend/run_converter_ascend.sh ${user_name}@${device_ip}:${benchmark_test_path} || exit 1
scp ./scripts/ascend/run_benchmark_ascend.sh ${user_name}@${device_ip}:${benchmark_test_path} || exit 1
scp ./scripts/base_functions.sh ${user_name}@${device_ip}:${benchmark_test_path} || exit 1
scp ./../config/models_ascend.cfg ${user_name}@${device_ip}:${benchmark_test_path} || exit 1
scp ${release_path}/centos_x86/ascend/*-linux-x64.tar.gz ${user_name}@${device_ip}:${benchmark_test_path} || exit 1
echo "Copy file success"
}
function Run_Ascend() {
if [ ${is_local} = 0 ]; then
cd ${benchmark_test_path} || exit 1
sh run_converter_ascend.sh ${backend} ${device_id}
else
ssh ${user_name}@${device_ip} "cd ${benchmark_test_path}; sh run_converter_ascend.sh ${backend} ${device_id}"
fi
if [[ $? = 0 ]]; then if [[ $? = 0 ]]; then
run_result="run in ${backend} pass"; echo ${run_result} >> ${run_ascend_result_file}; run_result="run in ${backend} pass"; echo ${run_result} >> ${run_ascend_result_file};
else else
@ -15,7 +40,7 @@ function Run_Ascend() {
fi fi
} }
# Example:sh run_benchmark_nets.sh -r /home/temp_test -m /home/temp_test/models -e ascend -d 8.92.9.131 # Example:sh run_benchmark_nets.sh -r /home/temp_test -m /home/temp_test/models -e Ascend310 -d 10.92.9.100:2
while getopts "r:m:d:e:" opt; do while getopts "r:m:d:e:" opt; do
case ${opt} in case ${opt} in
r) r)
@ -27,7 +52,8 @@ while getopts "r:m:d:e:" opt; do
;; ;;
d) d)
device_ip=`echo ${OPTARG} | cut -d \: -f 1` device_ip=`echo ${OPTARG} | cut -d \: -f 1`
echo "device_ip is ${device_ip}." device_id=`echo ${OPTARG} | cut -d \: -f 2`
echo "device_ip is ${device_ip}, ascend_device_id is ${device_id}."
;; ;;
e) e)
backend=${OPTARG} backend=${OPTARG}
@ -39,22 +65,29 @@ while getopts "r:m:d:e:" opt; do
esac esac
done done
basepath=$(pwd)/result user_name=${USER}
echo "Current user name is ${user_name}"
basepath=$(pwd)/"${backend}_log_${device_id}"
rm -rf ${basepath} rm -rf ${basepath}
mkdir -p ${basepath} mkdir -p ${basepath}
echo "Ascend base path is ${basepath}" echo "Ascend base path is ${basepath}, device_ip: ${device_ip}, device_id: ${device_id}"
benchmark_test_path=/home/${user_name}/benchmark_test/${device_id}
device_release_path=/home/ascend/release ls /dev/davinci0
device_config_path=/home/ascend/config is_local=$?
device_scripts_path=/home/ascend/scripts if [ ${is_local} = 0 ]; then
PrePareLocal
ssh root@${device_ip} "sh /home/ascend/clear.sh" || exit 1 if [ $? != 0 ]; then
# copy release to device echo "Prepare local failed"
scp ${release_path}/centos_x86/ascend/*-linux-x64.tar.gz root@${device_ip}:${device_release_path} || exit 1 exit 1
# copy config to device fi
scp ./../config/models_ascend.cfg root@${device_ip}:${device_config_path} || exit 1 else
# copy comm func to device PrePareRemote
scp ./scripts/base_functions.sh root@${device_ip}:${device_scripts_path} || exit 1 if [ $? != 0 ]; then
echo "Prepare remote failed"
exit 1
fi
fi
# Write converter result to temp file # Write converter result to temp file
run_ascend_result_file=${basepath}'/run_'${backend}'_result.txt' run_ascend_result_file=${basepath}'/run_'${backend}'_result.txt'
@ -65,10 +98,20 @@ Run_Ascend
Run_ascend_status=$? Run_ascend_status=$?
run_converter_log_file=${basepath}'/run_'${backend}'_converter_log.txt' run_converter_log_file=${basepath}'/run_'${backend}'_converter_log.txt'
run_converter_result_file=${basepath}'/run_'${backend}'_converter_result.txt'
run_benchmark_log_file=${basepath}'/run_'${backend}'_benchmark_log.txt' run_benchmark_log_file=${basepath}'/run_'${backend}'_benchmark_log.txt'
scp root@${device_ip}:${device_scripts_path}/log/run_converter_log.txt ${run_converter_log_file} || exit 1 run_benchmark_result_file=${basepath}'/run_'${backend}'_benchmark_result.txt'
scp root@${device_ip}:${device_scripts_path}/log/run_benchmark_log.txt ${run_benchmark_log_file} || exit 1 if [ ${is_local} = 0 ]; then
cp ${benchmark_test_path}/run_converter_log.txt ${run_converter_log_file} || exit 1
cp ${benchmark_test_path}/run_converter_result.txt ${run_converter_result_file} || exit 1
cp ${benchmark_test_path}/run_benchmark_log.txt ${run_benchmark_log_file} || exit 1
cp ${benchmark_test_path}/run_benchmark_result.txt ${run_benchmark_result_file} || exit 1
else
scp ${user_name}@${device_ip}:${benchmark_test_path}/run_converter_log.txt ${run_converter_log_file} || exit 1
scp ${user_name}@${device_ip}:${benchmark_test_path}/run_converter_result.txt ${run_converter_result_file} || exit 1
scp ${user_name}@${device_ip}:${benchmark_test_path}/run_benchmark_log.txt ${run_benchmark_log_file} || exit 1
scp ${user_name}@${device_ip}:${benchmark_test_path}/run_benchmark_result.txt ${run_benchmark_result_file} || exit 1
fi
echo "Run in ${backend} ended" echo "Run in ${backend} ended"
Print_Converter_Result ${run_ascend_result_file} Print_Converter_Result ${run_ascend_result_file}
exit ${Run_ascend_status} exit ${Run_ascend_status}

41
mindspore/lite/test/st/scripts/ascend/run_benchmark_ascend.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
# Example:sh run_remote_ascend.sh -v version -b backend # Example:sh run_remote_ascend.sh -v version -b backend
while getopts "v:b:" opt; do while getopts "v:b:d:" opt; do
case ${opt} in case ${opt} in
v) v)
version=${OPTARG} version=${OPTARG}
@ -11,15 +11,21 @@ while getopts "v:b:" opt; do
backend=${OPTARG} backend=${OPTARG}
echo "backend is ${backend}" echo "backend is ${backend}"
;; ;;
d)
device_id=${OPTARG}
echo "device id is ${device_id}"
;;
?) ?)
echo "unknown para" echo "unknown para"
exit 1;; exit 1;;
esac esac
done done
export ASCEND_DEVICE_ID=${device_id}
# Run Benchmark in Ascend platform: # Run Benchmark in Ascend platform:
function Run_Benchmark() { function Run_Benchmark() {
cd ${x86_path}/mindspore-lite-${version}-linux-x64/ || exit 1 cd ${benchmark_test}/mindspore-lite-${version}-linux-x64 || exit 1
cp tools/benchmark/benchmark ./ || exit 1 cp tools/benchmark/benchmark ./ || exit 1
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./runtime/lib export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./runtime/lib
@ -51,7 +57,7 @@ function Run_Benchmark() {
model_file=${ms_models_path}'/'${model_name}'.ms' model_file=${ms_models_path}'/'${model_name}'.ms'
input_files="" input_files=""
output_file="" output_file=""
data_path=${basepath}'/data/' data_path=${model_data_path}'/data/'
if [[ ${input_num} == "" || ${input_num} == 1 ]]; then if [[ ${input_num} == "" || ${input_num} == 1 ]]; then
input_files=${data_path}'input/'${model_name}'.ms.bin' input_files=${data_path}'input/'${model_name}'.ms.bin'
else else
@ -76,8 +82,8 @@ function Run_Benchmark() {
fi fi
# different tensorrt run mode use different cuda command # different tensorrt run mode use different cuda command
echo './benchmark --modelFile='${model_file}' --inputShapes='${input_shapes}' --inDataFile='${input_files}' --benchmarkDataFile='${output_file}' --enableFp16='${enableFp16}' --accuracyThreshold='${acc_limit}' --device='${backend} >> "${run_ascend_log_file}" echo './benchmark --modelFile='${model_file}' --inputShapes='${input_shapes}' --inDataFile='${input_files}' --benchmarkDataFile='${output_file}' --enableFp16='${enableFp16}' --accuracyThreshold='${acc_limit}' --device='${ascend_device} >> "${run_ascend_log_file}"
./benchmark --modelFile=${model_file} --inputShapes=${input_shapes} --inDataFile=${input_files} --benchmarkDataFile=${output_file} --enableFp16=${enableFp16} --accuracyThreshold=${acc_limit} --device=${backend} >> ${run_ascend_log_file} ./benchmark --modelFile=${model_file} --inputShapes=${input_shapes} --inDataFile=${input_files} --benchmarkDataFile=${output_file} --enableFp16=${enableFp16} --accuracyThreshold=${acc_limit} --device=${ascend_device} >> ${run_ascend_log_file}
if [ $? = 0 ]; then if [ $? = 0 ]; then
run_result=${backend}': '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file} run_result=${backend}': '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}
@ -88,20 +94,29 @@ function Run_Benchmark() {
done < ${models_ascend_config} done < ${models_ascend_config}
} }
basepath=/home/ascend user_name=${USER}
x86_path=${basepath}/release benchmark_test=/home/${user_name}/benchmark_test/${device_id}
ms_models_path=${basepath}/ms_models ms_models_path=${benchmark_test}/ms_models
models_ascend_config=${basepath}/config/models_ascend.cfg models_ascend_config=${benchmark_test}/models_ascend.cfg
data_path=${basepath}/data model_data_path=/home/workspace/mindspore_dataset/mslite
# Write benchmark result to temp file # Write benchmark result to temp file
run_benchmark_result_file=${basepath}/scripts/log/run_benchmark_result.txt run_benchmark_result_file=${benchmark_test}/run_benchmark_result.txt
echo ' ' > ${run_benchmark_result_file} echo ' ' > ${run_benchmark_result_file}
#################### run simple Ascend models #################### run simple Ascend models
run_ascend_log_file=${basepath}/scripts/log/run_benchmark_log.txt run_ascend_log_file=${benchmark_test}/run_benchmark_log.txt
echo 'run Ascend logs: ' > ${run_ascend_log_file} echo 'run Ascend logs: ' > ${run_ascend_log_file}
echo "Start to run benchmark in ${backend} ..." echo "Start to run benchmark in ${backend}, device id ${device_id}..."
if [[ ${backend} =~ "ascend310" ]]; then
ascend_device=Ascend310
elif [[ ${backend} =~ "ascend710" ]]; then
ascend_device=Ascend710
else
echo "${backend} is not support."
exit 1
fi
Run_Benchmark Run_Benchmark
Run_benchmark_status=$? Run_benchmark_status=$?
if [[ ${Run_benchmark_status} = 0 ]];then if [[ ${Run_benchmark_status} = 0 ]];then

43
mindspore/lite/test/st/scripts/ascend/run_remote_ascend.sh → mindspore/lite/test/st/scripts/ascend/run_converter_ascend.sh
View File

@ -1,21 +1,13 @@
#!/bin/bash #!/bin/bash
source /root/miniconda3/bin/activate base
source /home/ascend/scripts/base_functions.sh
# Run converter for ascend x86 platform: # Run converter for ascend x86 platform:
function Run_Converter() { function Run_Converter() {
# Unzip x86 runtime and converter
cd ${x86_path} || exit 1 cd ${x86_path} || exit 1
tar -zxf ${x86_path}/mindspore-lite-${version}-linux-x64.tar.gz || exit 1
tar -zxf mindspore-lite-${version}-linux-x64.tar.gz || exit 1 tar -zxf mindspore-lite-${version}-linux-x64.tar.gz || exit 1
cd ${x86_path}/mindspore-lite-${version}-linux-x64/ || exit 1 cd ${x86_path}/mindspore-lite-${version}-linux-x64/ || exit 1
cp tools/converter/converter/converter_lite ./ || exit 1 cp tools/converter/converter/converter_lite ./ || exit 1
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./tools/converter/lib/ export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./tools/converter/lib/
rm -rf ${ms_models_path}
mkdir -p ${ms_models_path}
# Prepare the config file list # Prepare the config file list
local ascend_cfg_file_list=("$models_ascend_config") local ascend_cfg_file_list=("$models_ascend_config")
# Convert models: # Convert models:
@ -31,29 +23,37 @@ export ASCEND_OPP_PATH=${ASCEND_HOME}/opp
export TBE_IMPL_PATH=${ASCEND_HOME}/opp/op_impl/built-in/ai_core/tbe export TBE_IMPL_PATH=${ASCEND_HOME}/opp/op_impl/built-in/ai_core/tbe
export PYTHONPATH=${TBE_IMPL_PATH}:${PYTHONPATH} export PYTHONPATH=${TBE_IMPL_PATH}:${PYTHONPATH}
basepath=/home/ascend backend=$1
x86_fail_not_return="OFF" # This value could not be set to ON. device_id=$2
x86_path=${basepath}/release user_name=${USER}
echo "Current user is ${USER}"
benchmark_test=/home/${user_name}/benchmark_test/${device_id}
echo "Benchmark test path is ${benchmark_test}"
x86_path=${benchmark_test}
file_name=$(ls ${x86_path}/*-linux-x64.tar.gz) file_name=$(ls ${x86_path}/*-linux-x64.tar.gz)
IFS="-" read -r -a file_name_array <<< "$file_name" IFS="-" read -r -a file_name_array <<< "$file_name"
version=${file_name_array[2]} version=${file_name_array[2]}
models_path=${basepath}/models ms_models_path=${benchmark_test}/ms_models
ms_models_path=${basepath}/ms_models rm -rf ${ms_models_path}
ms_scripts_path=${basepath}/scripts mkdir -p ${ms_models_path}
models_ascend_config=${basepath}/config/models_ascend.cfg
model_data_path=/home/workspace/mindspore_dataset/mslite
models_path=${model_data_path}/models
models_ascend_config=${benchmark_test}/models_ascend.cfg
# Write converter result to temp file # Write converter result to temp file
run_converter_log_file=${ms_scripts_path}/log/run_converter_log.txt run_converter_log_file=${benchmark_test}/run_converter_log.txt
echo ' ' > ${run_converter_log_file} echo ' ' > ${run_converter_log_file}
run_converter_result_file=${ms_scripts_path}/log/run_converter_result.txt run_converter_result_file=${benchmark_test}/run_converter_result.txt
echo ' ' > ${run_converter_result_file} echo ' ' > ${run_converter_result_file}
backend=$1
# Run converter # Run converter
echo "Start to run converter in ${backend} ..." x86_fail_not_return="OFF" # This value could not be set to ON.
source ${benchmark_test}/base_functions.sh
echo "Start to run converter in ${backend}, device id ${device_id} ..."
Run_Converter Run_Converter
if [[ $? = 0 ]]; then if [[ $? = 0 ]]; then
echo "Run converter success" echo "Run converter success"
@ -66,7 +66,6 @@ else
fi fi
# Run Benchmark # Run Benchmark
source ${ms_scripts_path}/run_benchmark_ascend.sh -v $version -b $backend source ${benchmark_test}/run_benchmark_ascend.sh -v $version -b $backend -d $device_id
Run_Benchmark_status=$? Run_Benchmark_status=$?
exit ${Run_Benchmark_status} exit ${Run_Benchmark_status}

4
mindspore/lite/test/st/scripts/base_functions.sh
View File

@ -63,8 +63,8 @@ function Convert() {
elif [[ ${cfg_file_name} =~ "_train" ]]; then elif [[ ${cfg_file_name} =~ "_train" ]]; then
train_model="true" train_model="true"
elif [[ ${cfg_file_name} =~ "_ascend" ]]; then elif [[ ${cfg_file_name} =~ "_ascend" ]]; then
model_option_path="${cfg_file%/*}/model_options" model_path=$2
option_file="${model_option_path}/${model_name}.txt" option_file="${model_path}/model_option/${model_name}.txt"
if [ -f "$option_file" ]; then if [ -f "$option_file" ]; then
config_file=${option_file} config_file=${option_file}
fi fi

12
mindspore/lite/tools/benchmark/benchmark_unified_api.cc
View File

@ -391,8 +391,18 @@ int BenchmarkUnifiedApi::InitMSContext(const std::shared_ptr<mindspore::Context>
} }
if (flags_->device_ == "Ascend310" || flags_->device_ == "Ascend710") { if (flags_->device_ == "Ascend310" || flags_->device_ == "Ascend710") {
uint32_t device_id = 0;
auto device_id_env = std::getenv("ASCEND_DEVICE_ID");
if (device_id_env != nullptr) {
try {
device_id = static_cast<uint32_t>(std::stoul(device_id_env));
} catch (std::invalid_argument &e) {
MS_LOG(WARNING) << "Invalid device id env:" << device_id_env << ". Set default device id 0.";
}
MS_LOG(INFO) << "Ascend device_id = " << device_id;
}
std::shared_ptr<AscendDeviceInfo> ascend_device_info = std::make_shared<AscendDeviceInfo>(); std::shared_ptr<AscendDeviceInfo> ascend_device_info = std::make_shared<AscendDeviceInfo>();
ascend_device_info->SetDeviceID(0); ascend_device_info->SetDeviceID(device_id);
device_list.push_back(ascend_device_info); device_list.push_back(ascend_device_info);
} }