From 5c46b87f640b600b299e24eb8d5f3a43ab30db4b Mon Sep 17 00:00:00 2001 From: zhengyuanhua Date: Wed, 12 Jan 2022 15:16:48 +0800 Subject: [PATCH] ascend ci smoke support to multi user and multi device id --- mindspore/lite/test/st/run_benchmark_nets.sh | 4 +- .../lite/test/st/scripts/ascend/run_ascend.sh | 91 ++++++++++++++----- .../st/scripts/ascend/run_benchmark_ascend.sh | 41 ++++++--- ...mote_ascend.sh => run_converter_ascend.sh} | 43 +++++---- .../lite/test/st/scripts/base_functions.sh | 4 +- .../tools/benchmark/benchmark_unified_api.cc | 12 ++- 6 files changed, 131 insertions(+), 64 deletions(-) rename mindspore/lite/test/st/scripts/ascend/{run_remote_ascend.sh => run_converter_ascend.sh} (69%) diff --git a/mindspore/lite/test/st/run_benchmark_nets.sh b/mindspore/lite/test/st/run_benchmark_nets.sh index 2f69534da4f..eec0794184f 100644 --- a/mindspore/lite/test/st/run_benchmark_nets.sh +++ b/mindspore/lite/test/st/run_benchmark_nets.sh @@ -153,11 +153,11 @@ if [[ $backend == "all" || $backend == "x86_gpu" ]]; then fi fi -if [[ $backend == "all" || $backend == "Ascend310" || $backend == "Ascend710" ]]; then +if [[ $backend == "all" || $backend == "x86_ascend310" || $backend == "x86_ascend710" ]]; then sh $cur_path/scripts/ascend/run_ascend.sh -r $release_path -m $models_path -d $device_id -e $backend ascend_status=$? if [[ ascend_status -ne 0 ]]; then - echo "Run ascend failed" + echo "Run ${backend} failed" exit 1 fi fi diff --git a/mindspore/lite/test/st/scripts/ascend/run_ascend.sh b/mindspore/lite/test/st/scripts/ascend/run_ascend.sh index 31a0a60577c..8d9e117be52 100644 --- a/mindspore/lite/test/st/scripts/ascend/run_ascend.sh +++ b/mindspore/lite/test/st/scripts/ascend/run_ascend.sh @@ -1,13 +1,38 @@ #!/bin/bash source ./scripts/base_functions.sh -function Run_Ascend() { - # copy converter files to scripts - scp ./scripts/ascend/run_remote_ascend.sh root@${device_ip}:${device_scripts_path} || exit 1 - # copy benchmark files to sc - scp ./scripts/ascend/run_benchmark_ascend.sh root@${device_ip}:${device_scripts_path} || exit 1 +function PrePareLocal() { + echo "Start to copy local file" + rm -rf ${benchmark_test_path} + mkdir -p ${benchmark_test_path} - ssh root@${device_ip} "cd ${device_scripts_path}; sh run_remote_ascend.sh ${backend}" + cp ./scripts/base_functions.sh ${benchmark_test_path} || exit 1 + cp ./scripts/ascend/run_converter_ascend.sh ${benchmark_test_path} || exit 1 + cp ./scripts/ascend/run_benchmark_ascend.sh ${benchmark_test_path} || exit 1 + cp ./../config/models_ascend.cfg ${benchmark_test_path} || exit 1 + cp ${release_path}/centos_x86/ascend/*-linux-x64.tar.gz ${benchmark_test_path} || exit 1 + echo "Copy file success" +} + +function PrePareRemote() { + echo "Start to copy remote file" + ssh ${user_name}@${device_ip} "rm -rf ${benchmark_test_path}; mkdir -p ${benchmark_test_path}" || exit 1 + + scp ./scripts/ascend/run_converter_ascend.sh ${user_name}@${device_ip}:${benchmark_test_path} || exit 1 + scp ./scripts/ascend/run_benchmark_ascend.sh ${user_name}@${device_ip}:${benchmark_test_path} || exit 1 + scp ./scripts/base_functions.sh ${user_name}@${device_ip}:${benchmark_test_path} || exit 1 + scp ./../config/models_ascend.cfg ${user_name}@${device_ip}:${benchmark_test_path} || exit 1 + scp ${release_path}/centos_x86/ascend/*-linux-x64.tar.gz ${user_name}@${device_ip}:${benchmark_test_path} || exit 1 + echo "Copy file success" +} + +function Run_Ascend() { + if [ ${is_local} = 0 ]; then + cd ${benchmark_test_path} || exit 1 + sh run_converter_ascend.sh ${backend} ${device_id} + else + ssh ${user_name}@${device_ip} "cd ${benchmark_test_path}; sh run_converter_ascend.sh ${backend} ${device_id}" + fi if [[ $? = 0 ]]; then run_result="run in ${backend} pass"; echo ${run_result} >> ${run_ascend_result_file}; else @@ -15,7 +40,7 @@ function Run_Ascend() { fi } -# Example:sh run_benchmark_nets.sh -r /home/temp_test -m /home/temp_test/models -e ascend -d 8.92.9.131 +# Example:sh run_benchmark_nets.sh -r /home/temp_test -m /home/temp_test/models -e Ascend310 -d 10.92.9.100:2 while getopts "r:m:d:e:" opt; do case ${opt} in r) @@ -27,7 +52,8 @@ while getopts "r:m:d:e:" opt; do ;; d) device_ip=`echo ${OPTARG} | cut -d \: -f 1` - echo "device_ip is ${device_ip}." + device_id=`echo ${OPTARG} | cut -d \: -f 2` + echo "device_ip is ${device_ip}, ascend_device_id is ${device_id}." ;; e) backend=${OPTARG} @@ -39,22 +65,29 @@ while getopts "r:m:d:e:" opt; do esac done -basepath=$(pwd)/result +user_name=${USER} +echo "Current user name is ${user_name}" +basepath=$(pwd)/"${backend}_log_${device_id}" rm -rf ${basepath} mkdir -p ${basepath} -echo "Ascend base path is ${basepath}" +echo "Ascend base path is ${basepath}, device_ip: ${device_ip}, device_id: ${device_id}" +benchmark_test_path=/home/${user_name}/benchmark_test/${device_id} -device_release_path=/home/ascend/release -device_config_path=/home/ascend/config -device_scripts_path=/home/ascend/scripts - -ssh root@${device_ip} "sh /home/ascend/clear.sh" || exit 1 -# copy release to device -scp ${release_path}/centos_x86/ascend/*-linux-x64.tar.gz root@${device_ip}:${device_release_path} || exit 1 -# copy config to device -scp ./../config/models_ascend.cfg root@${device_ip}:${device_config_path} || exit 1 -# copy comm func to device -scp ./scripts/base_functions.sh root@${device_ip}:${device_scripts_path} || exit 1 +ls /dev/davinci0 +is_local=$? +if [ ${is_local} = 0 ]; then + PrePareLocal + if [ $? != 0 ]; then + echo "Prepare local failed" + exit 1 + fi +else + PrePareRemote + if [ $? != 0 ]; then + echo "Prepare remote failed" + exit 1 + fi +fi # Write converter result to temp file run_ascend_result_file=${basepath}'/run_'${backend}'_result.txt' @@ -65,10 +98,20 @@ Run_Ascend Run_ascend_status=$? run_converter_log_file=${basepath}'/run_'${backend}'_converter_log.txt' +run_converter_result_file=${basepath}'/run_'${backend}'_converter_result.txt' run_benchmark_log_file=${basepath}'/run_'${backend}'_benchmark_log.txt' -scp root@${device_ip}:${device_scripts_path}/log/run_converter_log.txt ${run_converter_log_file} || exit 1 -scp root@${device_ip}:${device_scripts_path}/log/run_benchmark_log.txt ${run_benchmark_log_file} || exit 1 - +run_benchmark_result_file=${basepath}'/run_'${backend}'_benchmark_result.txt' +if [ ${is_local} = 0 ]; then + cp ${benchmark_test_path}/run_converter_log.txt ${run_converter_log_file} || exit 1 + cp ${benchmark_test_path}/run_converter_result.txt ${run_converter_result_file} || exit 1 + cp ${benchmark_test_path}/run_benchmark_log.txt ${run_benchmark_log_file} || exit 1 + cp ${benchmark_test_path}/run_benchmark_result.txt ${run_benchmark_result_file} || exit 1 +else + scp ${user_name}@${device_ip}:${benchmark_test_path}/run_converter_log.txt ${run_converter_log_file} || exit 1 + scp ${user_name}@${device_ip}:${benchmark_test_path}/run_converter_result.txt ${run_converter_result_file} || exit 1 + scp ${user_name}@${device_ip}:${benchmark_test_path}/run_benchmark_log.txt ${run_benchmark_log_file} || exit 1 + scp ${user_name}@${device_ip}:${benchmark_test_path}/run_benchmark_result.txt ${run_benchmark_result_file} || exit 1 +fi echo "Run in ${backend} ended" Print_Converter_Result ${run_ascend_result_file} exit ${Run_ascend_status} diff --git a/mindspore/lite/test/st/scripts/ascend/run_benchmark_ascend.sh b/mindspore/lite/test/st/scripts/ascend/run_benchmark_ascend.sh index b09877b5235..0403d1a03a0 100644 --- a/mindspore/lite/test/st/scripts/ascend/run_benchmark_ascend.sh +++ b/mindspore/lite/test/st/scripts/ascend/run_benchmark_ascend.sh @@ -1,7 +1,7 @@ #!/bin/bash # Example:sh run_remote_ascend.sh -v version -b backend -while getopts "v:b:" opt; do +while getopts "v:b:d:" opt; do case ${opt} in v) version=${OPTARG} @@ -11,15 +11,21 @@ while getopts "v:b:" opt; do backend=${OPTARG} echo "backend is ${backend}" ;; + d) + device_id=${OPTARG} + echo "device id is ${device_id}" + ;; ?) echo "unknown para" exit 1;; esac done +export ASCEND_DEVICE_ID=${device_id} + # Run Benchmark in Ascend platform: function Run_Benchmark() { - cd ${x86_path}/mindspore-lite-${version}-linux-x64/ || exit 1 + cd ${benchmark_test}/mindspore-lite-${version}-linux-x64 || exit 1 cp tools/benchmark/benchmark ./ || exit 1 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./runtime/lib @@ -51,7 +57,7 @@ function Run_Benchmark() { model_file=${ms_models_path}'/'${model_name}'.ms' input_files="" output_file="" - data_path=${basepath}'/data/' + data_path=${model_data_path}'/data/' if [[ ${input_num} == "" || ${input_num} == 1 ]]; then input_files=${data_path}'input/'${model_name}'.ms.bin' else @@ -76,8 +82,8 @@ function Run_Benchmark() { fi # different tensorrt run mode use different cuda command - echo './benchmark --modelFile='${model_file}' --inputShapes='${input_shapes}' --inDataFile='${input_files}' --benchmarkDataFile='${output_file}' --enableFp16='${enableFp16}' --accuracyThreshold='${acc_limit}' --device='${backend} >> "${run_ascend_log_file}" - ./benchmark --modelFile=${model_file} --inputShapes=${input_shapes} --inDataFile=${input_files} --benchmarkDataFile=${output_file} --enableFp16=${enableFp16} --accuracyThreshold=${acc_limit} --device=${backend} >> ${run_ascend_log_file} + echo './benchmark --modelFile='${model_file}' --inputShapes='${input_shapes}' --inDataFile='${input_files}' --benchmarkDataFile='${output_file}' --enableFp16='${enableFp16}' --accuracyThreshold='${acc_limit}' --device='${ascend_device} >> "${run_ascend_log_file}" + ./benchmark --modelFile=${model_file} --inputShapes=${input_shapes} --inDataFile=${input_files} --benchmarkDataFile=${output_file} --enableFp16=${enableFp16} --accuracyThreshold=${acc_limit} --device=${ascend_device} >> ${run_ascend_log_file} if [ $? = 0 ]; then run_result=${backend}': '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file} @@ -88,20 +94,29 @@ function Run_Benchmark() { done < ${models_ascend_config} } -basepath=/home/ascend -x86_path=${basepath}/release -ms_models_path=${basepath}/ms_models -models_ascend_config=${basepath}/config/models_ascend.cfg -data_path=${basepath}/data +user_name=${USER} +benchmark_test=/home/${user_name}/benchmark_test/${device_id} +ms_models_path=${benchmark_test}/ms_models +models_ascend_config=${benchmark_test}/models_ascend.cfg +model_data_path=/home/workspace/mindspore_dataset/mslite + # Write benchmark result to temp file -run_benchmark_result_file=${basepath}/scripts/log/run_benchmark_result.txt +run_benchmark_result_file=${benchmark_test}/run_benchmark_result.txt echo ' ' > ${run_benchmark_result_file} #################### run simple Ascend models -run_ascend_log_file=${basepath}/scripts/log/run_benchmark_log.txt +run_ascend_log_file=${benchmark_test}/run_benchmark_log.txt echo 'run Ascend logs: ' > ${run_ascend_log_file} -echo "Start to run benchmark in ${backend} ..." +echo "Start to run benchmark in ${backend}, device id ${device_id}..." +if [[ ${backend} =~ "ascend310" ]]; then + ascend_device=Ascend310 +elif [[ ${backend} =~ "ascend710" ]]; then + ascend_device=Ascend710 +else + echo "${backend} is not support." + exit 1 +fi Run_Benchmark Run_benchmark_status=$? if [[ ${Run_benchmark_status} = 0 ]];then diff --git a/mindspore/lite/test/st/scripts/ascend/run_remote_ascend.sh b/mindspore/lite/test/st/scripts/ascend/run_converter_ascend.sh similarity index 69% rename from mindspore/lite/test/st/scripts/ascend/run_remote_ascend.sh rename to mindspore/lite/test/st/scripts/ascend/run_converter_ascend.sh index 29846e9bee0..e33475a0c0a 100644 --- a/mindspore/lite/test/st/scripts/ascend/run_remote_ascend.sh +++ b/mindspore/lite/test/st/scripts/ascend/run_converter_ascend.sh @@ -1,21 +1,13 @@ #!/bin/bash -source /root/miniconda3/bin/activate base -source /home/ascend/scripts/base_functions.sh # Run converter for ascend x86 platform: function Run_Converter() { - # Unzip x86 runtime and converter cd ${x86_path} || exit 1 - tar -zxf ${x86_path}/mindspore-lite-${version}-linux-x64.tar.gz || exit 1 tar -zxf mindspore-lite-${version}-linux-x64.tar.gz || exit 1 cd ${x86_path}/mindspore-lite-${version}-linux-x64/ || exit 1 - cp tools/converter/converter/converter_lite ./ || exit 1 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./tools/converter/lib/ - rm -rf ${ms_models_path} - mkdir -p ${ms_models_path} - # Prepare the config file list local ascend_cfg_file_list=("$models_ascend_config") # Convert models: @@ -31,29 +23,37 @@ export ASCEND_OPP_PATH=${ASCEND_HOME}/opp export TBE_IMPL_PATH=${ASCEND_HOME}/opp/op_impl/built-in/ai_core/tbe export PYTHONPATH=${TBE_IMPL_PATH}:${PYTHONPATH} -basepath=/home/ascend -x86_fail_not_return="OFF" # This value could not be set to ON. -x86_path=${basepath}/release +backend=$1 +device_id=$2 +user_name=${USER} +echo "Current user is ${USER}" +benchmark_test=/home/${user_name}/benchmark_test/${device_id} +echo "Benchmark test path is ${benchmark_test}" + +x86_path=${benchmark_test} file_name=$(ls ${x86_path}/*-linux-x64.tar.gz) IFS="-" read -r -a file_name_array <<< "$file_name" version=${file_name_array[2]} -models_path=${basepath}/models -ms_models_path=${basepath}/ms_models -ms_scripts_path=${basepath}/scripts -models_ascend_config=${basepath}/config/models_ascend.cfg +ms_models_path=${benchmark_test}/ms_models +rm -rf ${ms_models_path} +mkdir -p ${ms_models_path} + +model_data_path=/home/workspace/mindspore_dataset/mslite +models_path=${model_data_path}/models +models_ascend_config=${benchmark_test}/models_ascend.cfg # Write converter result to temp file -run_converter_log_file=${ms_scripts_path}/log/run_converter_log.txt +run_converter_log_file=${benchmark_test}/run_converter_log.txt echo ' ' > ${run_converter_log_file} -run_converter_result_file=${ms_scripts_path}/log/run_converter_result.txt +run_converter_result_file=${benchmark_test}/run_converter_result.txt echo ' ' > ${run_converter_result_file} -backend=$1 - # Run converter -echo "Start to run converter in ${backend} ..." +x86_fail_not_return="OFF" # This value could not be set to ON. +source ${benchmark_test}/base_functions.sh +echo "Start to run converter in ${backend}, device id ${device_id} ..." Run_Converter if [[ $? = 0 ]]; then echo "Run converter success" @@ -66,7 +66,6 @@ else fi # Run Benchmark -source ${ms_scripts_path}/run_benchmark_ascend.sh -v $version -b $backend +source ${benchmark_test}/run_benchmark_ascend.sh -v $version -b $backend -d $device_id Run_Benchmark_status=$? exit ${Run_Benchmark_status} - diff --git a/mindspore/lite/test/st/scripts/base_functions.sh b/mindspore/lite/test/st/scripts/base_functions.sh index c5f8aa8fbf3..895af9f3906 100644 --- a/mindspore/lite/test/st/scripts/base_functions.sh +++ b/mindspore/lite/test/st/scripts/base_functions.sh @@ -63,8 +63,8 @@ function Convert() { elif [[ ${cfg_file_name} =~ "_train" ]]; then train_model="true" elif [[ ${cfg_file_name} =~ "_ascend" ]]; then - model_option_path="${cfg_file%/*}/model_options" - option_file="${model_option_path}/${model_name}.txt" + model_path=$2 + option_file="${model_path}/model_option/${model_name}.txt" if [ -f "$option_file" ]; then config_file=${option_file} fi diff --git a/mindspore/lite/tools/benchmark/benchmark_unified_api.cc b/mindspore/lite/tools/benchmark/benchmark_unified_api.cc index f82a1fc9b00..4e393f40c6c 100644 --- a/mindspore/lite/tools/benchmark/benchmark_unified_api.cc +++ b/mindspore/lite/tools/benchmark/benchmark_unified_api.cc @@ -391,8 +391,18 @@ int BenchmarkUnifiedApi::InitMSContext(const std::shared_ptr } if (flags_->device_ == "Ascend310" || flags_->device_ == "Ascend710") { + uint32_t device_id = 0; + auto device_id_env = std::getenv("ASCEND_DEVICE_ID"); + if (device_id_env != nullptr) { + try { + device_id = static_cast(std::stoul(device_id_env)); + } catch (std::invalid_argument &e) { + MS_LOG(WARNING) << "Invalid device id env:" << device_id_env << ". Set default device id 0."; + } + MS_LOG(INFO) << "Ascend device_id = " << device_id; + } std::shared_ptr ascend_device_info = std::make_shared(); - ascend_device_info->SetDeviceID(0); + ascend_device_info->SetDeviceID(device_id); device_list.push_back(ascend_device_info); }