From 33583ce7944ce700a86ad972c489c603f1677253 Mon Sep 17 00:00:00 2001 From: zengxianglong Date: Wed, 24 Mar 2021 10:37:07 +0800 Subject: [PATCH] multi-inputs models support fp16 mode and add models to the entrance guard --- mindspore/lite/test/models_caffe.cfg | 3 + .../lite/test/models_for_process_only.cfg | 1 - mindspore/lite/test/models_npu.cfg | 4 +- mindspore/lite/test/models_onnx.cfg | 4 +- mindspore/lite/test/models_onnx_fp16.cfg | 3 + mindspore/lite/test/models_tflite.cfg | 9 +++ mindspore/lite/test/models_tflite_fp16.cfg | 9 +++ .../test/models_with_multiple_inputs_fp16.cfg | 7 ++ mindspore/lite/test/run_benchmark_nets.sh | 78 ++++++++++++++++--- 9 files changed, 103 insertions(+), 15 deletions(-) create mode 100644 mindspore/lite/test/models_with_multiple_inputs_fp16.cfg diff --git a/mindspore/lite/test/models_caffe.cfg b/mindspore/lite/test/models_caffe.cfg index 10a1e0517d9..c12ed5ae21f 100644 --- a/mindspore/lite/test/models_caffe.cfg +++ b/mindspore/lite/test/models_caffe.cfg @@ -89,3 +89,6 @@ ml_video_edit_person_divison_pic ml_video_edit_reid ml_video_edit_v10_best_model_nomean_20200723 ml_handpose +ml_lable_model_hebing_device +ml_face_sex +ml_face_hat diff --git a/mindspore/lite/test/models_for_process_only.cfg b/mindspore/lite/test/models_for_process_only.cfg index e6401e46003..ba1c96b98f1 100644 --- a/mindspore/lite/test/models_for_process_only.cfg +++ b/mindspore/lite/test/models_for_process_only.cfg @@ -18,7 +18,6 @@ efficientnet_lite3_int8_2.tflite efficientnet_lite4_int8_2.tflite mtk_transformer_encoder.tflite mtk_transformer_decoder_joint.tflite -ml_ei_facedetection.onnx quant_aware_bank_card_detection_inception.onnx quant_aware_bank_card_recognition_fcny.onnx quant_aware_identify_card_detect.onnx diff --git a/mindspore/lite/test/models_npu.cfg b/mindspore/lite/test/models_npu.cfg index 4bf2a39bfeb..6831197ebad 100644 --- a/mindspore/lite/test/models_npu.cfg +++ b/mindspore/lite/test/models_npu.cfg @@ -41,7 +41,6 @@ ml_video_edit_img_segment_adaptise.pb 0.5 2 ml_video_edit_video_segment_gauss_adaptis_part2.pb 10 2 ml_video_edit_person_divison_pic 0.5 ml_video_edit_person_divison_video 13 2 -ml_video_edit_imitate_filter.onnx 230 ml_video_edit_judge.onnx 5 ml_video_edit_vignet.onnx 0.5 hdc_Face_Aesthetic_MTI_Aesthetic 0.5 @@ -73,6 +72,7 @@ densenet.tflite 3 ml_video_edit_enhance.pb 2 ml_video_edit_video_segment_gauss_adaptis_part2_pb2tflite.tflite 3 2 ml_video_edit_img_segment_adaptise_pb2tflite.tflite 0.5 2 -ml_video_edit_imitate_filter.onnx 103 +#the fifth value of the ml_video_edit_imitate_filter.onnx's output is very small (10-5). +ml_video_edit_imitate_filter.onnx 200 hdc_mobilenet_1w_class.onnx 20 hdc_age_medium 504 diff --git a/mindspore/lite/test/models_onnx.cfg b/mindspore/lite/test/models_onnx.cfg index 26fad6a0e13..10bff56c02d 100644 --- a/mindspore/lite/test/models_onnx.cfg +++ b/mindspore/lite/test/models_onnx.cfg @@ -27,7 +27,7 @@ psenet_lite_mbv2.onnx;1,32,32,3 super-resolution-10.onnx;1,224,224,1 tinyyolov2-8.onnx;1,416,416,3 ml_2012_ocr_cn.onnx -#ml_2012_ocr_cn_noLSTM.onnx +ml_2012_ocr_cn_noLSTM.onnx candy-9.onnx mosaic-9.onnx pointilism-9.onnx @@ -56,3 +56,5 @@ hdc_ocr_attention.onnx hdc_ocr_detect.onnx ml_edu_kit_hand_detection.onnx ml_edu_kit_hand_key_position.onnx +ml_facedetector.onnx +ml_ei_facedetection.onnx diff --git a/mindspore/lite/test/models_onnx_fp16.cfg b/mindspore/lite/test/models_onnx_fp16.cfg index c8aad34fba3..eedfd269c95 100644 --- a/mindspore/lite/test/models_onnx_fp16.cfg +++ b/mindspore/lite/test/models_onnx_fp16.cfg @@ -52,3 +52,6 @@ hdc_Image_Aesthetic_MTI_Aesthetic.onnx 0.5 hdc_resnet_1w_class.onnx 6 gts_text_detection.onnx;1,224,224,3 10 hdc_Face_Emotion_MTI_Aesthetic.onnx 144 +ml_video_edit_imitate_filter.onnx 103 +ml_facedetector.onnx 3 +ml_ei_facedetection.onnx 2 diff --git a/mindspore/lite/test/models_tflite.cfg b/mindspore/lite/test/models_tflite.cfg index bb8909455b8..2af015f204b 100644 --- a/mindspore/lite/test/models_tflite.cfg +++ b/mindspore/lite/test/models_tflite.cfg @@ -138,3 +138,12 @@ mindspore_text_classification_tflite.tflite # ml_location.tflite ml_text_correction.tflite ml_pic_shopping.tflite +ml_vision_guide_detection3_pb2tflite.tflite +ml_pic_shopping_pb2tflite.tflite +ml_ocr_jk_pb2tflite.tflite +ml_ocr_latin_pb2tflite.tflite +scan_hms_angle_pb2tflite.tflite +scan_hms_detect_pb2tflite.tflite +ml_location.tflite +ml_face_openclose_tflite.tflite +ml_object_detect_pb2tflite.tflite diff --git a/mindspore/lite/test/models_tflite_fp16.cfg b/mindspore/lite/test/models_tflite_fp16.cfg index 5309061db0e..a824753fe48 100644 --- a/mindspore/lite/test/models_tflite_fp16.cfg +++ b/mindspore/lite/test/models_tflite_fp16.cfg @@ -140,3 +140,12 @@ ml_text_correction.tflite 1 # e.g. fp32: 27.5 -27.4 = 0.1 # fp16: 27.6 - 27.4 = 0.2 #ml_pic_shopping.tflite 0.1 +ml_vision_guide_detection3_pb2tflite.tflite 0.5 +ml_pic_shopping_pb2tflite.tflite 95 +ml_ocr_jk_pb2tflite.tflite 0.5 +ml_ocr_latin_pb2tflite.tflite 11 +scan_hms_angle_pb2tflite.tflite 2.5 +scan_hms_detect_pb2tflite.tflite 1.5 +ml_location.tflite 0.5 +ml_face_openclose_tflite.tflite 0.5 +ml_object_detect_pb2tflite.tflite 1.5 diff --git a/mindspore/lite/test/models_with_multiple_inputs_fp16.cfg b/mindspore/lite/test/models_with_multiple_inputs_fp16.cfg new file mode 100644 index 00000000000..167c1ec8e3c --- /dev/null +++ b/mindspore/lite/test/models_with_multiple_inputs_fp16.cfg @@ -0,0 +1,7 @@ +ml_video_edit_video_segment_gauss_adaptis_part2_pb2tflite.tflite;2 11 +ml_video_edit_video_segment_gauss_adaptis_part2.pb;2 11 +ml_video_edit_img_segment_adaptise.pb;2 40 +ml_video_edit_img_segment_adaptise_pb2tflite.tflite;2 0.5 +ml_video_edit_person_divison_video;2 38 +ml_video_edit_oneclick_adaptis.pb;3 6 +hdc_tb_cn_neg.tflite;3 281 diff --git a/mindspore/lite/test/run_benchmark_nets.sh b/mindspore/lite/test/run_benchmark_nets.sh index 2a1711eba44..3c35f8d9388 100755 --- a/mindspore/lite/test/run_benchmark_nets.sh +++ b/mindspore/lite/test/run_benchmark_nets.sh @@ -315,6 +315,22 @@ function Run_Converter() { fi done < ${models_with_multiple_inputs_config} + while read line; do + fp16_line_info=${line} + if [[ $fp16_line_info == \#* ]]; then + continue + fi + model_info=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'` + model_name=${model_info%%;*} + echo 'cp '${ms_models_path}'/'${model_name}'.ms' ${ms_models_path}'/'${model_name}'.fp16.ms' + cp ${ms_models_path}/${model_name}.ms ${ms_models_path}/${model_name}.fp16.ms + if [ $? = 0 ]; then + converter_result='converter fp16 '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file} + else + converter_result='converter fp16 '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1 + fi + done < ${models_multiple_inputs_fp16_config} + # Convert models which does not need to be cared about the accuracy: while read line; do if [[ $line == \#* ]]; then @@ -1778,6 +1794,44 @@ function Run_arm64_fp16() { run_result='arm64_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1 fi done < ${models_tflite_fp16_config} + + # Run converted models which has multiple inputs in fp16 mode: + while read line; do + fp16_line_info=${line} + if [[ $fp16_line_info == \#* ]]; then + continue- + fi + model_info=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'` + accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $2}'` + model_name=`echo ${model_info}|awk -F ';' '{print $1}'` + input_num=`echo ${model_info} | awk -F ';' '{print $2}'` + input_shapes=`echo ${model_info} | awk -F ';' '{print $3}'` + input_files='' + output_file='' + data_path="/data/local/tmp/input_output/" + for i in $(seq 1 $input_num) + do + input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i',' + done + output_file=${data_path}'output/'${model_name}'.ms.out' + if [[ ${model_name##*.} == "caffemodel" ]]; then + model_name=${model_name%.*} + fi + echo "---------------------------------------------------------" >> "${run_arm64_fp16_log_file}" + echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_arm64_fp16_log_file}" + + echo 'cd /data/local/tmp/benchmark_test' > adb_run_cmd.txt + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt + echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile='${input_files}' --inputShapes='${input_shapes}' --benchmarkDataFile='${output_file} '--enableFp16=true --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt + + cat adb_run_cmd.txt >> "${run_arm64_fp16_log_file}" + adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp16_log_file}" + if [ $? = 0 ]; then + run_result='arm64_fp16: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file} + else + run_result='arm64_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1 + fi + done < ${models_multiple_inputs_fp16_config} } # Run on gpu platform: function Run_gpu() { @@ -2002,6 +2056,7 @@ models_onnx_config=${basepath}/models_onnx.cfg models_onnx_fp16_config=${basepath}/models_onnx_fp16.cfg models_caffe_fp16_config=${basepath}/models_caffe_fp16.cfg models_tflite_fp16_config=${basepath}/models_tflite_fp16.cfg +models_multiple_inputs_fp16_config=${basepath}/models_with_multiple_inputs_fp16.cfg models_mindspore_config=${basepath}/models_mindspore.cfg models_mindspore_train_config=${basepath}/models_mindspore_train.cfg models_mindspore_mixbit_config=${basepath}/models_mindspore_mixbit.cfg @@ -2175,17 +2230,6 @@ if [[ $backend == "all" || $backend == "gpu_npu" || $backend == "npu" ]]; then sleep 1 fi -if [[ $backend == "all" || $backend == "x86-all" || $backend == "x86" ]]; then - wait ${Run_x86_PID} - Run_x86_status=$? - - # Check benchmark result and return value - if [[ ${Run_x86_status} != 0 ]];then - echo "Run_x86 failed" - cat ${run_x86_log_file} - isFailed=1 - fi -fi if [[ $backend == "all" || $backend == "x86-all" || $backend == "x86-sse" ]]; then wait ${Run_x86_sse_PID} Run_x86_sse_status=$? @@ -2207,6 +2251,18 @@ if [[ $backend == "all" || $backend == "x86-all" || $backend == "x86-avx" ]]; th fi fi +if [[ $backend == "all" || $backend == "x86-all" || $backend == "x86" ]]; then + wait ${Run_x86_PID} + Run_x86_status=$? + + # Check benchmark result and return value + if [[ ${Run_x86_status} != 0 ]];then + echo "Run_x86 failed" + cat ${run_x86_log_file} + isFailed=1 + fi +fi + if [[ $backend == "all" || $backend == "arm_cpu" || $backend == "arm64_fp32" ]]; then if [[ ${Run_arm64_fp32_status} != 0 ]];then echo "Run_arm64_fp32 failed"