From 295a4dba302e3d5ad7e909d7f36abe681017259c Mon Sep 17 00:00:00 2001 From: luoyuan Date: Sun, 12 Feb 2023 10:32:35 +0800 Subject: [PATCH] [MS][LITE]fix bug for import cloud and lite at the same time --- cmake/external_libs/protobuf.cmake | 4 ++-- mindspore/core/mindrt/include/actor/actor.h | 4 ++-- mindspore/core/mindrt/src/actor/actormgr.cc | 5 ++++- mindspore/core/mindrt/src/thread/actor_threadpool.cc | 4 ++-- .../core/mindrt/src/thread/parallel_threadpool.h | 12 ++++++------ .../run_benchmark_server_inference_tensorrt.sh | 1 - .../run_benchmark_server_inference_tensorrt_cloud.sh | 1 - mindspore/lite/test/st/scripts/run_benchmark_x86.sh | 2 +- mindspore/lite/test/st/scripts/run_net_train.sh | 4 ++-- mindspore/lite/tools/cropper/build_cropper_config.sh | 9 +++++++++ 10 files changed, 28 insertions(+), 18 deletions(-) diff --git a/cmake/external_libs/protobuf.cmake b/cmake/external_libs/protobuf.cmake index e4120b57530..8d0d034e8e9 100644 --- a/cmake/external_libs/protobuf.cmake +++ b/cmake/external_libs/protobuf.cmake @@ -165,7 +165,7 @@ function(ms_protobuf_generate_py c_var h_var py_var) COMMAND perl -pi.bak -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/proto_py/proto/${file_name}_pb2.py" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_BINARY_DIR}/proto_py/proto/${file_name}_pb2.py" - "${PROJECT_SOURCE_DIR}/mindspore/python/mindspore/train/" + "${TOP_DIR}/mindspore/python/mindspore/train/" DEPENDS protobuf::protoc ${abs_file} COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM) else() @@ -180,7 +180,7 @@ function(ms_protobuf_generate_py c_var h_var py_var) COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/proto_py/proto/${file_name}_pb2.py" COMMAND cp "${CMAKE_BINARY_DIR}/proto_py/proto/${file_name}_pb2.py" - "${PROJECT_SOURCE_DIR}/mindspore/python/mindspore/train/" + "${TOP_DIR}/mindspore/python/mindspore/train/" DEPENDS protobuf::protoc ${abs_file} COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM) endif() diff --git a/mindspore/core/mindrt/include/actor/actor.h b/mindspore/core/mindrt/include/actor/actor.h index 9075f30a236..764db71efd6 100644 --- a/mindspore/core/mindrt/include/actor/actor.h +++ b/mindspore/core/mindrt/include/actor/actor.h @@ -1,5 +1,5 @@ /** - * Copyright 2021 Huawei Technologies Co., Ltd + * Copyright 2021-2023 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -205,7 +205,7 @@ class MS_CORE_API ActorBase { void Spawn(const std::shared_ptr, std::unique_ptr mailbox); std::unique_ptr mailbox; - std::atomic_bool terminating_ = false; + std::atomic_bool terminating_{false}; AID id; std::map actionFunctions; diff --git a/mindspore/core/mindrt/src/actor/actormgr.cc b/mindspore/core/mindrt/src/actor/actormgr.cc index ba05fc0e702..aac129236ea 100644 --- a/mindspore/core/mindrt/src/actor/actormgr.cc +++ b/mindspore/core/mindrt/src/actor/actormgr.cc @@ -1,5 +1,5 @@ /** - * Copyright 2021 Huawei Technologies Co., Ltd + * Copyright 2021-2023 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -125,6 +125,9 @@ void ActorMgr::RemoveActor(const std::string &name) { } void ActorMgr::TerminateAll() { + if (actors.empty()) { + return; + } // copy all the actors std::list actorsWaiting; actorsMutex.lock(); diff --git a/mindspore/core/mindrt/src/thread/actor_threadpool.cc b/mindspore/core/mindrt/src/thread/actor_threadpool.cc index 1f7697f274f..2b9aeac0998 100644 --- a/mindspore/core/mindrt/src/thread/actor_threadpool.cc +++ b/mindspore/core/mindrt/src/thread/actor_threadpool.cc @@ -1,5 +1,5 @@ /** - * Copyright 2021 Huawei Technologies Co., Ltd + * Copyright 2021-2023 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ void ActorWorker::RunWithSpin() { SetAffinity(); } #if !defined(__APPLE__) && !defined(_MSC_VER) - static std::atomic_int index = {0}; + static std::atomic_int index{0}; (void)pthread_setname_np(pthread_self(), ("ActorThread_" + std::to_string(index++)).c_str()); #endif #ifdef PLATFORM_86 diff --git a/mindspore/core/mindrt/src/thread/parallel_threadpool.h b/mindspore/core/mindrt/src/thread/parallel_threadpool.h index 8e918178efb..bb72b01c51e 100644 --- a/mindspore/core/mindrt/src/thread/parallel_threadpool.h +++ b/mindspore/core/mindrt/src/thread/parallel_threadpool.h @@ -1,5 +1,5 @@ /** - * Copyright 2022 Huawei Technologies Co., Ltd + * Copyright 2022-2023 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,8 +33,8 @@ typedef struct Distributor { typedef struct ParallelTask : public Task { ParallelTask() : Task(nullptr, nullptr) {} std::atomic distributor; - std::atomic_bool valid = false; - std::atomic_bool occupied = false; + std::atomic_bool valid{false}; + std::atomic_bool occupied{false}; } ParallelTask; class ParallelThreadPool; class ParallelWorker : public Worker { @@ -151,14 +151,14 @@ class ParallelThreadPool : public ActorThreadPool { ParallelThreadPool() {} int CreateParallelThreads(size_t actor_thread_num, size_t all_thread_num, const std::vector &core_list); - std::atomic_int tasks_start_ = 0; - std::atomic_int tasks_end_ = 0; + std::atomic_int tasks_start_{0}; + std::atomic_int tasks_end_{0}; ParallelTask *tasks_; size_t tasks_size_ = 0; bool enable_shared_ = false; std::string bind_runner_id_; std::mutex mutex_pool_ref_count_; - std::atomic_int pool_ref_count_ = 0; + std::atomic_int pool_ref_count_{0}; int thread_num_; }; } // namespace mindspore diff --git a/mindspore/lite/test/st/scripts/run_benchmark_server_inference_tensorrt.sh b/mindspore/lite/test/st/scripts/run_benchmark_server_inference_tensorrt.sh index e6a05dbf05b..4acaedda116 100644 --- a/mindspore/lite/test/st/scripts/run_benchmark_server_inference_tensorrt.sh +++ b/mindspore/lite/test/st/scripts/run_benchmark_server_inference_tensorrt.sh @@ -218,7 +218,6 @@ if [[ $(Exist_File_In_Path ${ms_models_path} ".ms") != "true" ]]; then fi # Write benchmark result to temp file -export GLOG_logtostderr=0 run_benchmark_result_file=${basepath}/run_benchmark_result.txt echo ' ' > ${run_benchmark_result_file} diff --git a/mindspore/lite/test/st/scripts/run_benchmark_server_inference_tensorrt_cloud.sh b/mindspore/lite/test/st/scripts/run_benchmark_server_inference_tensorrt_cloud.sh index 710588257ba..4cec25375a2 100644 --- a/mindspore/lite/test/st/scripts/run_benchmark_server_inference_tensorrt_cloud.sh +++ b/mindspore/lite/test/st/scripts/run_benchmark_server_inference_tensorrt_cloud.sh @@ -235,7 +235,6 @@ if [[ $(Exist_File_In_Path ${ms_models_path} ".mindir") != "true" ]]; then fi # Write benchmark result to temp file -export GLOG_logtostderr=0 run_benchmark_result_file=${basepath}/run_benchmark_result.txt echo ' ' > ${run_benchmark_result_file} diff --git a/mindspore/lite/test/st/scripts/run_benchmark_x86.sh b/mindspore/lite/test/st/scripts/run_benchmark_x86.sh index ee15da669df..1d83e8f4f0f 100644 --- a/mindspore/lite/test/st/scripts/run_benchmark_x86.sh +++ b/mindspore/lite/test/st/scripts/run_benchmark_x86.sh @@ -158,7 +158,7 @@ function Run_x86_parallel_split() { rm -rf parallel_split mkdir parallel_split cd parallel_split || exit 1 - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../runtime/lib:./runtime/third_party/glog + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../runtime/lib:../runtime/third_party/glog cp ../tools/benchmark/benchmark ./ || exit 1 # Run tflite parallel split converted models: diff --git a/mindspore/lite/test/st/scripts/run_net_train.sh b/mindspore/lite/test/st/scripts/run_net_train.sh index 8d2122dcb77..95a41090c54 100755 --- a/mindspore/lite/test/st/scripts/run_net_train.sh +++ b/mindspore/lite/test/st/scripts/run_net_train.sh @@ -224,7 +224,7 @@ function parse_line() { function Run_x86() { # $1:cfgFileList; cd ${x86_path}/mindspore-lite-${version}-linux-x64 || return 1 - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./runtime/lib:./runtime/third_party/libjpeg-turbo/lib + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${x86_path}/mindspore-lite-${version}-linux-x64/runtime/lib:${x86_path}/mindspore-lite-${version}-linux-x64/runtime/third_party/libjpeg-turbo/lib # Run mindspore converted train models: local fail=0 while read line; do @@ -676,7 +676,7 @@ adb_cmd_arm32_file=${logs_path}/adb_arm32_cmd.txt adb_cmd_arm32_run_file=${logs_path}/adb_arm32_cmd_run.txt run_code_examples_log_file=${logs_path}/run_code_examples_log.txt -echo 'run code examlpe logs: ' > ${run_code_examples_log_file} +echo 'run code example logs: ' > ${run_code_examples_log_file} isFailed=0 if [[ $backend == "all" || $backend == "train" || $backend == "x86_train" ]]; then diff --git a/mindspore/lite/tools/cropper/build_cropper_config.sh b/mindspore/lite/tools/cropper/build_cropper_config.sh index f97a0e67e1b..52a21063182 100644 --- a/mindspore/lite/tools/cropper/build_cropper_config.sh +++ b/mindspore/lite/tools/cropper/build_cropper_config.sh @@ -66,6 +66,9 @@ HEADER_LOCATION="-I${MINDSPORE_HOME} REMOVE_LISTS_STR="" getDeep() { map_files=$(gcc -MM ${2} ${DEFINE_STR} ${HEADER_LOCATION}) + if [[ ${map_files} == "" ]]; then + echo "failed to get deep any file from file: ${2}, compile terminated unexpectedly" + fi # first is *.o second is *.cc array_deep=() while IFS='' read -r line; do array_deep+=("$line"); done < <(echo ${map_files} | awk -F '\' '{for(i=3;i<=NF;i++){print $i}}' | egrep -v 'flatbuffers|build|third_party|type_id.h|core/utils' | egrep -v ${REMOVE_LISTS_STR}) @@ -104,6 +107,9 @@ getOpsFile() { echo "${type},${3},${out_file}.o" >>${MAPPING_OUTPUT_FILE_NAME_TMP} echo "${type},${3},${out_file}.o" >>${MAPPING_OUTPUT_FILE_NAME_TRAIN_TMP} map_files=$(gcc -MM ${file} ${DEFINE_STR} ${HEADER_LOCATION}) + if [[ ${map_files} == "" ]]; then + echo "failed to get operator mapping any file from file: ${file}, compile terminated unexpectedly" + fi # first is *.o second is *.cc array_file=() while IFS='' read -r line; do array_file+=("$line"); done < <(echo ${map_files} | awk -F '\' '{for(i=3;i<=NF;i++){print $i}}' | egrep -v 'flatbuffers|build|third_party|type_id.h|core/utils' | egrep -v ${REMOVE_LISTS_STR}) @@ -136,6 +142,9 @@ getFilesFromArr() { # shellcheck disable=SC2068 for file in ${arr_files[*]}; do map_files=$(gcc -MM ${file} ${DEFINE_STR} ${HEADER_LOCATION}) + if [[ ${map_files} == "" ]]; then + echo "failed to get any file from arr_file: ${file}, compile terminated unexpectedly" + fi # first is *.o second is *.cc # shellcheck disable=SC2207 array_runtime=($(echo ${map_files} | awk -F '\' '{for(i=3;i<=NF;i++){print $i}}' | egrep -v 'flatbuffers|build|third_party|type_id.h' | egrep -v ${REMOVE_LISTS_STR}))