Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[MS][LITE]fix bug for import cloud and lite at the same time

This commit is contained in:
luoyuan 2023-02-12 10:32:35 +08:00
parent c29a919b72
commit 295a4dba30
10 changed files with 28 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmake/external_libs/protobuf.cmake
View File

@ -165,7 +165,7 @@ function(ms_protobuf_generate_py c_var h_var py_var)
COMMAND perl -pi.bak -e "s/import (.+_pb2.*)/from . import \\1/" COMMAND perl -pi.bak -e "s/import (.+_pb2.*)/from . import \\1/"
"${CMAKE_BINARY_DIR}/proto_py/proto/${file_name}_pb2.py" "${CMAKE_BINARY_DIR}/proto_py/proto/${file_name}_pb2.py"
COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_BINARY_DIR}/proto_py/proto/${file_name}_pb2.py" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_BINARY_DIR}/proto_py/proto/${file_name}_pb2.py"
"${PROJECT_SOURCE_DIR}/mindspore/python/mindspore/train/" "${TOP_DIR}/mindspore/python/mindspore/train/"
DEPENDS protobuf::protoc ${abs_file} DEPENDS protobuf::protoc ${abs_file}
COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM) COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM)
else() else()
@ -180,7 +180,7 @@ function(ms_protobuf_generate_py c_var h_var py_var)
COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/" COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/"
"${CMAKE_BINARY_DIR}/proto_py/proto/${file_name}_pb2.py" "${CMAKE_BINARY_DIR}/proto_py/proto/${file_name}_pb2.py"
COMMAND cp "${CMAKE_BINARY_DIR}/proto_py/proto/${file_name}_pb2.py" COMMAND cp "${CMAKE_BINARY_DIR}/proto_py/proto/${file_name}_pb2.py"
"${PROJECT_SOURCE_DIR}/mindspore/python/mindspore/train/" "${TOP_DIR}/mindspore/python/mindspore/train/"
DEPENDS protobuf::protoc ${abs_file} DEPENDS protobuf::protoc ${abs_file}
COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM) COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM)
endif() endif()

4
mindspore/core/mindrt/include/actor/actor.h
View File

@ -1,5 +1,5 @@
/** /**
* Copyright 2021 Huawei Technologies Co., Ltd * Copyright 2021-2023 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -205,7 +205,7 @@ class MS_CORE_API ActorBase {
void Spawn(const std::shared_ptr<ActorBase>, std::unique_ptr<MailBox> mailbox); void Spawn(const std::shared_ptr<ActorBase>, std::unique_ptr<MailBox> mailbox);
std::unique_ptr<MailBox> mailbox; std::unique_ptr<MailBox> mailbox;
std::atomic_bool terminating_ = false; std::atomic_bool terminating_{false};
AID id; AID id;
std::map<std::string, ActorFunction> actionFunctions; std::map<std::string, ActorFunction> actionFunctions;

5
mindspore/core/mindrt/src/actor/actormgr.cc
View File

@ -1,5 +1,5 @@
/** /**
* Copyright 2021 Huawei Technologies Co., Ltd * Copyright 2021-2023 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -125,6 +125,9 @@ void ActorMgr::RemoveActor(const std::string &name) {
} }
void ActorMgr::TerminateAll() { void ActorMgr::TerminateAll() {
if (actors.empty()) {
return;
}
// copy all the actors // copy all the actors
std::list<ActorReference> actorsWaiting; std::list<ActorReference> actorsWaiting;
actorsMutex.lock(); actorsMutex.lock();

4
mindspore/core/mindrt/src/thread/actor_threadpool.cc
View File

@ -1,5 +1,5 @@
/** /**
* Copyright 2021 Huawei Technologies Co., Ltd * Copyright 2021-2023 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -30,7 +30,7 @@ void ActorWorker::RunWithSpin() {
SetAffinity(); SetAffinity();
} }
#if !defined(__APPLE__) && !defined(_MSC_VER) #if !defined(__APPLE__) && !defined(_MSC_VER)
static std::atomic_int index = {0}; static std::atomic_int index{0};
(void)pthread_setname_np(pthread_self(), ("ActorThread_" + std::to_string(index++)).c_str()); (void)pthread_setname_np(pthread_self(), ("ActorThread_" + std::to_string(index++)).c_str());
#endif #endif
#ifdef PLATFORM_86 #ifdef PLATFORM_86

12
mindspore/core/mindrt/src/thread/parallel_threadpool.h
View File

@ -1,5 +1,5 @@
/** /**
* Copyright 2022 Huawei Technologies Co., Ltd * Copyright 2022-2023 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -33,8 +33,8 @@ typedef struct Distributor {
typedef struct ParallelTask : public Task { typedef struct ParallelTask : public Task {
ParallelTask() : Task(nullptr, nullptr) {} ParallelTask() : Task(nullptr, nullptr) {}
std::atomic<Distributor> distributor; std::atomic<Distributor> distributor;
std::atomic_bool valid = false; std::atomic_bool valid{false};
std::atomic_bool occupied = false; std::atomic_bool occupied{false};
} ParallelTask; } ParallelTask;
class ParallelThreadPool; class ParallelThreadPool;
class ParallelWorker : public Worker { class ParallelWorker : public Worker {
@ -151,14 +151,14 @@ class ParallelThreadPool : public ActorThreadPool {
ParallelThreadPool() {} ParallelThreadPool() {}
int CreateParallelThreads(size_t actor_thread_num, size_t all_thread_num, const std::vector<int> &core_list); int CreateParallelThreads(size_t actor_thread_num, size_t all_thread_num, const std::vector<int> &core_list);
std::atomic_int tasks_start_ = 0; std::atomic_int tasks_start_{0};
std::atomic_int tasks_end_ = 0; std::atomic_int tasks_end_{0};
ParallelTask *tasks_; ParallelTask *tasks_;
size_t tasks_size_ = 0; size_t tasks_size_ = 0;
bool enable_shared_ = false; bool enable_shared_ = false;
std::string bind_runner_id_; std::string bind_runner_id_;
std::mutex mutex_pool_ref_count_; std::mutex mutex_pool_ref_count_;
std::atomic_int pool_ref_count_ = 0; std::atomic_int pool_ref_count_{0};
int thread_num_; int thread_num_;
}; };
} // namespace mindspore } // namespace mindspore

1
mindspore/lite/test/st/scripts/run_benchmark_server_inference_tensorrt.sh
View File

@ -218,7 +218,6 @@ if [[ $(Exist_File_In_Path ${ms_models_path} ".ms") != "true" ]]; then
fi fi
# Write benchmark result to temp file # Write benchmark result to temp file
export GLOG_logtostderr=0
run_benchmark_result_file=${basepath}/run_benchmark_result.txt run_benchmark_result_file=${basepath}/run_benchmark_result.txt
echo ' ' > ${run_benchmark_result_file} echo ' ' > ${run_benchmark_result_file}

1
mindspore/lite/test/st/scripts/run_benchmark_server_inference_tensorrt_cloud.sh
View File

@ -235,7 +235,6 @@ if [[ $(Exist_File_In_Path ${ms_models_path} ".mindir") != "true" ]]; then
fi fi
# Write benchmark result to temp file # Write benchmark result to temp file
export GLOG_logtostderr=0
run_benchmark_result_file=${basepath}/run_benchmark_result.txt run_benchmark_result_file=${basepath}/run_benchmark_result.txt
echo ' ' > ${run_benchmark_result_file} echo ' ' > ${run_benchmark_result_file}

2
mindspore/lite/test/st/scripts/run_benchmark_x86.sh
View File

@ -158,7 +158,7 @@ function Run_x86_parallel_split() {
rm -rf parallel_split rm -rf parallel_split
mkdir parallel_split mkdir parallel_split
cd parallel_split || exit 1 cd parallel_split || exit 1
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../runtime/lib:./runtime/third_party/glog export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../runtime/lib:../runtime/third_party/glog
cp ../tools/benchmark/benchmark ./ || exit 1 cp ../tools/benchmark/benchmark ./ || exit 1
# Run tflite parallel split converted models: # Run tflite parallel split converted models:

4
mindspore/lite/test/st/scripts/run_net_train.sh
View File

@ -224,7 +224,7 @@ function parse_line() {
function Run_x86() { function Run_x86() {
# $1:cfgFileList; # $1:cfgFileList;
cd ${x86_path}/mindspore-lite-${version}-linux-x64 || return 1 cd ${x86_path}/mindspore-lite-${version}-linux-x64 || return 1
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./runtime/lib:./runtime/third_party/libjpeg-turbo/lib export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${x86_path}/mindspore-lite-${version}-linux-x64/runtime/lib:${x86_path}/mindspore-lite-${version}-linux-x64/runtime/third_party/libjpeg-turbo/lib
# Run mindspore converted train models: # Run mindspore converted train models:
local fail=0 local fail=0
while read line; do while read line; do
@ -676,7 +676,7 @@ adb_cmd_arm32_file=${logs_path}/adb_arm32_cmd.txt
adb_cmd_arm32_run_file=${logs_path}/adb_arm32_cmd_run.txt adb_cmd_arm32_run_file=${logs_path}/adb_arm32_cmd_run.txt
run_code_examples_log_file=${logs_path}/run_code_examples_log.txt run_code_examples_log_file=${logs_path}/run_code_examples_log.txt
echo 'run code examlpe logs: ' > ${run_code_examples_log_file} echo 'run code example logs: ' > ${run_code_examples_log_file}
isFailed=0 isFailed=0
if [[ $backend == "all" || $backend == "train" || $backend == "x86_train" ]]; then if [[ $backend == "all" || $backend == "train" || $backend == "x86_train" ]]; then

9
mindspore/lite/tools/cropper/build_cropper_config.sh
View File

@ -66,6 +66,9 @@ HEADER_LOCATION="-I${MINDSPORE_HOME}
REMOVE_LISTS_STR="" REMOVE_LISTS_STR=""
getDeep() { getDeep() {
map_files=$(gcc -MM ${2} ${DEFINE_STR} ${HEADER_LOCATION}) map_files=$(gcc -MM ${2} ${DEFINE_STR} ${HEADER_LOCATION})
if [[ ${map_files} == "" ]]; then
echo "failed to get deep any file from file: ${2}, compile terminated unexpectedly"
fi
# first is *.o second is *.cc # first is *.o second is *.cc
array_deep=() array_deep=()
while IFS='' read -r line; do array_deep+=("$line"); done < <(echo ${map_files} | awk -F '\' '{for(i=3;i<=NF;i++){print $i}}' | egrep -v 'flatbuffers|build|third_party|type_id.h|core/utils' | egrep -v ${REMOVE_LISTS_STR}) while IFS='' read -r line; do array_deep+=("$line"); done < <(echo ${map_files} | awk -F '\' '{for(i=3;i<=NF;i++){print $i}}' | egrep -v 'flatbuffers|build|third_party|type_id.h|core/utils' | egrep -v ${REMOVE_LISTS_STR})
@ -104,6 +107,9 @@ getOpsFile() {
echo "${type},${3},${out_file}.o" >>${MAPPING_OUTPUT_FILE_NAME_TMP} echo "${type},${3},${out_file}.o" >>${MAPPING_OUTPUT_FILE_NAME_TMP}
echo "${type},${3},${out_file}.o" >>${MAPPING_OUTPUT_FILE_NAME_TRAIN_TMP} echo "${type},${3},${out_file}.o" >>${MAPPING_OUTPUT_FILE_NAME_TRAIN_TMP}
map_files=$(gcc -MM ${file} ${DEFINE_STR} ${HEADER_LOCATION}) map_files=$(gcc -MM ${file} ${DEFINE_STR} ${HEADER_LOCATION})
if [[ ${map_files} == "" ]]; then
echo "failed to get operator mapping any file from file: ${file}, compile terminated unexpectedly"
fi
# first is *.o second is *.cc # first is *.o second is *.cc
array_file=() array_file=()
while IFS='' read -r line; do array_file+=("$line"); done < <(echo ${map_files} | awk -F '\' '{for(i=3;i<=NF;i++){print $i}}' | egrep -v 'flatbuffers|build|third_party|type_id.h|core/utils' | egrep -v ${REMOVE_LISTS_STR}) while IFS='' read -r line; do array_file+=("$line"); done < <(echo ${map_files} | awk -F '\' '{for(i=3;i<=NF;i++){print $i}}' | egrep -v 'flatbuffers|build|third_party|type_id.h|core/utils' | egrep -v ${REMOVE_LISTS_STR})
@ -136,6 +142,9 @@ getFilesFromArr() {
# shellcheck disable=SC2068 # shellcheck disable=SC2068
for file in ${arr_files[*]}; do for file in ${arr_files[*]}; do
map_files=$(gcc -MM ${file} ${DEFINE_STR} ${HEADER_LOCATION}) map_files=$(gcc -MM ${file} ${DEFINE_STR} ${HEADER_LOCATION})
if [[ ${map_files} == "" ]]; then
echo "failed to get any file from arr_file: ${file}, compile terminated unexpectedly"
fi
# first is *.o second is *.cc # first is *.o second is *.cc
# shellcheck disable=SC2207 # shellcheck disable=SC2207
array_runtime=($(echo ${map_files} | awk -F '\' '{for(i=3;i<=NF;i++){print $i}}' | egrep -v 'flatbuffers|build|third_party|type_id.h' | egrep -v ${REMOVE_LISTS_STR})) array_runtime=($(echo ${map_files} | awk -F '\' '{for(i=3;i<=NF;i++){print $i}}' | egrep -v 'flatbuffers|build|third_party|type_id.h' | egrep -v ${REMOVE_LISTS_STR}))