mindspore-Ecosystem
/
mindspore
21
0
Fork 27
Code Issues Pull Requests 6 Projects Releases Wiki Activity

!34000 Bugfix for symbol error on mac 

Merge pull request !34000 from chengang/bugfix_mac_symbol_2
This commit is contained in:
i-robot 2022-05-07 11:35:09 +00:00 committed by Gitee
parent efe40a1ffc 36ab26b96b
commit c29d6bb764
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
10 changed files with 46 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ccsrc/distributed/init.cc
View File

@ -29,7 +29,7 @@ bool Initialize() {
return false;
}
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (cluster::ClusterContext::instance()->initialized() && !collective::CollectiveManager::instance()->initialized()) {
// Server and Scheduler don't use collective communication library.
auto node = cluster::ClusterContext::instance()->node();

2
mindspore/ccsrc/distributed/init.h
View File

@ -20,7 +20,7 @@
#include <vector>
#include <string>
#include "distributed/collective/collective_manager.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "distributed/cluster/cluster_context.h"
#else
#include "distributed/cluster/dummy_cluster_context.h"

2
mindspore/ccsrc/frontend/parallel/graph_util/graph_splitter.h
View File

@ -29,7 +29,7 @@
#include "include/common/utils/utils.h"
#include "ir/func_graph.h"
#include "distributed/constants.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "distributed/cluster/cluster_context.h"
#else
#include "distributed/cluster/dummy_cluster_context.h"

10
mindspore/ccsrc/pipeline/jit/action.cc
View File

@ -52,7 +52,7 @@
#include "backend/graph_compiler/transform.h"
#include "load_mindir/infer_mindir.h"
#include "debug/data_dump/dump_json_parser.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "ps/parameter_server.h"
#include "ps/scheduler.h"
#include "ps/worker.h"
@ -657,7 +657,7 @@ bool OptInlineAction(const ResourcePtr &res) {
bool GeOptimizeAction(const ResourcePtr &res) { return OptimizeAction(res, kGePasses); }
bool VmOptimizeAction(const ResourcePtr &res) {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::PSContext::instance()->is_ps_mode()) {
(void)kVmPasses.emplace_back(PassItem("server_communication_op_fusion", ps::Util::FuseServerCommOps));
}
@ -1075,7 +1075,7 @@ bool ExecuteAction(const ResourcePtr &res) {
return true;
}
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
bool StartPSWorkerAction(const ResourcePtr &) {
ps::Worker::GetInstance().Run();
return true;
@ -1461,7 +1461,7 @@ std::vector<ActionItem> VmPipeline(const ResourcePtr &resource) {
(void)actions.emplace_back(std::make_pair("validate", ValidateAction));
}
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
(void)actions.emplace_back(std::make_pair("distribtued_split", DistributedSplitAction));
if (ps::PSContext::instance()->is_worker()) {
if (distributed::cluster::ClusterContext::instance()->initialized()) {
@ -1503,7 +1503,7 @@ std::vector<ActionItem> MindIRPipeline() {
return actions;
}
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
std::vector<ActionItem> ServerPipeline(const ResourcePtr &resource) {
if (resource->EnableCompileCache() && resource->func_graph() != nullptr) {
return {std::make_pair("server", StartServerAction)};

4
mindspore/ccsrc/pipeline/jit/compile_cache_manager.cc
View File

@ -29,7 +29,7 @@
#include "include/common/utils/utils.h"
#include "frontend/parallel/step_parallel.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "ps/ps_context.h"
#include "ps/core/node.h"
#include "distributed/cluster/cluster_context.h"
@ -69,7 +69,7 @@ std::string GetCompileCacheDir() {
}
std::string GetRole() {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
const std::string &server_mode = ps::PSContext::instance()->server_mode();
if ((server_mode == ps::kServerModeFL || server_mode == ps::kServerModeHybrid) &&
ps::PSContext::instance()->is_server()) {

8
mindspore/ccsrc/pipeline/jit/pipeline.cc
View File

@ -70,7 +70,7 @@
#endif
#endif
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "ps/constants.h"
#include "ps/util.h"
#include "ps/worker.h"
@ -734,7 +734,7 @@ std::vector<ActionItem> GetPipeline(const ResourcePtr &resource, const std::stri
std::string backend = MsContext::GetInstance()->backend_policy();
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
const std::string &server_mode = ps::PSContext::instance()->server_mode();
if ((server_mode == ps::kServerModeFL || server_mode == ps::kServerModeHybrid) &&
ps::PSContext::instance()->is_server()) {
@ -1319,7 +1319,7 @@ bool InitExecDataset(const std::string &queue_name, int64_t iter_num, int64_t ba
bool InitExecDatasetVm(const std::string &queue_name, int64_t size, int64_t batch_size,
const std::vector<TypePtr> &types, const std::vector<std::vector<int64_t>> &shapes,
const std::vector<int64_t> &input_indexes, bool need_run) {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if ((ps::PSContext::instance()->is_ps_mode()) && (!ps::PSContext::instance()->is_worker())) {
return true;
}
@ -1579,7 +1579,7 @@ void ClearResAtexit() {
device::DeviceContextManager::GetInstance().WaitTaskFinishOnDevice();
RecordExitStatus();
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::PSContext::instance()->is_ps_mode() && ps::PSContext::instance()->is_worker()) {
if (ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps::ps_cache_instance.Finalize();

4
mindspore/ccsrc/plugin/device/cpu/hal/hardware/cpu_device_context.cc
View File

@ -40,7 +40,7 @@
#include "backend/common/session/anf_runtime_algorithm.h"
#include "include/common/utils/anfalgo.h"
#include "profiler/device/cpu/cpu_profiling.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "plugin/device/cpu/hal/hardware/ms_collective_comm_lib.h"
#endif
#ifndef ENABLE_SECURITY
@ -414,7 +414,7 @@ bool CPUDeviceContext::LoadCollectiveCommLib() {
collective_comm_lib_ = instance_func();
MS_EXCEPTION_IF_NULL(collective_comm_lib_);
} else {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
collective_comm_lib_ = &MsCollectiveCommLib::GetInstance();
MS_EXCEPTION_IF_NULL(collective_comm_lib_);
#endif

12
mindspore/ccsrc/plugin/device/cpu/kernel/allreduce_cpu_kernel.cc
View File

@ -20,19 +20,24 @@
#include <functional>
#include <memory>
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "plugin/device/cpu/hal/hardware/ms_collective_comm_lib.h"
#endif
namespace mindspore {
namespace kernel {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
using device::CollectiveOpReduceType::Reduce_Sum;
using device::cpu::kMCCLGlobalGroupName;
using device::cpu::MsCollectiveCommLib;
#endif
namespace {
constexpr char kSupportedReduceOp[] = "sum";
} // namespace
void AllReduceCPUKernelMod::InitKernel(const CNodePtr &kernel_node) {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
MS_EXCEPTION_IF_NULL(kernel_node);
kernel_name_ = common::AnfAlgo::GetCNodeName(kernel_node);
auto kernel_attr = GetKernelAttrFromNode(kernel_node);
@ -48,6 +53,9 @@ void AllReduceCPUKernelMod::InitKernel(const CNodePtr &kernel_node) {
if (reduce_op != kSupportedReduceOp) {
MS_LOG(EXCEPTION) << kernel_name_ << " only support reduce sum on CPU, but got " << reduce_op;
}
#else
MS_LOG(EXCEPTION) << "The CPU kernel allreduce is only supported on linux platform.";
#endif
}
std::vector<KernelAttr> AllReduceCPUKernelMod::GetOpSupport() {
@ -59,6 +67,7 @@ std::vector<KernelAttr> AllReduceCPUKernelMod::GetOpSupport() {
bool AllReduceCPUKernelMod::Launch(const std::vector<kernel::AddressPtr> &inputs,
const std::vector<kernel::AddressPtr> &,
const std::vector<kernel::AddressPtr> &outputs) {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (inputs.empty() || outputs.empty()) {
MS_LOG(EXCEPTION) << kernel_name_ << " has at least one input and one output, but got 0.";
}
@ -72,6 +81,9 @@ bool AllReduceCPUKernelMod::Launch(const std::vector<kernel::AddressPtr> &inputs
MS_LOG(ERROR) << "AllReduceCPUKernelMod launch failed.";
}
return ret;
#else
MS_LOG(EXCEPTION) << "The CPU kernel allreduce is only supported on linux platform.";
#endif
}
MS_KERNEL_FACTORY_REG(NativeCpuKernelMod, AllReduce, AllReduceCPUKernelMod);

20
mindspore/ccsrc/ps/ps_context.cc
View File

@ -19,7 +19,7 @@
#include "kernel/kernel.h"
#include "utils/log_adapter.h"
#include "utils/ms_utils.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "distributed/cluster/cluster_context.h"
#include "ps/ps_cache/ps_cache_manager.h"
#include "ps/ps_cache/ps_data/ps_data_prefetch.h"
@ -95,7 +95,7 @@ void PSContext::Reset() {
is_worker_ = false;
is_pserver_ = false;
is_sched_ = false;
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps_cache_instance.Finalize();
set_cache_enable(false);
@ -160,51 +160,51 @@ uint32_t PSContext::ps_rank_id() const { return rank_id_; }
void PSContext::InsertHashTableSize(const std::string &param_name, size_t cache_vocab_size, size_t embedding_size,
size_t vocab_size) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
ps_cache_instance.InsertHashTableSize(param_name, cache_vocab_size, embedding_size, vocab_size);
#endif
}
void PSContext::ReInsertHashTableSize(const std::string &new_param_name, const std::string &cur_param_name,
size_t cache_vocab_size, size_t embedding_size) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
ps_cache_instance.ReInsertHashTableSize(new_param_name, cur_param_name, cache_vocab_size, embedding_size);
#endif
}
void PSContext::InsertWeightInitInfo(const std::string &param_name, size_t global_seed, size_t op_seed) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
ps_cache_instance.InsertWeightInitInfo(param_name, global_seed, op_seed);
#endif
}
void PSContext::InsertAccumuInitInfo(const std::string &param_name, float init_val) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
ps_cache_instance.InsertAccumuInitInfo(param_name, init_val);
#endif
}
void PSContext::CloneHashTable(const std::string &dest_param_name, const std::string &src_param_name) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
ps_cache_instance.CloneHashTable(dest_param_name, src_param_name);
#endif
}
void PSContext::set_cache_enable(bool cache_enable) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
PsDataPrefetch::GetInstance().set_cache_enable(cache_enable);
#endif
}
bool PSContext::cache_enable() const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32)) && !defined(__APPLE__)
return PsDataPrefetch::GetInstance().cache_enable();
#endif
return false;
}
void PSContext::set_rank_id(uint32_t rank_id) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
ps_cache_instance.set_rank_id(rank_id);
#endif
}

16
mindspore/ccsrc/runtime/graph_scheduler/graph_scheduler.cc
View File

@ -32,7 +32,7 @@
#include "include/common/utils/convert_utils.h"
#include "utils/ms_context.h"
#include "utils/profile.h"
#if !defined(_WIN32) && !defined(_WIN64)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__)
#include "include/common/utils/signal_util.h"
#endif
#ifndef ENABLE_SECURITY
@ -48,7 +48,7 @@
#include "include/common/debug/common.h"
#include "distributed/recovery/recovery_context.h"
#include "distributed/collective/collective_manager.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined _WIN64))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined _WIN64) && !defined(__APPLE__))
#include "distributed/cluster/cluster_context.h"
#else
#include "distributed/cluster/dummy_cluster_context.h"
@ -138,7 +138,7 @@ void ClearNodeInfo(const KernelGraphPtr &graph) {
}
}
#if !defined(_WIN32) && !defined(_WIN64)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__)
void IntHandler(int, siginfo_t *, void *) {
int this_pid = getpid();
MS_LOG(WARNING) << "Process " << this_pid << " receive KeyboardInterrupt signal.";
@ -146,7 +146,7 @@ void IntHandler(int, siginfo_t *, void *) {
}
#endif
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined _WIN64))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined _WIN64) && !defined(__APPLE__))
bool SendFinishTransform(const std::string &actor_set_name) {
auto node = ClusterContext::instance()->node();
MS_EXCEPTION_IF_NULL(node);
@ -470,7 +470,7 @@ ActorSet *GraphScheduler::Transform(const GraphCompilerInfo &graph_compiler_info
}
MS_LOG(INFO) << "Graph(" << graph_compiler_info.name_ << ") transforms actor end.";
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined _WIN64))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined _WIN64) && !defined(__APPLE__))
if (ClusterContext::instance()->initialized() && RecoveryContext::GetInstance()->enable_recovery()) {
while (!SendFinishTransform(graph_compiler_info.name_)) {
MS_LOG(WARNING) << "Send finish transform graph failed.";
@ -506,7 +506,7 @@ void GraphScheduler::Run(ActorSet *const actor_set, const std::vector<DeviceCont
const std::vector<TensorPtr> &input_tensors_with_value_node, GraphExecutionStrategy strategy) {
MS_EXCEPTION_IF_NULL(actor_set);
MS_EXCEPTION_IF_NULL(actor_set->data_prepare_actor_);
#if !defined(_WIN32) && !defined(_WIN64)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__)
SignalGuard sg(IntHandler);
#endif
@ -568,7 +568,7 @@ void GraphScheduler::Run(ActorSet *const actor_set, const std::vector<DeviceCont
const size_t kSecondsToMilliseconds = 1000;
SetActorExecutionStrategy(actor_set, strategy, (end_time - start_time) * kSecondsToMilliseconds);
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined _WIN64))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined _WIN64) && !defined(__APPLE__))
DoDisasterRecovery(actor_set->name_);
#endif
}
@ -580,7 +580,7 @@ void GraphScheduler::SetActorExecutionStrategy(ActorSet *const actor_set, GraphE
++actor_set->execution_count_;
MS_LOG(DEBUG) << "Execution count: " << actor_set->execution_count_ << ", execution time cost: " << execution_time
<< " ms in multi thread or not: " << actor_set->is_multi_thread_execution_ << ".";
#if defined(_WIN32) || defined(_WIN64)
#if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__)
return;
#endif