sfqcyy
/
mindspore
forked from OSSInnovation/mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Use white list to control pslite compilation

This commit is contained in:
ZPaC 2020-07-21 16:37:24 +08:00
parent 37cc6e2628
commit 8ff1a9ab47
12 changed files with 25 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/mind_expression.cmake
View File

@ -30,7 +30,7 @@ include(${CMAKE_SOURCE_DIR}/cmake/external_libs/flatbuffers.cmake)
if(USE_GLOG) if(USE_GLOG)
include(${CMAKE_SOURCE_DIR}/cmake/external_libs/glog.cmake) include(${CMAKE_SOURCE_DIR}/cmake/external_libs/glog.cmake)
endif() endif()
if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows" AND NOT ENABLE_GE) if (ENABLE_CPU AND (ENABLE_D OR ENABLE_GPU))
include(${CMAKE_SOURCE_DIR}/cmake/external_libs/zeromq.cmake) include(${CMAKE_SOURCE_DIR}/cmake/external_libs/zeromq.cmake)
include(${CMAKE_SOURCE_DIR}/cmake/external_libs/pslite.cmake) include(${CMAKE_SOURCE_DIR}/cmake/external_libs/pslite.cmake)
endif() endif()

8
cmake/options.cmake
View File

@ -70,6 +70,10 @@ if (ENABLE_GPU)
add_compile_definitions(ENABLE_GPU_COLLECTIVE) add_compile_definitions(ENABLE_GPU_COLLECTIVE)
endif() endif()
if (ENABLE_CPU)
add_compile_definitions(ENABLE_CPU)
endif()
if (ENABLE_GE) if (ENABLE_GE)
add_compile_definitions(ENABLE_GE) add_compile_definitions(ENABLE_GE)
add_compile_definitions(CUSTOM_OP) add_compile_definitions(CUSTOM_OP)
@ -119,7 +123,3 @@ endif()
if(ENABLE_DEBUGGER) if(ENABLE_DEBUGGER)
add_compile_definitions(ENABLE_DEBUGGER) add_compile_definitions(ENABLE_DEBUGGER)
endif() endif()
if(ENABLE_TESTCASES)
add_compile_definitions(ENABLE_TESTCASES)
endif()

2
mindspore/ccsrc/CMakeLists.txt
View File

@ -252,7 +252,7 @@ if (CMAKE_SYSTEM_NAME MATCHES "Windows")
target_link_libraries(mindspore mindspore_gvar) target_link_libraries(mindspore mindspore_gvar)
target_link_libraries(_c_expression PRIVATE -Wl,--whole-archive mindspore -Wl,--no-whole-archive) target_link_libraries(_c_expression PRIVATE -Wl,--whole-archive mindspore -Wl,--no-whole-archive)
else () else ()
if (NOT ENABLE_GE) if (ENABLE_CPU AND (ENABLE_D OR ENABLE_GPU))
target_link_libraries(mindspore mindspore::pslite mindspore::protobuf ${zeromq_DIRPATH}/zmq_install/lib/libzmq.a) target_link_libraries(mindspore mindspore::pslite mindspore::protobuf ${zeromq_DIRPATH}/zmq_install/lib/libzmq.a)
if (${ENABLE_IBVERBS} STREQUAL "ON") if (${ENABLE_IBVERBS} STREQUAL "ON")
target_link_libraries(mindspore ibverbs rdmacm) target_link_libraries(mindspore ibverbs rdmacm)

2
mindspore/ccsrc/backend/kernel_compiler/CMakeLists.txt
View File

@ -33,7 +33,7 @@ if (ENABLE_CPU)
endif () endif ()
endif () endif ()
if (${CMAKE_SYSTEM_NAME} MATCHES "Windows" OR ENABLE_GE) if (NOT (ENABLE_CPU AND (ENABLE_D OR ENABLE_GPU)))
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/apply_momentum_ps_kernel.cc") list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/apply_momentum_ps_kernel.cc")
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/embedding_look_up_proxy_kernel.cc") list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/embedding_look_up_proxy_kernel.cc")
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/embedding_look_up_ps_kernel.cc") list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/embedding_look_up_ps_kernel.cc")

4
mindspore/ccsrc/backend/session/ascend_session.cc
View File

@ -353,7 +353,7 @@ GraphId AscendSession::CompileGraph(NotNull<FuncGraphPtr> func_graph) {
RootGraphExecutorValidate(NOT_NULL(root_graph)); RootGraphExecutorValidate(NOT_NULL(root_graph));
// adjust kernel // adjust kernel
AdjustKernel(root_graph); AdjustKernel(root_graph);
#if (!_WIN32 && !ENABLE_GE && !ENABLE_TESTCASES) #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
// Assign parameter keys. // Assign parameter keys.
AssignParamKey(root_graph); AssignParamKey(root_graph);
#endif #endif
@ -515,7 +515,7 @@ void AscendSession::RunGraph(const GraphId &graph_id, const std::vector<tensor::
} }
// load input data from user input // load input data from user input
LoadInputData(kernel_graph, inputs); LoadInputData(kernel_graph, inputs);
#if (!_WIN32 && !ENABLE_GE && !ENABLE_TESTCASES) #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
// Initialize parameter server // Initialize parameter server
if (!ps_init_) { if (!ps_init_) {
InitPSParamAndOptim(kernel_graph, inputs); InitPSParamAndOptim(kernel_graph, inputs);

6
mindspore/ccsrc/backend/session/cpu_session.cc
View File

@ -32,7 +32,7 @@
#ifdef ENABLE_DEBUGGER #ifdef ENABLE_DEBUGGER
#include "debug/debugger/debugger.h" #include "debug/debugger/debugger.h"
#endif #endif
#if (!_WIN32 && !ENABLE_GE && !ENABLE_TESTCASES) #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
#include "frontend/parallel/ps/util.h" #include "frontend/parallel/ps/util.h"
#endif #endif
@ -73,7 +73,7 @@ GraphId CPUSession::CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrList
MS_EXCEPTION_IF_NULL(graph); MS_EXCEPTION_IF_NULL(graph);
MS_LOG(INFO) << "Set kernel info"; MS_LOG(INFO) << "Set kernel info";
SetKernelInfo(graph.get()); SetKernelInfo(graph.get());
#if (!_WIN32 && !ENABLE_GE && !ENABLE_TESTCASES) #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
AssignParamKey(graph); AssignParamKey(graph);
if (parallel::ps::Util::IsRoleOfWorker()) { if (parallel::ps::Util::IsRoleOfWorker()) {
Optimize(graph); Optimize(graph);
@ -90,7 +90,7 @@ GraphId CPUSession::CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrList
void CPUSession::RunGraph(const GraphId &graph_id, const std::vector<tensor::TensorPtr> &inputs, VectorRef *outputs) { void CPUSession::RunGraph(const GraphId &graph_id, const std::vector<tensor::TensorPtr> &inputs, VectorRef *outputs) {
auto &kernel_graph = graphs_[graph_id]; auto &kernel_graph = graphs_[graph_id];
MS_EXCEPTION_IF_NULL(kernel_graph); MS_EXCEPTION_IF_NULL(kernel_graph);
#if (!_WIN32 && !ENABLE_GE && !ENABLE_TESTCASES) #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
// Initialize parameter server // Initialize parameter server
if (!ps_init_) { if (!ps_init_) {
InitPSParamAndOptim(kernel_graph, inputs); InitPSParamAndOptim(kernel_graph, inputs);

4
mindspore/ccsrc/backend/session/gpu_session.cc
View File

@ -182,7 +182,7 @@ GraphId GPUSession::CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrList
Optimize(graph); Optimize(graph);
// Select kernel build info // Select kernel build info
SelectKernel(graph); SelectKernel(graph);
#if (!_WIN32 && !ENABLE_GE && !ENABLE_TESTCASES) #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
// Assign parameter keys. // Assign parameter keys.
AssignParamKey(graph); AssignParamKey(graph);
#endif #endif
@ -231,10 +231,12 @@ void GPUSession::RunGraph(const GraphId &graph_id, const std::vector<tensor::Ten
auto &kernel_graph = graphs_[graph_id]; auto &kernel_graph = graphs_[graph_id];
// Load input data from user input // Load input data from user input
LoadInputData(kernel_graph, inputs); LoadInputData(kernel_graph, inputs);
#if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
// Initialize parameter server // Initialize parameter server
if (!ps_init_) { if (!ps_init_) {
InitPSParamAndOptim(kernel_graph, inputs); InitPSParamAndOptim(kernel_graph, inputs);
} }
#endif
MS_EXCEPTION_IF_NULL(kernel_graph); MS_EXCEPTION_IF_NULL(kernel_graph);
// Convert inputs to model // Convert inputs to model
predictmodel::StepConvertWeight(inputs); predictmodel::StepConvertWeight(inputs);

4
mindspore/ccsrc/backend/session/session_basic.cc
View File

@ -35,7 +35,7 @@
#include "ir/dtype.h" #include "ir/dtype.h"
#include "ir/anf.h" #include "ir/anf.h"
#include "ir/func_graph_cloner.h" #include "ir/func_graph_cloner.h"
#if (!_WIN32 && !ENABLE_GE && !ENABLE_TESTCASES) #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
#include "frontend/parallel/ps/worker.h" #include "frontend/parallel/ps/worker.h"
#include "frontend/parallel/ps/common.h" #include "frontend/parallel/ps/common.h"
#include "frontend/parallel/ps/util.h" #include "frontend/parallel/ps/util.h"
@ -1141,7 +1141,7 @@ AnfNodePtr SessionBasic::FindPullNode(const AnfNodePtr &push_node, const std::ve
return nullptr; return nullptr;
} }
#if (!_WIN32 && !ENABLE_GE && !ENABLE_TESTCASES) #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
void SessionBasic::AssignParamKey(const KernelGraphPtr &kernel_graph) { void SessionBasic::AssignParamKey(const KernelGraphPtr &kernel_graph) {
if (!parallel::ps::Util::IsRoleOfWorker()) { if (!parallel::ps::Util::IsRoleOfWorker()) {
MS_LOG(INFO) << "Not parameter server mode."; MS_LOG(INFO) << "Not parameter server mode.";

2
mindspore/ccsrc/frontend/parallel/CMakeLists.txt
View File

@ -1,6 +1,6 @@
file(GLOB_RECURSE _PARALLEL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") file(GLOB_RECURSE _PARALLEL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
if (${CMAKE_SYSTEM_NAME} MATCHES "Windows" OR ENABLE_GE) if (NOT (ENABLE_CPU AND (ENABLE_D OR ENABLE_GPU)))
list(REMOVE_ITEM _PARALLEL_SRC_FILES "ps/optimizer_info_builder.cc") list(REMOVE_ITEM _PARALLEL_SRC_FILES "ps/optimizer_info_builder.cc")
list(REMOVE_ITEM _PARALLEL_SRC_FILES "ps/optimizer_info.cc") list(REMOVE_ITEM _PARALLEL_SRC_FILES "ps/optimizer_info.cc")
list(REMOVE_ITEM _PARALLEL_SRC_FILES "ps/scheduler.cc") list(REMOVE_ITEM _PARALLEL_SRC_FILES "ps/scheduler.cc")

2
mindspore/ccsrc/minddata/dataset/CMakeLists.txt
View File

@ -146,7 +146,7 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord ${MINDRECORD_LINK_OBJECT} mindspore::sqlite) target_link_libraries(_c_dataengine PRIVATE _c_mindrecord ${MINDRECORD_LINK_OBJECT} mindspore::sqlite)
else() else()
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord) target_link_libraries(_c_dataengine PRIVATE _c_mindrecord)
if (NOT ENABLE_GE) if (ENABLE_CPU AND (ENABLE_D OR ENABLE_GPU))
target_link_libraries(_c_dataengine PRIVATE mindspore::pslite mindspore::protobuf ${zeromq_DIRPATH}/zmq_install/lib/libzmq.a) target_link_libraries(_c_dataengine PRIVATE mindspore::pslite mindspore::protobuf ${zeromq_DIRPATH}/zmq_install/lib/libzmq.a)
if (${ENABLE_IBVERBS} STREQUAL "ON") if (${ENABLE_IBVERBS} STREQUAL "ON")
target_link_libraries(_c_dataengine PRIVATE ibverbs rdmacm) target_link_libraries(_c_dataengine PRIVATE ibverbs rdmacm)

8
mindspore/ccsrc/pipeline/jit/action.cc
View File

@ -40,7 +40,7 @@
#include "vm/transform.h" #include "vm/transform.h"
#include "parse/python_adapter.h" #include "parse/python_adapter.h"
#include "frontend/optimizer/py_pass_manager.h" #include "frontend/optimizer/py_pass_manager.h"
#if (!_WIN32 && !ENABLE_GE && !ENABLE_TESTCASES) #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
#include "frontend/parallel/ps/parameter_server.h" #include "frontend/parallel/ps/parameter_server.h"
#include "frontend/parallel/ps/scheduler.h" #include "frontend/parallel/ps/scheduler.h"
#include "frontend/parallel/ps/worker.h" #include "frontend/parallel/ps/worker.h"
@ -379,7 +379,7 @@ bool ExecuteAction(const ResourcePtr &res) {
return true; return true;
} }
#if (!_WIN32 && !ENABLE_GE && !ENABLE_TESTCASES) #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
bool StartPSWorkerAction(const ResourcePtr &res) { bool StartPSWorkerAction(const ResourcePtr &res) {
parallel::ps::Worker<float>::GetInstance().Run(); parallel::ps::Worker<float>::GetInstance().Run();
return true; return true;
@ -505,7 +505,7 @@ std::vector<ActionItem> VmPipeline() {
actions.emplace_back(std::make_pair("py_opt", OptActionPyStub)); actions.emplace_back(std::make_pair("py_opt", OptActionPyStub));
actions.emplace_back(std::make_pair("validate", ValidateAction)); actions.emplace_back(std::make_pair("validate", ValidateAction));
#if (!_WIN32 && !ENABLE_GE && !ENABLE_TESTCASES) #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
if (parallel::ps::Util::IsRoleOfWorker()) { if (parallel::ps::Util::IsRoleOfWorker()) {
actions.emplace_back(std::make_pair("worker", StartPSWorkerAction)); actions.emplace_back(std::make_pair("worker", StartPSWorkerAction));
} }
@ -519,7 +519,7 @@ std::vector<ActionItem> VmPipeline() {
return actions; return actions;
} }
#if (!_WIN32 && !ENABLE_GE && !ENABLE_TESTCASES) #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
std::vector<ActionItem> PServerPipeline() { std::vector<ActionItem> PServerPipeline() {
auto actions = CommonPipeline(); auto actions = CommonPipeline();
actions.emplace_back(std::make_pair("optimize", VmOptimizeAction)); actions.emplace_back(std::make_pair("optimize", VmOptimizeAction));

4
mindspore/ccsrc/pipeline/jit/pipeline.cc
View File

@ -42,7 +42,7 @@
#include "frontend/optimizer/py_pass_manager.h" #include "frontend/optimizer/py_pass_manager.h"
#include "pybind_api/pybind_patch.h" #include "pybind_api/pybind_patch.h"
#if (!_WIN32 && !ENABLE_GE && !ENABLE_TESTCASES) #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
#include "frontend/parallel/ps/common.h" #include "frontend/parallel/ps/common.h"
#include "frontend/parallel/ps/util.h" #include "frontend/parallel/ps/util.h"
#endif #endif
@ -403,7 +403,7 @@ std::vector<ActionItem> GetPipline(const ResourcePtr &resource, const std::strin
std::string backend = MsContext::GetInstance()->backend_policy(); std::string backend = MsContext::GetInstance()->backend_policy();
#if (!_WIN32 && !ENABLE_GE && !ENABLE_TESTCASES) #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
if (mindspore::parallel::ps::Util::IsParamServerMode()) { if (mindspore::parallel::ps::Util::IsParamServerMode()) {
mindspore::parallel::ps::Util::SetInternalEnvVar(); mindspore::parallel::ps::Util::SetInternalEnvVar();
} }