p12346870
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

mac compile

This commit is contained in:
xiefangqi 2021-10-25 16:59:19 +08:00 committed by xulei
parent 888a104b0a
commit 24ac2382ff
68 changed files with 581 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -108,6 +108,8 @@ if(MODE_ASCEND_ACL)
include(cmake/package_tar.cmake)
elseif(CMAKE_SYSTEM_NAME MATCHES "Windows")
include(cmake/package_win.cmake)
elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
include(cmake/package_mac.cmake)
else()
include(cmake/package.cmake)
endif()

6
cmake/external_libs/absl.cmake
View File

@ -15,7 +15,11 @@ mindspore_add_pkg(absl
LIBS absl_strings absl_throw_delegate absl_raw_logging_internal absl_int128 absl_bad_optional_access
URL ${REQ_URL}
MD5 ${MD5}
CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=TRUE)
CMAKE_OPTION
-DCMAKE_BUILD_TYPE:STRING=Release
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=TRUE
-DCMAKE_CXX_STANDARD=11
)
include_directories(${absl_INC})

3
cmake/external_libs/flatbuffers.cmake
View File

@ -19,6 +19,9 @@ else()
set(MD5 "a27992324c3cbf86dd888268a23d17bd")
endif()
if(APPLE)
set(flatbuffers_CXXFLAGS "${flatbuffers_CXXFLAGS} -Wno-deprecated")
endif()
if(APPLE)
mindspore_add_pkg(flatbuffers
VER 2.0.0

25
cmake/external_libs/grpc.cmake
View File

@ -1,41 +1,44 @@
set(grpc_USE_STATIC_LIBS OFF)
if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(grpc_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
set(grpc_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2 \
-Dgrpc=mindspore_grpc -Dgrpc_impl=mindspore_grpc_impl -Dgrpc_core=mindspore_grpc_core")
elseif(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
else()
set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2 \
-Dgrpc=mindspore_grpc -Dgrpc_impl=mindspore_grpc_impl -Dgrpc_core=mindspore_grpc_core")
-Dgrpc=mindspore_grpc -Dgrpc_impl=mindspore_grpc_impl -Dgrpc_core=mindspore_grpc_core")
if(NOT ENABLE_GLIBCXX)
set(grpc_CXXFLAGS "${grpc_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
endif()
endif()
set(grpc_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(grpc_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
endif()
if(EXISTS ${protobuf_ROOT}/lib64)
set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib64/cmake/protobuf")
set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib64/cmake/protobuf")
else()
set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib/cmake/protobuf")
set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib/cmake/protobuf")
endif()
message("grpc using Protobuf_DIR : " ${_FINDPACKAGE_PROTOBUF_CONFIG_DIR})
if(EXISTS ${absl_ROOT}/lib64)
set(_FINDPACKAGE_ABSL_CONFIG_DIR "${absl_ROOT}/lib64/cmake/absl")
set(_FINDPACKAGE_ABSL_CONFIG_DIR "${absl_ROOT}/lib64/cmake/absl")
else()
set(_FINDPACKAGE_ABSL_CONFIG_DIR "${absl_ROOT}/lib/cmake/absl")
set(_FINDPACKAGE_ABSL_CONFIG_DIR "${absl_ROOT}/lib/cmake/absl")
endif()
message("grpc using absl_DIR : " ${_FINDPACKAGE_ABSL_CONFIG_DIR})
if(EXISTS ${re2_ROOT}/lib64)
set(_FINDPACKAGE_RE2_CONFIG_DIR "${re2_ROOT}/lib64/cmake/re2")
set(_FINDPACKAGE_RE2_CONFIG_DIR "${re2_ROOT}/lib64/cmake/re2")
else()
set(_FINDPACKAGE_RE2_CONFIG_DIR "${re2_ROOT}/lib/cmake/re2")
set(_FINDPACKAGE_RE2_CONFIG_DIR "${re2_ROOT}/lib/cmake/re2")
endif()
message("grpc using re2_DIR : " ${_FINDPACKAGE_RE2_CONFIG_DIR})
if(EXISTS ${openssl_ROOT})
set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${openssl_ROOT}")
set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${openssl_ROOT}")
endif()
if(ENABLE_GITEE)
@ -78,7 +81,7 @@ add_library(mindspore::grpc++ ALIAS grpc::mindspore_grpc++)
# link other grpc libs
target_link_libraries(grpc::mindspore_grpc++ INTERFACE grpc::mindspore_grpc grpc::mindspore_gpr grpc::mindspore_upb
grpc::mindspore_address_sorting)
grpc::mindspore_address_sorting)
# modify mindspore macro define
add_compile_definitions(grpc=mindspore_grpc)

2
cmake/external_libs/openssl.cmake
View File

@ -12,7 +12,7 @@ else()
set(OPENSSL_PATCH_ROOT ${CMAKE_SOURCE_DIR}/third_party/patch/openssl)
endif()
if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
if(${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR APPLE)
mindspore_add_pkg(openssl
VER 1.1.1k
LIBS ssl crypto

2
cmake/external_libs/sentencepiece.cmake
View File

@ -27,7 +27,7 @@ else()
LIBS sentencepiece sentencepiece_train
URL ${REQ_URL}
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DSPM_USE_BUILTIN_PROTOBUF=OFF -DSPM_ENABLE_SHARED=OFF
-DPROTOBUF_INC=${protobuf_INC}
-DPROTOBUF_INC=${protobuf_INC} -DCMAKE_CXX_STANDARD=11
MD5 ${MD5}
PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/sentencepiece/sentencepiece.patch001_cpu
)

9
cmake/options.cmake
View File

@ -28,6 +28,11 @@ option(MODE_ASCEND_ACL "supports ascend acl mode only" OFF)
option(ENABLE_SYM_FILE "enable sym file" OFF)
option(BUILD_DEV_MODE "MindSpore build nightly dev mode" OFF)
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(ENABLE_DEBUGGER OFF)
endif()
if(NOT ENABLE_D AND NOT ENABLE_TESTCASES AND NOT ENABLE_ACL AND NOT ENABLE_GE)
set(ENABLE_GLIBCXX ON)
endif()
@ -137,7 +142,7 @@ endif()
if(ENABLE_DEBUGGER OR ENABLE_TESTCASES)
set(MS_BUILD_GRPC ON)
endif()
if(ENABLE_MINDDATA AND NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
if(ENABLE_MINDDATA AND NOT CMAKE_SYSTEM_NAME MATCHES "Windows" AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(MS_BUILD_GRPC ON)
endif()
@ -149,7 +154,7 @@ if(ENABLE_ACL AND NOT ENABLE_D)
set(MODE_ASCEND_ACL ON)
endif()
if(ENABLE_CPU AND NOT WIN32)
if(ENABLE_CPU AND NOT WIN32 AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
add_compile_definitions(ENABLE_ARMOUR)
endif()

247
cmake/package_mac.cmake Normal file
View File

@ -0,0 +1,247 @@
# include dependency
include(CMakePackageConfigHelpers)
include(GNUInstallDirs)
# set package information
set(CPACK_PACKAGE_NAME ${PROJECT_NAME})
set(CPACK_GENERATOR "External")
set(CPACK_CMAKE_GENERATOR "Ninja")
set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/package_script.cmake)
set(CPACK_EXTERNAL_ENABLE_STAGING true)
set(CPACK_TEMPORARY_PACKAGE_FILE_NAME ${BUILD_PATH}/package/mindspore)
set(CPACK_TEMPORARY_INSTALL_DIRECTORY ${BUILD_PATH}/package/mindspore)
set(CPACK_PACK_ROOT_DIR ${BUILD_PATH}/package/)
set(CPACK_CMAKE_SOURCE_DIR ${CMAKE_SOURCE_DIR})
set(CPACK_ENABLE_SYM_FILE ${ENABLE_SYM_FILE})
set(CPACK_CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE})
set(CPACK_PYTHON_EXE ${Python3_EXECUTABLE})
set(CPACK_PYTHON_VERSION ${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR})
if(ENABLE_GPU)
set(CPACK_MS_BACKEND "ms")
set(CPACK_MS_TARGET "gpu or cpu")
set(CPACK_MS_PACKAGE_NAME "mindspore-gpu")
elseif(ENABLE_CPU)
set(CPACK_MS_BACKEND "ms")
set(CPACK_MS_TARGET "cpu")
set(CPACK_MS_PACKAGE_NAME "mindspore")
else()
set(CPACK_MS_BACKEND "debug")
set(CPACK_MS_TARGET "ascend or gpu or cpu")
set(CPACK_MS_PACKAGE_NAME "mindspore")
endif()
include(CPack)
# set install path
set(INSTALL_LIB_DIR ${CMAKE_INSTALL_LIBDIR} CACHE PATH "Installation directory for libraries")
set(INSTALL_PY_DIR ".")
set(INSTALL_BASE_DIR ".")
set(INSTALL_BIN_DIR "bin")
set(INSTALL_CFG_DIR "config")
set(INSTALL_LIB_DIR "lib")
message("offline debugger does not support macosx system temporarily")
# set package files
install(
TARGETS _c_expression
DESTINATION ${INSTALL_BASE_DIR}
COMPONENT mindspore
)
install(
TARGETS mindspore_shared_lib
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
install(
TARGETS mindspore_gvar
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
if(USE_GLOG)
file(GLOB_RECURSE GLOG_LIB_LIST ${glog_LIBPATH}/libmindspore_glog*)
install(
FILES ${GLOG_LIB_LIST}
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
endif()
if(MS_BUILD_GRPC)
file(GLOB_RECURSE GRPC_LIB_LIST
${grpc_LIBPATH}/libmindspore*
)
install(
FILES ${GRPC_LIB_LIST}
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
endif()
if(ENABLE_MINDDATA)
install(
TARGETS _c_dataengine _c_mindrecord
DESTINATION ${INSTALL_BASE_DIR}
COMPONENT mindspore
)
file(GLOB_RECURSE OPENCV_LIB_LIST
${opencv_LIBPATH}/libopencv_core*
${opencv_LIBPATH}/libopencv_imgcodecs*
${opencv_LIBPATH}/libopencv_imgproc*
)
install(
FILES ${OPENCV_LIB_LIST}
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
file(GLOB_RECURSE JPEG_LIB_LIST
${jpeg_turbo_LIBPATH}/*.dylib
)
install(
FILES ${JPEG_LIB_LIST}
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
file(GLOB_RECURSE TINYXML2_LIB_LIST ${tinyxml2_LIBPATH}/libtinyxml2*)
install(
FILES ${TINYXML2_LIB_LIST}
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
file(GLOB_RECURSE ICU4C_LIB_LIST
${icu4c_LIBPATH}/libicuuc*
${icu4c_LIBPATH}/libicudata*
${icu4c_LIBPATH}/libicui18n*
)
install(
FILES ${ICU4C_LIB_LIST}
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
endif()
if(ENABLE_CPU)
file(GLOB_RECURSE DNNL_LIB_LIST ${onednn_LIBPATH}/libdnnl*)
install(
FILES ${DNNL_LIB_LIST}
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
install(
TARGETS nnacl
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
endif()
if(ENABLE_MPI)
if(ENABLE_GPU)
install(
TARGETS _ms_mpi
DESTINATION ${INSTALL_BASE_DIR}
COMPONENT mindspore
)
endif()
if(ENABLE_CPU)
install(
TARGETS mpi_adapter
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
endif()
endif()
if(ENABLE_GPU)
if(ENABLE_MPI)
install(
TARGETS gpu_collective
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
endif()
install(
TARGETS gpu_queue
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
endif()
get_filename_component(CXX_DIR ${CMAKE_CXX_COMPILER} PATH)
file(GLOB CXX_LIB_LIST ${CXX_DIR}/*.dylib)
file(GLOB JPEG_LIB_LIST ${jpeg_turbo_LIBPATH}/*.dylib)
file(GLOB SQLITE_LIB_LIST ${sqlite_LIBPATH}/*.dylib)
install(
FILES ${CXX_LIB_LIST} ${SQLITE_LIB_LIST}
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
# set python files
file(GLOB MS_PY_LIST ${CMAKE_SOURCE_DIR}/mindspore/*.py)
install(
FILES ${MS_PY_LIST}
DESTINATION ${INSTALL_PY_DIR}
COMPONENT mindspore
)
install(
DIRECTORY
${CMAKE_SOURCE_DIR}/mindspore/nn
${CMAKE_SOURCE_DIR}/mindspore/_extends
${CMAKE_SOURCE_DIR}/mindspore/parallel
${CMAKE_SOURCE_DIR}/mindspore/mindrecord
${CMAKE_SOURCE_DIR}/mindspore/numpy
${CMAKE_SOURCE_DIR}/mindspore/scipy
${CMAKE_SOURCE_DIR}/mindspore/train
${CMAKE_SOURCE_DIR}/mindspore/boost
${CMAKE_SOURCE_DIR}/mindspore/common
${CMAKE_SOURCE_DIR}/mindspore/ops
${CMAKE_SOURCE_DIR}/mindspore/communication
${CMAKE_SOURCE_DIR}/mindspore/profiler
${CMAKE_SOURCE_DIR}/mindspore/compression
${CMAKE_SOURCE_DIR}/mindspore/run_check
DESTINATION ${INSTALL_PY_DIR}
COMPONENT mindspore
)
if(EXISTS ${CMAKE_SOURCE_DIR}/mindspore/dataset)
install(
DIRECTORY ${CMAKE_SOURCE_DIR}/mindspore/dataset
DESTINATION ${INSTALL_PY_DIR}
COMPONENT mindspore
)
endif()
## Public header files
install(
DIRECTORY ${CMAKE_SOURCE_DIR}/include
DESTINATION ${INSTALL_BASE_DIR}
COMPONENT mindspore
)
## Public header files for minddata
install(
FILES ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/config.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/constants.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/execute.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/text.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/transforms.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision_lite.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision_ascend.h
DESTINATION ${INSTALL_BASE_DIR}/include/dataset
COMPONENT mindspore
)
## config files
install(
FILES ${CMAKE_SOURCE_DIR}/config/op_info.config
DESTINATION ${INSTALL_CFG_DIR}
COMPONENT mindspore
)

17
mindspore/ccsrc/CMakeLists.txt
View File

@ -17,6 +17,8 @@ if(ENABLE_CPU)
if(${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "aarch64")
set(PLATFORM_ARM64 "on")
set(X86_64_SIMD "off")
elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(X86_64_SIMD "off")
elseif("${X86_64_SIMD}" STREQUAL "off" AND NOT ${ENABLE_ASAN})
set(X86_64_SIMD "avx")
endif()
@ -24,10 +26,13 @@ if(ENABLE_CPU)
add_subdirectory(backend/kernel_compiler/cpu/nnacl)
endif()
if(NOT(CMAKE_SYSTEM_NAME MATCHES "Darwin"))
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
link_directories(${CMAKE_SOURCE_DIR}/build/mindspore/graphengine)
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 \
-Wno-delete-non-abstract-non-virtual-dtor -Wno-unused-private-field -Wno-overloaded-virtual \
-Wno-unused-const-variable -Wno-pessimizing-move -Wno-range-loop-analysis -Wno-mismatched-tags \
-Wno-c++11-narrowing")
endif()
if(CMAKE_SYSTEM_NAME MATCHES "Windows")
@ -240,7 +245,11 @@ target_link_libraries(mindspore mindspore_core)
if(ENABLE_DEBUGGER)
# debugger: link grpc
target_link_libraries(mindspore -Wl,--no-as-needed mindspore::grpc++)
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(mindspore mindspore::grpc++)
else()
target_link_libraries(mindspore -Wl,--no-as-needed mindspore::grpc++)
endif()
endif()
target_link_libraries(mindspore securec mindspore::flatbuffers)
@ -351,7 +360,7 @@ if(CMAKE_SYSTEM_NAME MATCHES "Windows")
elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(mindspore mindspore::pybind11_module)
target_link_libraries(mindspore mindspore_gvar)
target_link_libraries(_c_expression PRIVATE -Wl,-force_load mindspore mindspore_core -Wl,-noall_load)
target_link_libraries(_c_expression PRIVATE -Wl,-all_load mindspore mindspore_core -Wl,-noall_load)
else()
if(ENABLE_CPU AND NOT WIN32)
target_link_libraries(mindspore proto_input mindspore::protobuf

4
mindspore/ccsrc/backend/kernel_compiler/CMakeLists.txt
View File

@ -6,7 +6,7 @@ file(GLOB_RECURSE KERNEL_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
)
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor -Wno-overloaded-virtual")
endif()
if(ENABLE_D)
@ -44,7 +44,7 @@ if(ENABLE_CPU)
endif()
endif()
if(NOT ENABLE_CPU OR WIN32)
if(NOT ENABLE_CPU OR WIN32 OR APPLE)
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/apply_momentum_ps_kernel.cc")
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/embedding_look_up_proxy_kernel.cc")
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/embedding_look_up_ps_kernel.cc")

2
mindspore/ccsrc/backend/kernel_compiler/cpu/cpu_kernel_factory.cc
View File

@ -37,7 +37,7 @@ CPUKernelFactory &CPUKernelFactory::GetInstance() {
void CPUKernelFactory::Register(const std::string &kernel_name, const KernelAttr &kernel_attr,
CPUKernelCreator &&kernel_creator) {
(void)name_to_attr_creator_[kernel_name].emplace_back(kernel_attr, kernel_creator);
#if !defined(_WIN32) && !defined(_WIN64)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__)
MS_LOG(DEBUG) << "CPUKernelFactory register operator: " << kernel_name;
#endif
}

4
mindspore/ccsrc/backend/kernel_compiler/cpu/eigen/qr_cpu_kernel.cc
View File

@ -77,10 +77,10 @@ bool QRCPUKernel<T>::Launch(const std::vector<kernel::AddressPtr> &inputs, const
auto householder_qr = input_a.householderQr();
if (economic_) {
// r_row_ = std::min(a_row_, a_col_)
output_r = Eigen::MatrixXd::Identity(r_row_, a_row_).cast<T>() *
output_r = Eigen::MatrixXd::Identity(r_row_, a_row_).template cast<T>() *
householder_qr.matrixQR().template triangularView<Eigen::Upper>();
// q_col_ = std::min(a_row_, a_col_)
output_q = householder_qr.householderQ() * Eigen::MatrixXd::Identity(q_row_, q_col_).cast<T>();
output_q = householder_qr.householderQ() * Eigen::MatrixXd::Identity(q_row_, q_col_).template cast<T>();
} else {
output_r = householder_qr.matrixQR().template triangularView<Eigen::Upper>();
output_q = householder_qr.householderQ();

4
mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/CMakeLists.txt
View File

@ -136,7 +136,9 @@ if(ENABLE_CPU)
target_compile_options(nnacl_mid PRIVATE -fPIC)
add_library(nnacl SHARED $<TARGET_OBJECTS:nnacl_mid>)
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
target_link_options(nnacl PRIVATE -Wl,-z,relro,-z,now,-z,noexecstack)
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_options(nnacl PRIVATE -Wl,-z,relro,-z,now,-z,noexecstack)
endif()
if("${CMAKE_BUILD_TYPE}" STREQUAL "Release")
target_link_options(nnacl PRIVATE -s)
endif()

4
mindspore/ccsrc/backend/kernel_compiler/cpu/rl/buffer_sample_cpu_kernel.h
View File

@ -81,7 +81,11 @@ class BufferCPUSampleKernel : public CPUKernel {
for (size_t i = 0; i < IntToSize(count_addr[0]); ++i) {
(void)indexes.emplace_back(i);
}
#if !defined(__APPLE__)
random_shuffle(indexes.begin(), indexes.end(), [&](int i) { return std::rand() % i; });
#else
std::shuffle(indexes.begin(), indexes.end(), generator_);
#endif
} else {
std::uniform_int_distribution<> distrib(0, count_addr[0]);
for (size_t i = 0; i < batch_size_; ++i) {

2
mindspore/ccsrc/backend/optimizer/CMakeLists.txt
View File

@ -30,7 +30,7 @@ if(ENABLE_GPU_INFER)
endif()
if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -Wno-user-defined-warnings -Wno-inconsistent-missing-override
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -Wno-user-defined-warnings -Wno-inconsistent-missing-override \
-Wno-overloaded-virtual -Wno-unused-const-variable -Wno-pessimizing-move")
endif()

2
mindspore/ccsrc/backend/session/CMakeLists.txt
View File

@ -15,7 +15,7 @@ if("${ENABLE_HIDDEN}" STREQUAL "OFF")
endif()
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-overloaded-virtual")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-overloaded-virtual -Wno-delete-abstract-non-virtual-dtor")
endif()
if(ENABLE_GPU)

4
mindspore/ccsrc/backend/session/cpu_session.cc
View File

@ -85,7 +85,7 @@ void CPUSession::Reorder(std::vector<CNodePtr> *node_list) { AnfAlgo::ReorderPos
void CPUSession::Optimize(const std::shared_ptr<KernelGraph> &kernel_graph) {
auto optimizer = std::make_shared<opt::GraphOptimizer>();
auto pm = std::make_shared<opt::PassManager>();
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
auto ms_context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(ms_context);
if (ms_context->get_param<int>(MS_CTX_EXECUTION_MODE) != kPynativeMode && ps::PSContext::instance()->is_ps_mode()) {
@ -205,7 +205,7 @@ void CPUSession::PreExecuteGraph(const std::shared_ptr<KernelGraph> &kernel_grap
MS_LOG(INFO) << "Bind input output address";
runtime_.BindInputOutput(kernel_graph.get(), inputs, outputs);
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
InitPSParamAndOptim(kernel_graph, inputs);
#endif
}

2
mindspore/ccsrc/backend/session/executor.cc
View File

@ -23,7 +23,7 @@
#include "utils/comm_manager.h"
#include "utils/scoped_long_running.h"
#include "pybind_api/ir/tensor_py.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "ps/ps_cache/ps_cache_manager.h"
#endif

4
mindspore/ccsrc/backend/session/session_basic.cc
View File

@ -44,7 +44,7 @@
#include "utils/file_utils.h"
#include "utils/trace_base.h"
#include "frontend/parallel/context.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "ps/ps_cache/ps_cache_manager.h"
#include "ps/constants.h"
#include "ps/util.h"
@ -2569,7 +2569,7 @@ void SessionBasic::DumpGraph(const std::shared_ptr<KernelGraph> &kernel_graph) {
void SessionBasic::UnifyMindIR(const KernelGraphPtr &graph) { opt::CommonUnifyMindIR(graph); }
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
void SessionBasic::InitPsWorker(const KernelGraphPtr &kernel_graph) {
if (!ps::PSContext::instance()->is_worker()) {
return;

2
mindspore/ccsrc/backend/session/session_basic.h
View File

@ -309,7 +309,7 @@ class SessionBasic : public std::enable_shared_from_this<SessionBasic> {
std::vector<uint32_t> GetAllReduceSplitIndex();
virtual std::string GetCommWorldGroup() { return std::string(); }
void DumpGraph(const std::shared_ptr<KernelGraph> &kernel_graph);
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
void CheckPSModeConsistence(const KernelGraphPtr &kernel_graph) const;
void GetBatchElements(const AnfNodePtr &kernel_node) const;
void InitPsWorker(const KernelGraphPtr &kernel_graph);

21
mindspore/ccsrc/cxx_api/CMakeLists.txt
View File

@ -88,21 +88,16 @@ if(NOT BUILD_LITE)
set_target_properties(mindspore_shared_lib PROPERTIES OUTPUT_NAME mindspore)
endif()
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(mindspore_shared_lib PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,-force_load mindspore -Wl,-noall_load mindspore_core proto_input mindspore_gvar mindspore::protobuf)
if(ENABLE_D OR ENABLE_GPU)
target_link_libraries(mindspore_shared_lib PRIVATE -Wl,--as-needed ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,--whole-archive mindspore -Wl,--no-whole-archive mindspore_core proto_input mindspore_gvar
mindspore::protobuf)
else()
if(ENABLE_D OR ENABLE_GPU)
target_link_libraries(mindspore_shared_lib PRIVATE -Wl,--as-needed ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,--whole-archive mindspore -Wl,--no-whole-archive mindspore_core proto_input mindspore_gvar
mindspore::protobuf)
if(BUILD_LITE)
target_link_libraries(mindspore_shared_lib PRIVATE ${SECUREC_LIBRARY})
else()
if(BUILD_LITE)
target_link_libraries(mindspore_shared_lib PRIVATE ${SECUREC_LIBRARY})
else()
target_link_libraries(mindspore_shared_lib PRIVATE ${PY3_LIBG} ${SECUREC_LIBRARY}
mindspore mindspore_core proto_input mindspore_gvar mindspore::protobuf)
endif()
target_link_libraries(mindspore_shared_lib PRIVATE ${PY3_LIBG} ${SECUREC_LIBRARY}
mindspore mindspore_core proto_input mindspore_gvar mindspore::protobuf)
endif()
endif()

12
mindspore/ccsrc/debug/CMakeLists.txt
View File

@ -33,6 +33,10 @@ if("${ENABLE_HIDDEN}" STREQUAL "OFF")
string(REPLACE " -fvisibility=hidden" " -fvisibility=default" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
endif()
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-private-field")
endif()
if(ENABLE_DEBUGGER)
list(APPEND _DEBUG_SRC_LIST
"${CMAKE_CURRENT_SOURCE_DIR}/debugger/debugger.cc"
@ -52,7 +56,7 @@ if(NOT ENABLE_SECURITY)
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/dump_utils.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/npy_header.cc"
)
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows" AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
list(APPEND _DEBUG_SRC_LIST
"${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/e2e_dump.cc"
@ -69,7 +73,11 @@ if(ENABLE_DEBUGGER)
set_property(SOURCE ${_OFFLINE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS
SUBMODULE_ID=mindspore::SubModuleId::SM_OFFLINE_DEBUG)
add_library(_mindspore_offline_debug SHARED ${_OFFLINE_SRC_LIST})
target_link_libraries(_mindspore_offline_debug PRIVATE mindspore::glog mindspore_gvar)
target_link_libraries(_mindspore_offline_debug PRIVATE
mindspore::pybind11_module
mindspore::glog
mindspore_gvar
)
set_target_properties(_mindspore_offline_debug PROPERTIES
PREFIX "${PYTHON_MODULE_PREFIX}"
SUFFIX "${PYTHON_MODULE_EXTENSION}"

6
mindspore/ccsrc/debug/common.h
View File

@ -69,7 +69,11 @@ inline std::string GetSaveGraphsPathName(const std::string &file_name, const std
inline std::string ErrnoToString(const int error_number) {
std::ostringstream ret_info;
ret_info << " Errno: " << error_number;
#if defined(SYSTEM_ENV_POSIX)
#if defined(__APPLE__)
char err_info[MAX_FILENAME_LENGTH];
(void)strerror_r(error_number, err_info, sizeof(err_info));
ret_info << ", ErrInfo: " << err_info;
#elif defined(SYSTEM_ENV_POSIX)
char err_info[MAX_FILENAME_LENGTH];
char *ret = strerror_r(error_number, err_info, sizeof(err_info));
if (ret != nullptr) {

4
mindspore/ccsrc/debug/debug_services.cc
View File

@ -1715,7 +1715,11 @@ std::string DebugServices::RealPath(const std::string &input_path) {
}
uint64_t DebugServices::BytestoUInt64(const std::vector<char> &buffer) {
#if defined(__APPLE__)
return *reinterpret_cast<const uint64_t *>(buffer.data());
#else
return le64toh(*reinterpret_cast<const uint64_t *>(buffer.data()));
#endif
}
bool DebugServices::TensorExistsInCurrent(const std::string &tensor_name) {

7
mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.cc
View File

@ -33,10 +33,17 @@ DbgServices &DbgServices::operator=(const DbgServices &other) {
return *this;
}
#if !defined(__APPLE__)
DbgServices::~DbgServices() noexcept {
MS_LOG(INFO) << "cpp DbgServices object is deleted";
debug_services_ = nullptr;
}
#else
DbgServices::~DbgServices() {
MS_LOG(INFO) << "cpp DbgServices object is deleted";
debug_services_ = nullptr;
}
#endif
std::string DbgServices::GetVersion() const {
MS_LOG(INFO) << "get version is called";

2
mindspore/ccsrc/fl/CMakeLists.txt
View File

@ -1,6 +1,6 @@
file(GLOB_RECURSE _FL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
if(NOT ENABLE_CPU OR WIN32)
if(NOT ENABLE_CPU OR WIN32 OR APPLE)
list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/apply_momentum_kernel.cc")
list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/aggregation_kernel_factory.cc")
list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/dense_grad_accum_kernel.cc")

4
mindspore/ccsrc/frontend/optimizer/irpass/symbol_resolver.cc
View File

@ -28,12 +28,12 @@ namespace irpass {
// {prim::kPrimGetAttr, bool, attr}
// {prim::kPrimResolve, namespace, symbol}
AnfNodePtr ResolverGetAttrResolve::operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) {
constexpr char PARSE_SUPER_NAME[] = "namespace";
constexpr std::string_view PARSE_SUPER_NAME = "namespace";
constexpr size_t namespace_index = 1;
constexpr size_t symbol_index = 2;
PatternNode<AnfNodePtr> resolve_node, ns_node, sym_node, attr_node, bool_node;
auto GetAttrResolveLambda = [&node, &resolve_node, &attr_node, &optimizer]() -> AnfNodePtr {
auto GetAttrResolveLambda = [&node, &resolve_node, &attr_node, &optimizer, &PARSE_SUPER_NAME]() -> AnfNodePtr {
auto inner = resolve_node.GetNode(node);
auto attr = attr_node.GetNode(node);
if (IsPrimitiveCNode(inner, prim::kPrimResolve)) {

6
mindspore/ccsrc/frontend/parallel/cache_embedding/cache_embedding.cc
View File

@ -216,7 +216,13 @@ void InitHashMapData(void *data, const int64_t host_size, const int64_t cache_si
for (int64_t i = 0; i < host_size; ++i) {
host_range.emplace_back(static_cast<T>(i));
}
#if defined(__APPLE__)
std::random_device rd;
std::mt19937 rng(rd());
std::shuffle(host_range.begin(), host_range.end(), rng);
#else
std::random_shuffle(host_range.begin(), host_range.end());
#endif
size_t size = static_cast<size_t>(cache_size);
size_t hashmap_count = 0;
for (size_t i = 0; i < size; ++i) {

6
mindspore/ccsrc/frontend/parallel/ops_info/gather_info.cc
View File

@ -25,7 +25,7 @@
#include "frontend/parallel/device_matrix.h"
#include "frontend/parallel/graph_util/generate_graph.h"
#include "frontend/parallel/context.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "ps/ps_cache/ps_cache_manager.h"
#include "utils/ms_context.h"
#endif
@ -160,7 +160,7 @@ Status GatherInfo::GetAttrs() {
if (std::find(inputs_shape_[1].begin(), inputs_shape_[1].end(), -1) != inputs_shape_[1].end()) {
dynamic_shape_indices_ = true;
}
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
MS_EXCEPTION_IF_NULL(MsContext::GetInstance());
bool enable_sparse = MsContext::GetInstance()->get_param<bool>(MS_CTX_ENABLE_SPARSE);
if (ps::PsDataPrefetch::GetInstance().cache_enable() && enable_sparse) {
@ -645,7 +645,7 @@ Status GatherInfo::InferBias() {
rank = rank % (params_strategy[0] * params_strategy[1]);
}
}
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::PsDataPrefetch::GetInstance().cache_enable()) {
bias_ = static_cast<int64_t>(ps::PsCacheManager::GetInstance().cache_indices_lower_bound());
return SUCCESS;

6
mindspore/ccsrc/frontend/parallel/ops_info/unique_info.cc
View File

@ -28,7 +28,7 @@
#include "frontend/parallel/strategy.h"
#include "frontend/parallel/context.h"
#include "frontend/parallel/tensor_layout/tensor_redistribution.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#include "ps/ps_cache/ps_cache_manager.h"
#endif
@ -101,7 +101,7 @@ std::vector<StrategyPtr> UniqueInfo::GenerateOpStrategies(int64_t stage_id) {
return sp_vector;
}
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
Status UniqueInfo::ComputeReplaceGraph(const CNodePtr &cnode) {
GenerateGraph gen_g = GenerateGraph(attrs_);
if (gen_g.Init(cnode) != SUCCESS) {
@ -138,7 +138,7 @@ Status UniqueInfo::ComputeReplaceGraph(const CNodePtr &cnode) {
#endif
ReplaceGraphPtr UniqueInfo::replace_graph(const CNodePtr &cnode) {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::PsDataPrefetch::GetInstance().cache_enable()) {
auto inputs = cnode->inputs();
if (inputs.empty()) {

2
mindspore/ccsrc/frontend/parallel/ops_info/unique_info.h
View File

@ -47,7 +47,7 @@ class UniqueInfo : public OperatorInfo {
Status InferDevMatrixShape() override;
Status InferForwardCommunication() override { return SUCCESS; }
Status InferAsLossDivisor() override { return SUCCESS; }
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
Status ComputeReplaceGraph(const CNodePtr &cnode);
#endif

2
mindspore/ccsrc/frontend/parallel/step_auto_parallel.cc
View File

@ -55,7 +55,7 @@
namespace mindspore {
namespace parallel {
bool StepAutoParallel(const FuncGraphPtr &root, const opt::OptimizerPtr &) {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::Util::IsRoleOfPServer() || ps::Util::IsRoleOfScheduler()) {
return false;
}

2
mindspore/ccsrc/frontend/parallel/step_parallel.cc
View File

@ -3182,7 +3182,7 @@ static void PipelinePostProcess(const FuncGraphPtr &root, const std::vector<AnfN
}
bool StepParallel(const FuncGraphPtr &root, const opt::OptimizerPtr &optimizer) {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::PSContext::instance()->is_server() || ps::PSContext::instance()->is_scheduler()) {
return false;
}

55
mindspore/ccsrc/minddata/dataset/CMakeLists.txt
View File

@ -3,9 +3,10 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reorder")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-switch")
if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-uninitialized -Wno-deprecated-declarations
-Wno-delete-non-abstract-non-virtual-dtor
-Wno-constant-conversion -Wno-unknown-warning-option")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-uninitialized -Wno-deprecated-declarations \
-Wno-delete-non-abstract-non-virtual-dtor \
-Wno-unused-comparison -Wno-constant-conversion \
-Wno-unknown-warning-option -Wno-macro-redefined")
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-maybe-uninitialized")
endif()
@ -73,7 +74,9 @@ add_subdirectory(core)
add_subdirectory(kernels)
add_subdirectory(engine)
add_subdirectory(api)
add_subdirectory(audio)
if(NOT APPLE)
add_subdirectory(audio)
endif()
add_subdirectory(text)
add_subdirectory(callback)
add_subdirectory(plugin)
@ -88,12 +91,16 @@ add_dependencies(engine-datasetops-source-sampler core)
add_dependencies(engine-datasetops core)
add_dependencies(engine-datasetops-mapop core)
add_dependencies(engine-opt core)
add_dependencies(engine-gnn core)
if(NOT APPLE)
add_dependencies(engine-gnn core)
endif()
add_dependencies(engine core)
add_dependencies(callback core)
add_dependencies(audio-kernels core)
add_dependencies(audio-ir core)
add_dependencies(audio-ir-kernels core)
if(NOT APPLE)
add_dependencies(audio-kernels core)
add_dependencies(audio-ir core)
add_dependencies(audio-ir-kernels core)
endif()
add_dependencies(text core)
add_dependencies(text-kernels core)
add_dependencies(text-ir core)
@ -154,14 +161,10 @@ set(submodules
$<TARGET_OBJECTS:engine-datasetops-source>
$<TARGET_OBJECTS:engine-datasetops-source-sampler>
$<TARGET_OBJECTS:engine-datasetops-mapop>
$<TARGET_OBJECTS:engine-gnn>
$<TARGET_OBJECTS:engine-datasetops>
$<TARGET_OBJECTS:engine-opt>
$<TARGET_OBJECTS:engine-cache-client>
$<TARGET_OBJECTS:engine>
$<TARGET_OBJECTS:audio-kernels>
$<TARGET_OBJECTS:audio-ir>
$<TARGET_OBJECTS:audio-ir-kernels>
$<TARGET_OBJECTS:text>
$<TARGET_OBJECTS:text-kernels>
$<TARGET_OBJECTS:text-ir>
@ -172,6 +175,14 @@ set(submodules
$<TARGET_OBJECTS:md_plugin>
)
if(NOT APPLE)
set(submodules
${submodules}
$<TARGET_OBJECTS:engine-gnn>
$<TARGET_OBJECTS:audio-kernels>
$<TARGET_OBJECTS:audio-ir>
$<TARGET_OBJECTS:audio-ir-kernels>)
endif()
if(ENABLE_ACL)
set(submodules
@ -212,12 +223,6 @@ endif()
################# Link with external libraries ########################
target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar)
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_MACOSX_RPATH 1)
set(CMAKE_INSTALL_RPATH "@loader_path/lib;@loader_path")
set_target_properties(_c_dataengine PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}")
endif()
set(targetso $<TARGET_FILE:_c_dataengine>)
set(target_tiny "libtinyxml")
set(target_icuuc "libicuuc")
@ -315,7 +320,9 @@ else()
if(${ENABLE_IBVERBS} STREQUAL "ON")
target_link_libraries(_c_dataengine PRIVATE ibverbs rdmacm)
endif()
target_link_libraries(_c_dataengine PRIVATE ps_cache)
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(_c_dataengine PRIVATE ps_cache)
endif()
endif()
endif()
@ -332,7 +339,13 @@ else()
endif()
if(MS_BUILD_GRPC)
target_link_libraries(_c_dataengine PRIVATE -Wl,--no-as-needed mindspore::grpc++)
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(_c_dataengine PRIVATE -Wl,--no-as-needed mindspore::grpc++)
else()
target_link_libraries(_c_dataengine PRIVATE mindspore::grpc++)
endif()
endif()
set_target_properties(_c_dataengine PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
set_target_properties(_c_dataengine PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})
endif()

77
mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt
View File

@ -1,42 +1,55 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
if(ENABLE_PYTHON)
add_library(APItoPython OBJECT
set(API_PYTHON_SRC_FILES
python/bindings/dataset/audio/bindings.cc
python/bindings/dataset/audio/kernels/ir/bindings.cc
python/bindings/dataset/callback/bindings.cc
python/bindings/dataset/core/bindings.cc
python/bindings/dataset/engine/cache/bindings.cc
python/bindings/dataset/engine/datasetops/bindings.cc
python/bindings/dataset/engine/gnn/bindings.cc
python/bindings/dataset/engine/ir/consumer/bindings.cc
python/bindings/dataset/engine/ir/datasetops/bindings.cc
python/bindings/dataset/engine/ir/datasetops/source/bindings.cc
python/bindings/dataset/engine/ir/datasetops/source/samplers/bindings.cc
python/bindings/dataset/engine/ir/execute/bindings.cc
python/bindings/dataset/engine/ir/schema/bindings.cc
python/bindings/dataset/engine/perf/bindings.cc
python/bindings/dataset/kernels/bindings.cc
python/bindings/dataset/kernels/ir/bindings.cc
python/bindings/dataset/kernels/ir/image/bindings.cc
python/bindings/dataset/text/bindings.cc
python/bindings/dataset/text/kernels/ir/bindings.cc
python/bindings/mindrecord/include/bindings.cc
python/pybind_conversion.cc
python/pybind_register.cc
)
if(APPLE)
list(REMOVE_ITEM API_PYTHON_SRC_FILES
python/bindings/dataset/audio/bindings.cc
python/bindings/dataset/audio/kernels/ir/bindings.cc
python/bindings/dataset/callback/bindings.cc
python/bindings/dataset/core/bindings.cc
python/bindings/dataset/engine/cache/bindings.cc
python/bindings/dataset/engine/datasetops/bindings.cc
python/bindings/dataset/engine/gnn/bindings.cc
python/bindings/dataset/engine/ir/consumer/bindings.cc
python/bindings/dataset/engine/ir/datasetops/bindings.cc
python/bindings/dataset/engine/ir/datasetops/source/bindings.cc
python/bindings/dataset/engine/ir/datasetops/source/samplers/bindings.cc
python/bindings/dataset/engine/ir/execute/bindings.cc
python/bindings/dataset/engine/ir/schema/bindings.cc
python/bindings/dataset/engine/perf/bindings.cc
python/bindings/dataset/kernels/bindings.cc
python/bindings/dataset/kernels/ir/bindings.cc
python/bindings/dataset/kernels/ir/image/bindings.cc
python/bindings/dataset/text/bindings.cc
python/bindings/dataset/text/kernels/ir/bindings.cc
python/bindings/mindrecord/include/bindings.cc
python/pybind_conversion.cc
python/pybind_register.cc
)
endif()
add_library(APItoPython OBJECT ${API_PYTHON_SRC_FILES})
target_include_directories(APItoPython PRIVATE ${pybind11_INCLUDE_DIRS})
endif()
add_library(cpp-API OBJECT
audio.cc
config.cc
data_helper.cc
datasets.cc
execute.cc
iterator.cc
transforms.cc
samplers.cc
text.cc
vision.cc
)
set(API_CPP_SRC_FILES
audio.cc
config.cc
data_helper.cc
datasets.cc
execute.cc
iterator.cc
transforms.cc
samplers.cc
text.cc
vision.cc
)
if(APPLE)
list(REMOVE_ITEM API_CPP_SRC_FILES audio.cc)
endif()
add_library(cpp-API OBJECT ${API_CPP_SRC_FILES})

21
mindspore/ccsrc/minddata/dataset/engine/CMakeLists.txt
View File

@ -1,6 +1,8 @@
add_subdirectory(datasetops)
add_subdirectory(opt)
add_subdirectory(gnn)
if(NOT APPLE)
add_subdirectory(gnn)
endif()
add_subdirectory(ir)
message("ENABLE_SECURITY = ${ENABLE_SECURITY}")
@ -43,13 +45,16 @@ if(ENABLE_PYTHON)
target_include_directories(engine PRIVATE ${pybind11_INCLUDE_DIRS})
endif()
add_dependencies(engine engine-datasetops
engine-datasetops-source
engine-opt
engine-gnn
engine-cache-client
engine-datasetops-mapop
)
add_dependencies(engine
engine-datasetops
engine-datasetops-source
engine-opt
engine-cache-client
engine-datasetops-mapop
)
if(NOT APPLE)
add_dependencies(engine engine-gnn)
endif()
if(NOT ENABLE_SECURITY)
add_dependencies(engine engine-perf)

21
mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt vendored
View File

@ -96,10 +96,15 @@ if(ENABLE_CACHE)
${PYTHON_LIBRARIES}
${SECUREC_LIBRARY}
pthread
-ldl
-Wl,--no-as-needed
mindspore::grpc++
-Wl,--as-needed)
-ldl)
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(cache_server mindspore::grpc++)
else()
target_link_libraries(cache_server
-Wl,--no-as-needed
mindspore::grpc++
-Wl,--as-needed)
endif()
endif()
if(USE_GLOG)
@ -116,8 +121,12 @@ if(ENABLE_CACHE)
target_link_libraries(cache_admin
-Wl,--start-group mindspore::protobuf mindspore_core mindspore_shared_lib ${ACL_TDT_CHANNEL} -Wl,--end-group)
else()
target_link_libraries(cache_admin
-Wl,--start-group mindspore::protobuf mindspore_core mindspore_shared_lib -Wl,--end-group)
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(cache_admin mindspore::protobuf mindspore_core mindspore_shared_lib)
else()
target_link_libraries(cache_admin
-Wl,--start-group mindspore::protobuf mindspore_core mindspore_shared_lib -Wl,--end-group)
endif()
endif()
if(USE_GLOG)

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc
View File

@ -130,7 +130,7 @@ Status BatchOp::operator()() {
RETURN_IF_NOT_OK(GetBatchSize(&cur_batch_size, CBatchInfo(epoch_num, batch_num, cnt - epoch_num)));
RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row));
#if !defined(_WIN32) && !defined(_WIN64) && ENABLE_PYTHON
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) && ENABLE_PYTHON
if ((num_workers_ > 1 || batch_map_func_) && GetMemoryUsage() > MAX_MEMORY_USAGE_THRESHOLD) {
MS_LOG(WARNING) << "Memory consumption is more than " << MAX_MEMORY_USAGE_THRESHOLD * 100 << "%, "
<< "which may cause oom error. Please reduce num_parallel_workers size / "

2
mindspore/ccsrc/minddata/dataset/util/status.cc
View File

@ -28,7 +28,7 @@
namespace mindspore {
namespace dataset {
#if !defined(_WIN32) && !defined(_WIN64)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__)
float GetMemoryUsage() {
char buf[128] = {0};

2
mindspore/ccsrc/minddata/dataset/util/status.h
View File

@ -89,7 +89,7 @@ namespace dataset {
} \
} while (false)
#if !defined(_WIN32) && !defined(_WIN64)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__)
const float MAX_MEMORY_USAGE_THRESHOLD = 0.95;
float GetMemoryUsage();
#endif

2
mindspore/ccsrc/minddata/dataset/util/task.cc
View File

@ -39,7 +39,7 @@ void Task::operator()() {
id_ = this_thread::get_id();
std::stringstream ss;
ss << id_;
#if defined(__ANDROID__) || defined(ANDROID)
#if defined(__ANDROID__) || defined(ANDROID) || defined(__APPLE__)
// The thread id in Linux may be duplicate
ss << Services::GetUniqueID();
#endif

2
mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc
View File

@ -225,7 +225,7 @@ Status ShardIndexGenerator::CreateShardNameTable(sqlite3 *db, const std::string
Status ShardIndexGenerator::CreateDatabase(int shard_no, sqlite3 **db) {
std::string shard_address = shard_header_.GetShardAddressByID(shard_no);
CHECK_FAIL_RETURN_UNEXPECTED(!shard_address.empty(), "Shard address is empty, shard No: " + shard_no);
CHECK_FAIL_RETURN_UNEXPECTED(!shard_address.empty(), "Shard address is empty, shard No: " + std::to_string(shard_no));
std::shared_ptr<std::string> fn_ptr;
RETURN_IF_NOT_OK(GetFileName(shard_address, &fn_ptr));
shard_address += ".db";

10
mindspore/ccsrc/pipeline/jit/action.cc
View File

@ -46,7 +46,7 @@
#include "frontend/optimizer/py_pass_manager.h"
#include "utils/ms_context.h"
#include "vm/transform.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#include "ps/parameter_server.h"
#include "ps/scheduler.h"
#include "ps/worker.h"
@ -623,7 +623,7 @@ bool OptInlineAction(const ResourcePtr &res) {
bool GeOptimizeAction(const ResourcePtr &res) { return OptimizeAction(res, kGePasses); }
bool VmOptimizeAction(const ResourcePtr &res) {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
if (ps::PSContext::instance()->is_ps_mode()) {
kVmPasses.push_back({"server_communication_op_fusion", ps::Util::FuseServerCommOps});
}
@ -823,7 +823,7 @@ bool ExecuteAction(const ResourcePtr &res) {
return true;
}
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
bool StartPSWorkerAction(const ResourcePtr &) {
ps::Worker::GetInstance().Run();
return true;
@ -1156,7 +1156,7 @@ std::vector<ActionItem> VmPipeline() {
(void)actions.emplace_back(std::make_pair("eliminate_forward_cnode", EliminateForwardCNode));
(void)actions.emplace_back(std::make_pair("validate", ValidateAction));
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::PSContext::instance()->is_worker()) {
std::string server_mode = ps::PSContext::instance()->server_mode();
if (server_mode == ps::kServerModeFL || server_mode == ps::kServerModeHybrid) {
@ -1194,7 +1194,7 @@ std::vector<ActionItem> MindIRPipeline() {
(void)actions.emplace_back(std::make_pair("execute", ExecuteAction));
return actions;
}
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
std::vector<ActionItem> ServerPipeline() {
auto actions = CommonPipeline();
(void)actions.emplace_back(std::make_pair("optimize", VmOptimizeAction));

2
mindspore/ccsrc/pipeline/jit/init.cc
View File

@ -36,7 +36,7 @@
#else
#include "runtime/device/gpu/distribution/collective_fake_init.h"
#endif
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "ps/util.h"
#endif
#include "ps/ps_context.h"

6
mindspore/ccsrc/pipeline/jit/pass.cc
View File

@ -48,7 +48,7 @@
#include "frontend/optimizer/irpass/gradient_eliminate.h"
#include "frontend/optimizer/irpass/parameter_eliminate.h"
#include "frontend/optimizer/irpass/updatestate_eliminate.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "ps/util.h"
#include "ps/ps_context.h"
#endif
@ -217,7 +217,7 @@ namespace {
bool ReAutoMonadWrapper(const FuncGraphPtr &root, const opt::OptimizerPtr &) { return ReAutoMonad(root); }
bool parallel_mode() {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::PSContext::instance()->is_server() || ps::PSContext::instance()->is_scheduler()) {
return false;
}
@ -596,7 +596,7 @@ bool AddRecomputationPass(const ResourcePtr &res) {
bool AddCacheEmbeddingPass(const ResourcePtr &res) {
MS_EXCEPTION_IF_NULL(res);
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::PSContext::instance()->is_ps_mode()) {
return true;
}

10
mindspore/ccsrc/pipeline/jit/pipeline.cc
View File

@ -64,7 +64,7 @@
#endif
#endif
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "ps/constants.h"
#include "ps/util.h"
#include "ps/worker.h"
@ -707,7 +707,7 @@ std::vector<ActionItem> GetPipeline(const ResourcePtr &resource, const std::stri
std::string backend = MsContext::GetInstance()->backend_policy();
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
const std::string &server_mode = ps::PSContext::instance()->server_mode();
if ((server_mode == ps::kServerModeFL || server_mode == ps::kServerModeHybrid) &&
ps::PSContext::instance()->is_server()) {
@ -1228,7 +1228,7 @@ bool InitExecDataset(const std::string &queue_name, int64_t iter_num, int64_t ba
bool InitExecDatasetVm(const std::string &queue_name, int64_t size, int64_t batch_size,
const std::vector<TypePtr> &types, const std::vector<std::vector<int64_t>> &shapes,
const std::vector<int64_t> &input_indexes, bool need_run) {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
if ((ps::PSContext::instance()->is_ps_mode()) && (!ps::PSContext::instance()->is_worker())) {
return true;
}
@ -1297,7 +1297,7 @@ bool InitExecDatasetVm(const std::string &queue_name, int64_t size, int64_t batc
auto runner = convert_fn(segment, "");
ConfigManager::GetInstance().set_iter_num(size);
// PS cache does not support loop sink.
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::PSContext::instance()->is_worker() && ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps::PsDataPrefetch::GetInstance().CreateDataChannel(queue_name, LongToSize(size));
ConfigManager::GetInstance().set_iter_num(1);
@ -1482,7 +1482,7 @@ void FinalizeBackend() {
void ClearResAtexit() {
MS_LOG(DEBUG) << "Pipeline clear all resource";
RecordExitStatus();
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::PSContext::instance()->is_ps_mode() && ps::PSContext::instance()->is_worker()) {
if (ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps::ps_cache_instance.Finalize();

5
mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.h
View File

@ -136,11 +136,14 @@ class AnfNodeConfig : public Config {
bool operator==(const AnfNodeConfig &other) const {
// compare node with pointer, context with pointer except DummyContext as it's created by make_shared;
// context should not be nullptr;
if (node_ != other.node_) {
return false;
}
if (context_->IsDummyContext() && other.context_->IsDummyContext()) {
return true;
}
// Don't check `func_graph_` equality.
return (node_ == other.node_) && (context_ == other.context_);
return context_ == other.context_;
}
std::string ToString() const override {

2
mindspore/ccsrc/ps/CMakeLists.txt
View File

@ -1,6 +1,6 @@
file(GLOB_RECURSE _PS_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
if(NOT ENABLE_CPU OR WIN32)
if(NOT ENABLE_CPU OR WIN32 OR APPLE)
list(REMOVE_ITEM _PS_SRC_FILES "optimizer_info_builder.cc")
list(REMOVE_ITEM _PS_SRC_FILES "optimizer_info.cc")
list(REMOVE_ITEM _PS_SRC_FILES "scheduler.cc")

2
mindspore/ccsrc/ps/ps_cache/CMakeLists.txt
View File

@ -1,4 +1,4 @@
if(ENABLE_CPU AND NOT WIN32)
if(ENABLE_CPU AND NOT WIN32 AND NOT APPLE)
file(GLOB_RECURSE _PS_CACHE_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "ps_data/*.cc")
set_property(SOURCE ${_PS_CACHE_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PS)
add_library(ps_cache SHARED ${_PS_CACHE_SRC_FILES})

18
mindspore/ccsrc/ps/ps_context.cc
View File

@ -18,7 +18,7 @@
#include "utils/log_adapter.h"
#include "utils/ms_utils.h"
#include "backend/kernel_compiler/kernel.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#include "ps/ps_cache/ps_cache_manager.h"
#include "ps/ps_cache/ps_data/ps_data_prefetch.h"
#endif
@ -88,7 +88,7 @@ void PSContext::Reset() {
is_worker_ = false;
is_pserver_ = false;
is_sched_ = false;
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
if (ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps_cache_instance.Finalize();
set_cache_enable(false);
@ -144,44 +144,44 @@ uint32_t PSContext::ps_rank_id() const { return rank_id_; }
void PSContext::InsertHashTableSize(const std::string &param_name, size_t cache_vocab_size, size_t embedding_size,
size_t vocab_size) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
ps_cache_instance.InsertHashTableSize(param_name, cache_vocab_size, embedding_size, vocab_size);
#endif
}
void PSContext::ReInsertHashTableSize(const std::string &new_param_name, const std::string &cur_param_name,
size_t cache_vocab_size, size_t embedding_size) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
ps_cache_instance.ReInsertHashTableSize(new_param_name, cur_param_name, cache_vocab_size, embedding_size);
#endif
}
void PSContext::InsertWeightInitInfo(const std::string &param_name, size_t global_seed, size_t op_seed) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
ps_cache_instance.InsertWeightInitInfo(param_name, global_seed, op_seed);
#endif
}
void PSContext::InsertAccumuInitInfo(const std::string &param_name, float init_val) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
ps_cache_instance.InsertAccumuInitInfo(param_name, init_val);
#endif
}
void PSContext::CloneHashTable(const std::string &dest_param_name, const std::string &src_param_name) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
ps_cache_instance.CloneHashTable(dest_param_name, src_param_name);
#endif
}
void PSContext::set_cache_enable(bool cache_enable) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
PsDataPrefetch::GetInstance().set_cache_enable(cache_enable);
#endif
}
void PSContext::set_rank_id(uint32_t rank_id) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
ps_cache_instance.set_rank_id(rank_id);
#endif
}

8
mindspore/ccsrc/runtime/device/kernel_runtime.cc
View File

@ -33,7 +33,7 @@
#include "frontend/parallel/context.h"
#include "debug/env_config_parser.h"
#include "pipeline/pynative/pynative_profiling.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "ps/ps_cache/ps_cache_manager.h"
#endif
@ -579,7 +579,7 @@ void KernelRuntime::AssignStaticMemoryInput(const session::KernelGraph &graph) {
}
add_need_alloc_nodes(input_node);
}
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
bool ps_cache_check = false;
#endif
for (auto &item : need_alloc_nodes) {
@ -593,7 +593,7 @@ void KernelRuntime::AssignStaticMemoryInput(const session::KernelGraph &graph) {
continue;
}
DeviceAddressPtr device_address = nullptr;
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
const std::string &param_name = item->fullname_with_scope();
if (ps::ps_cache_instance.IsHashTable(param_name)) {
MS_LOG(INFO) << "Parameter(" << param_name << ")"
@ -1606,7 +1606,7 @@ void KernelRuntime::ClearGraphRuntimeResource(uint32_t graph_id) {
MS_LOG(INFO) << "Clear graph:" << graph_id << " runtime resource";
}
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
void KernelRuntime::GetFirstPSEmbeddingCache(const session::KernelGraph &graph,
AnfNodePtr *const first_cache_input_index,
size_t *const first_cache_size) {

2
mindspore/ccsrc/runtime/device/kernel_runtime.h
View File

@ -173,7 +173,7 @@ class KernelRuntime {
void RunOpAssignOutputNodeMemory(const ValuePtr &pre_output_value, const session::KernelGraph &graph);
void AssignValueNodeTensor(const ValueNodePtr &value_node, const ValuePtr &node_value, size_t output_idx);
DeviceAddressPtr PreAssignCNodeMemory(const AnfNodePtr &anf_node, size_t index) const;
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
void GetFirstPSEmbeddingCache(const session::KernelGraph &graph, AnfNodePtr *const first_cache_input_index,
size_t *const first_cache_size);
void CheckIfSupportPSEmbeddingCache(const session::KernelGraph &graph);

6
mindspore/ccsrc/runtime/device/kernel_runtime_manager.cc
View File

@ -16,7 +16,7 @@
#include "runtime/device/kernel_runtime_manager.h"
#include "utils/log_adapter.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#include "ps/ps_cache/ps_cache_manager.h"
#endif
#include "backend/session/pynative_task_manager.h"
@ -26,7 +26,7 @@ namespace device {
void KernelRuntimeManager::ClearRuntimeResource() {
// Just remove PyNative tasks before runtime resource release.
session::PynativeTaskManager::GetInstance().Reset();
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::PSContext::instance()->is_worker() && ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps::ps_cache_instance.SyncEmbeddingTable();
}
@ -132,7 +132,7 @@ void KernelRuntimeManager::ReleaseKernelRuntime(const std::string &device_name,
if (runtime == nullptr) {
return;
}
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::PSContext::instance()->is_worker() && ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps::ps_cache_instance.SyncEmbeddingTable();
}

9
mindspore/ccsrc/runtime/framework/actor/super_kernel_actor.cc
View File

@ -52,21 +52,22 @@ void SuperKernelActor::Run(OpContext<DeviceTensor> *const context) {
MS_EXCEPTION_IF_NULL(context);
MS_EXCEPTION_IF_NULL(graph_);
MS_EXCEPTION_IF_NULL(device_contexts_[0]);
MS_LOG(INFO) << "Super kernel actor(" << GetAID().Name() << ") launches graph: " << graph_->graph_id();
MS_LOG(INFO) << "Super kernel actor(" << GetAID().Name()
<< ") launches graph: " << std::to_string(graph_->graph_id());
if (!CheckInputData(context)) {
std::string error_info = "Check the input data invalid, graph id: " + graph_->graph_id();
std::string error_info = "Check the input data invalid, graph id: " + std::to_string(graph_->graph_id());
SET_OPCONTEXT_FAIL_RET_WITH_ERROR((*context), error_info);
}
try {
auto ret = device_contexts_[0]->LaunchGraph(graph_);
if (!ret) {
std::string error_info = "Launch graph failed, graph id: " + graph_->graph_id();
std::string error_info = "Launch graph failed, graph id: " + std::to_string(graph_->graph_id());
SET_OPCONTEXT_FAIL_RET_WITH_ERROR((*context), error_info);
}
} catch (const std::exception &e) {
MsException::Instance().SetException();
std::string error_info = "Launch graph exception, graph id: " + graph_->graph_id();
std::string error_info = "Launch graph exception, graph id: " + std::to_string(graph_->graph_id());
SET_OPCONTEXT_FAIL_RET_WITH_ERROR((*context), error_info);
}

4
mindspore/ccsrc/runtime/hardware/CMakeLists.txt
View File

@ -35,6 +35,10 @@ if(ENABLE_CPU)
endif()
endif()
if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-abstract-non-virtual-dtor")
endif()
set_property(SOURCE ${HARDWARE_SRC_LIST} ${HARDWARE_D_SRC_LIST} ${HARDWARE_GPU_SRC_LIST} ${HARDWARE_CPU_SRC_LIST}
PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_DEVICE)

2
mindspore/ccsrc/runtime/hardware/cpu/cpu_memory_pool.cc
View File

@ -27,7 +27,7 @@ const size_t kKBToByte = 1024;
const size_t kLineMaxSize = 1024;
size_t GetSystemMemorySize(const std::string &key) {
#if defined(_WIN32) || defined(_WIN64)
#if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__)
return SIZE_MAX;
#else
FILE *file = fopen("/proc/meminfo", "r");

2
mindspore/ccsrc/vm/transform.cc
View File

@ -632,7 +632,7 @@ void SetMindRTEnable() {
return;
#endif
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
if (ps::PSContext::instance()->is_ps_mode()) {
return;
}

3
mindspore/core/CMakeLists.txt
View File

@ -42,7 +42,8 @@ if(CMAKE_SYSTEM_NAME MATCHES "Windows" AND NOT MSVC)
add_compile_definitions(BUILDING_DLL)
elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
-Wuser-defined-warnings -Winconsistent-missing-override -Wno-delete-non-abstract-non-virtual-dtor")
-Wno-overloaded-virtual -Wno-user-defined-warnings -Winconsistent-missing-override -Wno-delete-non-virtual-dtor \
-Wno-unused-private-field")
endif()
set_property(SOURCE ${CORE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_CORE)

7
mindspore/core/gvar/log_adapter_common.cc
View File

@ -76,7 +76,12 @@ std::string GetTimeString() {
constexpr int64_t time_convert_unit = 1000;
(void)localtime_r(&cur_time.tv_sec, &now);
(void)strftime(buf, BUFLEN, "%Y-%m-%d-%H:%M:%S", &now); // format date and time
(void)snprintf(buf + time_str_len, BUFLEN - time_str_len, ".%03ld.%03ld", cur_time.tv_usec / time_convert_unit,
#ifdef __APPLE__
const std::string fmt_str = ".%03lld.%03lld";
#else
const std::string fmt_str = ".%03ld.%03ld";
#endif
(void)snprintf(buf + time_str_len, BUFLEN - time_str_len, fmt_str.c_str(), cur_time.tv_usec / time_convert_unit,
cur_time.tv_usec % time_convert_unit);
#endif
return std::string(buf);

4
mindspore/core/ir/anf.h
View File

@ -74,8 +74,8 @@ using FuncGraphVector = std::vector<FuncGraphPtr>;
class Primitive;
using PrimitivePtr = std::shared_ptr<Primitive>;
class PrimitiveHasher;
class PrimitiveEqual;
struct PrimitiveHasher;
struct PrimitiveEqual;
using PrimitiveSet = std::unordered_set<PrimitivePtr, PrimitiveHasher, PrimitiveEqual>;
class BaseRef;

6
mindspore/core/ir/scalar.h
View File

@ -435,6 +435,12 @@ class MS_CORE_API UInt64Imm final : public IntergerImm {
};
using UInt64ImmPtr = std::shared_ptr<UInt64Imm>;
IMM_TRAITS(UInt64ImmPtr, uint64_t);
#if defined(__APPLE__)
using SizetImmPtr = std::shared_ptr<UInt64Imm>;
IMM_TRAITS(SizetImmPtr, size_t);
#endif
/// \beief FloatImm defines interface for float data.
class MS_CORE_API FloatImm : public Scalar {
public:

6
mindspore/core/utils/log_adapter.cc
View File

@ -422,7 +422,7 @@ void InitSubModulesLogLevel() {
} // namespace mindspore
extern "C" {
#if defined(_WIN32) || defined(_WIN64)
#if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__)
#ifdef _MSC_VER
void common_log_init(void) {
#else
@ -483,7 +483,7 @@ MS_CORE_API void common_log_init(void) {
}
// shared lib init hook
#if defined(_WIN32) || defined(_WIN64)
#if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__)
#ifdef _MSC_VER
void mindspore_log_init(void) {
#else
@ -496,7 +496,7 @@ void mindspore_log_init(void) {
#define google mindspore_private
static bool is_glog_initialzed = false;
if (!is_glog_initialzed) {
#if !defined(_WIN32) && !defined(_WIN64)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__)
google::InitGoogleLogging("mindspore");
#endif
is_glog_initialzed = true;

10
mindspore/core/utils/log_adapter.h
View File

@ -80,6 +80,11 @@ struct LocationInfo {
const char *func_;
};
template <class T, typename std::enable_if<std::is_enum<T>::value, int>::type = 0>
constexpr std::ostream &operator<<(std::ostream &stream, const T &value) {
return stream << static_cast<typename std::underlying_type<T>::type>(value);
}
class LogStream {
public:
LogStream() { sstream_ = std::make_shared<std::stringstream>(); }
@ -102,11 +107,6 @@ class LogStream {
std::shared_ptr<std::stringstream> sstream_;
};
template <class T, typename std::enable_if<std::is_enum<T>::value, int>::type = 0>
constexpr std::ostream &operator<<(std::ostream &stream, const T &value) {
return stream << static_cast<typename std::underlying_type<T>::type>(value);
}
enum MsLogLevel : int { DEBUG = 0, INFO, WARNING, ERROR, EXCEPTION };
enum SubModuleId : int {

4
mindspore/dataset/__init__.py
View File

@ -31,13 +31,15 @@ Common imported modules in corresponding API examples are as follows:
from mindspore.dataset.transforms import c_transforms
"""
import platform
from .core import config
from .engine import *
from .engine.cache_client import DatasetCache
from .engine.datasets import *
from .engine.graphdata import GraphData
from .engine.samplers import *
from .engine.serializer_deserializer import compare, deserialize, serialize, show
if platform.system().lower() != "darwin":
from .engine.graphdata import GraphData
__all__ = []
__all__.extend(engine.__all__)

6
mindspore/dataset/core/config.py
View File

@ -422,9 +422,9 @@ def get_enable_shared_mem():
>>> # Get the flag of shared memory feature.
>>> shared_mem_flag = ds.config.get_enable_shared_mem()
"""
# For windows we forbid shared mem function temporarily
if platform.system().lower() == 'windows':
logger.warning("For windows we forbid shared mem function temporarily.")
# For windows and macos we forbid shared mem function temporarily
if platform.system().lower() in {"windows", "darwin"}:
logger.warning("For windows and macos we forbid shared mem function temporarily.")
return False
return _config.get_enable_shared_mem()

8
mindspore/dataset/engine/__init__.py
View File

@ -22,19 +22,23 @@ high performance and parse data precisely. It also provides the following
operations for users to preprocess data: shuffle, batch, repeat, map, and zip.
"""
import platform
from ..callback import DSCallback, WaitedDSCallback
from ..core import config
from .cache_client import DatasetCache
from .datasets import *
from .graphdata import GraphData, SamplingStrategy, OutputFormat
from .iterators import *
from .samplers import *
from .serializer_deserializer import compare, deserialize, serialize, show
if platform.system().lower() != "darwin":
from .graphdata import GraphData, SamplingStrategy, OutputFormat
__all__ = ["CelebADataset", "Cifar100Dataset", "Cifar10Dataset", "CLUEDataset", "CocoDataset", "CSVDataset",
"GeneratorDataset", "GraphData", "ImageFolderDataset", "ManifestDataset", "MindDataset", "MnistDataset",
"GeneratorDataset", "ImageFolderDataset", "ManifestDataset", "MindDataset", "MnistDataset",
"NumpySlicesDataset", "PaddedDataset", "TextFileDataset", "TFRecordDataset", "VOCDataset",
"DistributedSampler", "PKSampler", "RandomSampler", "SequentialSampler", "SubsetRandomSampler",
"WeightedRandomSampler", "SubsetSampler",
"DatasetCache", "DSCallback", "Schema", "WaitedDSCallback", "compare", "deserialize",
"serialize", "show", "zip"]
if platform.system().lower() != "darwin":
__all__.append("GraphData")

4
mindspore/dataset/engine/datasets.py
View File

@ -81,6 +81,8 @@ try:
except ModuleNotFoundError:
context = None
if platform.system().lower() == "darwin":
multiprocessing.set_start_method("fork")
class Shuffle(str, Enum):
GLOBAL: str = "global"
@ -4384,7 +4386,7 @@ def _check_shm_usage(num_worker, queue_size, max_rowsize, num_queues=1):
when training in parallel mode.
"""
threshold_ratio = 0.8
if platform.system() != "Windows":
if platform.system().lower() not in {"windows", "darwin"}:
shm_estimate_usage = _get_device_num() * num_worker * num_queues * \
(queue_size + 2) * max_rowsize * 1024 * 1024
try:

18
third_party/patch/sentencepiece/sentencepiece.patch001_cpu vendored
View File

@ -88,4 +88,20 @@ diff -Npur sentencepiece-0.1.92/src/CMakeLists.txt sentencepiece-0.1.92_bak/src/
+install(FILES sentencepiece_trainer.h sentencepiece_processor.h "${CMAKE_BINARY_DIR}/sentencepiece_model.pb.h"
DESTINATION ${CMAKE_INSTALL_INCDIR})
file(TO_NATIVE_PATH "${PROJECT_SOURCE_DIR}/data" data_dir)
file(TO_NATIVE_PATH "${PROJECT_SOURCE_DIR}/data" data_dir)
diff -Npur sentencepiece-0.1.92/src/VERSION sentencepiece-0.1.92_bak/src/VERSION
--- a/VERSION 2021-10-29 14:48:06.343250100 +0800
+++ b/VERSION 1970-01-01 08:00:00.000000000 +0800
@@ -1 +0,0 @@
-0.1.92
diff -Npur sentencepiece-0.1.92/src/VERSION.txt sentencepiece-0.1.92_bak/src/VERSION.txt
--- a/VERSION.txt 1970-01-01 08:00:00.000000000 +0800
+++ b/VERSION.txt 2021-10-29 14:48:09.481862200 +0800
@@ -0,0 +1 @@
+0.1.92
diff -Npur sentencepiece-0.1.92/CMakeLists.txt sentencepiece-0.1.92_bak/CMakeLists.txt
--- sentencepiece-0.1.92/CMakeLists.txt 2020-06-08 16:25:01.000000000 +0800
+++ sentencepiece-0.1.92_bak/CMakeLists.txt 2020-07-02 17:42:33.306933546 +0800
@@ -16,1 +16,1 @@
-file(STRINGS "VERSION" SPM_VERSION)
+file(STRINGS "VERSION.txt" SPM_VERSION)