p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!26808 fix mindspore mac compile 

Merge pull request !26808 from xulei/fix_mac_compile_components
This commit is contained in:
i-robot 2021-11-30 02:00:29 +00:00 committed by Gitee
parent 53804eec0a 78b37afc30
commit 17fcbaae94
43 changed files with 317 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -20,7 +20,7 @@ endif()
if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(CMAKE_OSX_SYSROOT "")
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Winconsistent-missing-override -Wuser-defined-warnings \
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Winconsistent-missing-override -Wno-user-defined-warnings \
-Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare \
-Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move \
-Wno-inconsistent-missing-override -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2")

54
cmake/change_rpath.cmake Normal file
View File

@ -0,0 +1,54 @@
function(changerpath target_so target_name link_sos)
set(depend_so "")
foreach(link_so ${link_sos})
set(some-file "${CMAKE_SOURCE_DIR}/build/${target_name}_${link_so}.txt")
set(some-file1 "${CMAKE_SOURCE_DIR}/build/${target_name}_${link_so}1.txt")
set(some-file2 "${CMAKE_SOURCE_DIR}/build/${target_name}_${link_so}2.txt")
add_custom_command(
OUTPUT
${some-file}
COMMAND
otool -L ${target_so} | tail -n +2 | grep ${link_so} | head -n1 > ${some-file}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
VERBATIM
)
add_custom_command(
OUTPUT
${some-file1}
COMMAND
cat ${some-file} | cut -d " " -f 1 | sed -E "s/^.//g" > ${some-file1}
DEPENDS
${some-file}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
VERBATIM
)
add_custom_command(
OUTPUT
${some-file2}
COMMAND
awk -F "/" "{print $NF}" ${some-file1} > ${some-file2}
DEPENDS
${some-file1}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
VERBATIM
)
add_custom_target(
link_${target_name}_${link_so} ALL
COMMAND install_name_tool -change `cat ${some-file1}` @rpath/`cat ${some-file2}` ${target_so}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
DEPENDS ${target_so} ${some-file1} ${some-file2}
COMMENT "install tool name"
)
add_custom_command(
TARGET link_${target_name}_${link_so}
POST_BUILD
COMMAND rm ${some-file} ${some-file1} ${some-file2}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
VERBATIM
)
if(depend_so)
add_dependencies(link_${target_name}_${link_so} link_${target_name}_${depend_so})
endif()
set(depend_so ${link_so})
endforeach()
endfunction()

8
cmake/external_libs/icu4c.cmake
View File

@ -28,7 +28,7 @@ else()
URL ${REQ_URL}
MD5 ${MD5}
PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/icu4c/icu4c.patch01
CONFIGURE_COMMAND ./icu4c/source/runConfigureICU MacOSX --enable-rpath --disable-tests
CONFIGURE_COMMAND ./icu4c/source/runConfigureICU MacOSX --disable-tests
--disable-samples --disable-icuio --disable-extras
ICU_DATA_FILTER_FILE=${CMAKE_BINARY_DIR}/icu4c_filter.json
)
@ -45,6 +45,12 @@ else()
)
endif()
include_directories(${icu4c_INC})
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
include(${CMAKE_SOURCE_DIR}/cmake/change_rpath.cmake)
changerpath($<TARGET_FILE:icu4c::${LIB_ICU_COMMON}> ${LIB_ICU_COMMON} "libicuuc;libicudata")
changerpath($<TARGET_FILE:icu4c::${LIB_ICU_DATA}> ${LIB_ICU_DATA} "libicudata")
changerpath($<TARGET_FILE:icu4c::${LIB_ICU_I18N}> ${LIB_ICU_I18N} "libicuuc;libicudata;libicui18n")
endif()
add_library(mindspore::icuuc ALIAS icu4c::${LIB_ICU_COMMON})
add_library(mindspore::icudata ALIAS icu4c::${LIB_ICU_DATA})
add_library(mindspore::icui18n ALIAS icu4c::${LIB_ICU_I18N})

9
cmake/options.cmake
View File

@ -29,11 +29,6 @@ option(ENABLE_SYM_FILE "enable sym file" OFF)
option(BUILD_DEV_MODE "MindSpore build nightly dev mode" OFF)
option(ENABLE_FAST_HASH_TABLE "Enable use fast hash table instead of std ones" ON)
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(ENABLE_DEBUGGER OFF)
endif()
if(NOT ENABLE_D AND NOT ENABLE_TESTCASES AND NOT ENABLE_ACL AND NOT ENABLE_GE)
set(ENABLE_GLIBCXX ON)
endif()
@ -143,7 +138,7 @@ endif()
if(ENABLE_DEBUGGER OR ENABLE_TESTCASES)
set(MS_BUILD_GRPC ON)
endif()
if(ENABLE_MINDDATA AND NOT CMAKE_SYSTEM_NAME MATCHES "Windows" AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
if(ENABLE_MINDDATA AND NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
set(MS_BUILD_GRPC ON)
endif()
@ -155,7 +150,7 @@ if(ENABLE_ACL AND NOT ENABLE_D)
set(MODE_ASCEND_ACL ON)
endif()
if(ENABLE_CPU AND NOT WIN32 AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
if(ENABLE_CPU AND NOT WIN32)
add_compile_definitions(ENABLE_ARMOUR)
endif()

38
cmake/package_mac.cmake
View File

@ -40,9 +40,6 @@ set(INSTALL_BASE_DIR ".")
set(INSTALL_BIN_DIR "bin")
set(INSTALL_CFG_DIR "config")
set(INSTALL_LIB_DIR "lib")
message("offline debugger does not support macosx system temporarily")
# set package files
install(
TARGETS _c_expression
@ -50,6 +47,14 @@ install(
COMPONENT mindspore
)
if(ENABLE_DEBUGGER)
install(
TARGETS _mindspore_offline_debug
DESTINATION ${INSTALL_BASE_DIR}
COMPONENT mindspore
)
endif()
install(
TARGETS mindspore_shared_lib
DESTINATION ${INSTALL_LIB_DIR}
@ -71,6 +76,25 @@ if(USE_GLOG)
)
endif()
install(FILES ${libevent_LIBPATH}/libevent-2.1.7.dylib
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
install(FILES ${libevent_LIBPATH}/libevent_core-2.1.7.dylib
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
install(FILES ${libevent_LIBPATH}/libevent_extra-2.1.7.dylib
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
install(FILES ${libevent_LIBPATH}/libevent_openssl-2.1.7.dylib
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
install(FILES ${libevent_LIBPATH}/libevent_pthreads-2.1.7.dylib
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
if(ENABLE_CPU AND NOT WIN32)
install(
TARGETS ps_cache
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
endif()
if(MS_BUILD_GRPC)
file(GLOB_RECURSE GRPC_LIB_LIST
${grpc_LIBPATH}/libmindspore*
@ -218,6 +242,14 @@ if(EXISTS ${CMAKE_SOURCE_DIR}/mindspore/dataset)
)
endif()
if(EXISTS ${CMAKE_SOURCE_DIR}/mindspore/offline_debug)
install(
DIRECTORY ${CMAKE_SOURCE_DIR}/mindspore/offline_debug
DESTINATION ${INSTALL_PY_DIR}
COMPONENT mindspore
)
endif()
## Public header files
install(
DIRECTORY ${CMAKE_SOURCE_DIR}/include

9
mindspore/ccsrc/CMakeLists.txt
View File

@ -374,9 +374,12 @@ if(CMAKE_SYSTEM_NAME MATCHES "Windows")
target_link_libraries(mindspore mindspore_gvar)
target_link_libraries(_c_expression PRIVATE -Wl,--whole-archive mindspore mindspore_core -Wl,--no-whole-archive)
elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(mindspore mindspore::pybind11_module)
target_link_libraries(mindspore mindspore_gvar)
target_link_libraries(_c_expression PRIVATE -Wl,-all_load mindspore mindspore_core -Wl,-noall_load)
target_link_libraries(mindspore proto_input mindspore::protobuf
mindspore::event mindspore::event_pthreads mindspore::event_openssl mindspore::eigen mindspore::json)
target_link_libraries(mindspore mindspore::event_core ps_cache)
target_link_libraries(_c_expression PRIVATE -Wl,-all_load mindspore mindspore_core proto_input -Wl,-noall_load)
target_link_libraries(_c_expression PRIVATE mindspore::pybind11_module)
target_link_libraries(_c_expression PRIVATE mindspore_gvar)
else()
if(ENABLE_CPU AND NOT WIN32)
target_link_libraries(mindspore proto_input mindspore::protobuf

2
mindspore/ccsrc/backend/kernel_compiler/CMakeLists.txt
View File

@ -44,7 +44,7 @@ if(ENABLE_CPU)
endif()
endif()
if(NOT ENABLE_CPU OR WIN32 OR APPLE)
if(NOT ENABLE_CPU OR WIN32)
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/apply_momentum_ps_kernel.cc")
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/embedding_look_up_proxy_kernel.cc")
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/embedding_look_up_ps_kernel.cc")

3
mindspore/ccsrc/backend/kernel_compiler/cpu/fl/get_keys_kernel.cc
View File

@ -96,7 +96,8 @@ void GetKeysKernel::BuildGetKeysReq(const std::shared_ptr<fl::FBBuilder> &fbb) {
MS_LOG(INFO) << "BuildGetKeysReq successfully.";
}
bool GetKeysKernel::SavePublicKeyList(auto remote_public_key) {
bool GetKeysKernel::SavePublicKeyList(
const flatbuffers::Vector<flatbuffers::Offset<mindspore::schema::ClientPublicKeys>> *remote_public_key) {
if (remote_public_key == nullptr) {
MS_LOG(EXCEPTION) << "Input remote_pubic_key is nullptr.";
}

3
mindspore/ccsrc/backend/kernel_compiler/cpu/fl/get_keys_kernel.h
View File

@ -41,7 +41,8 @@ class GetKeysKernel : public CPUKernel {
private:
void BuildGetKeysReq(const std::shared_ptr<fl::FBBuilder> &fbb);
bool SavePublicKeyList(auto remote_public_key);
bool SavePublicKeyList(
const flatbuffers::Vector<flatbuffers::Offset<mindspore::schema::ClientPublicKeys>> *remote_public_key);
uint32_t rank_id_;
uint32_t server_num_;

6
mindspore/ccsrc/debug/CMakeLists.txt
View File

@ -33,10 +33,6 @@ if("${ENABLE_HIDDEN}" STREQUAL "OFF")
string(REPLACE " -fvisibility=hidden" " -fvisibility=default" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
endif()
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-private-field")
endif()
if(ENABLE_DEBUGGER)
list(APPEND _DEBUG_SRC_LIST
"${CMAKE_CURRENT_SOURCE_DIR}/debugger/debugger.cc"
@ -56,7 +52,7 @@ if(NOT ENABLE_SECURITY)
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/dump_utils.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/npy_header.cc"
)
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows" AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
list(APPEND _DEBUG_SRC_LIST
"${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/e2e_dump.cc"

17
mindspore/ccsrc/debug/debug_services.cc
View File

@ -37,10 +37,19 @@
#include "nlohmann/json.hpp"
#include "debug/debugger/tensor_summary.h"
#include "utils/file_utils.h"
#include "linux/limits.h"
#include "climits"
#ifdef ONLINE_DBG_MODE
namespace mindspore {
#endif
namespace {
#ifdef __APPLE__
constexpr int kStrErrorNone = 0;
#else
constexpr char *kStrErrorNone = nullptr;
#endif
} // namespace
DebugServices::DebugServices() { tensor_loader_ = std::make_shared<TensorLoader>(); }
DebugServices::DebugServices(const DebugServices &other) {
@ -632,7 +641,7 @@ void DebugServices::ReadTensorFromNpy(const std::string &tensor_name, const std:
const int kMaxFilenameLength = 128;
char err_info[kMaxFilenameLength];
auto ret = strerror_r(errno, err_info, sizeof(err_info));
if (ret != nullptr) {
if (ret != kStrErrorNone) {
MS_LOG(ERROR) << " ErrInfo:" << ret;
}
return;
@ -1059,7 +1068,7 @@ void DebugServices::ReadGraphRunIter(std::string file_path, std::tuple<uint32_t,
MS_LOG(ERROR) << "Failed to open file (In ReadGraphRunIter) " << file_path << " Errno:" << errno;
const int kMaxFilenameLength = NAME_MAX;
char err_info[kMaxFilenameLength];
if (strerror_r(errno, err_info, sizeof(err_info)) != nullptr) {
if (strerror_r(errno, err_info, sizeof(err_info)) != kStrErrorNone) {
MS_LOG(ERROR) << " ErrInfo:" << strerror_r(errno, err_info, sizeof(err_info));
}
@ -1190,7 +1199,7 @@ void DebugServices::ReadFileAndAddToTensor(const bool found, const std::vector<s
std::vector<std::shared_ptr<TensorData>> *result_list) {
std::string time_stamp = "";
std::string type_name = "";
uint64_t data_size = 0;
size_t data_size = 0;
std::vector<int64_t> shape;
std::vector<char> *buffer = nullptr;
if (found) {

6
mindspore/ccsrc/debug/tensor_load.h
View File

@ -32,7 +32,11 @@ namespace mindspore {
#endif
class TensorLoader {
public:
#ifndef __APPLE__
TensorLoader() : iter_num_(-1), mem_total_(0), mem_usage_(0) {}
#else
TensorLoader() : mem_total_(0), mem_usage_(0) {}
#endif
~TensorLoader() { EmptyTensor(); }
@ -227,7 +231,9 @@ class TensorLoader {
// the pair is (device_id, iteration)
std::map<std::string, std::shared_ptr<TensorData>> tensor_list_map_;
std::map<std::string, std::shared_ptr<TensorData>> prev_tensor_list_map_;
#ifndef __APPLE__
uint32_t iter_num_;
#endif
std::mutex lock_;
std::mutex mem_lock_;
uint64_t mem_total_;

6
mindspore/ccsrc/fl/CMakeLists.txt
View File

@ -1,6 +1,6 @@
file(GLOB_RECURSE _FL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
if(NOT ENABLE_CPU OR WIN32 OR APPLE)
if(NOT ENABLE_CPU OR WIN32)
list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/apply_momentum_kernel.cc")
list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/aggregation_kernel_factory.cc")
list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/dense_grad_accum_kernel.cc")
@ -49,6 +49,10 @@ if(NOT ENABLE_CPU OR WIN32 OR APPLE)
list(REMOVE_ITEM _FL_SRC_FILES "armour/cipher/cipher_unmask.cc")
endif()
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-tautological-pointer-compare")
endif()
list(LENGTH _FL_SRC_FILES fl_file_num)
if(NOT fl_file_num EQUAL 0)
set_property(SOURCE ${_FL_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_FL)

4
mindspore/ccsrc/fl/worker/fl_worker.cc
View File

@ -104,10 +104,14 @@ bool FLWorker::SendToServer(uint32_t server_rank, const void *data, size_t size,
std::this_thread::yield();
}
#ifdef __APPLE__
std::shared_ptr<unsigned char> message(new unsigned char[size], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> message;
std::unique_ptr<unsigned char[]> message_addr = std::make_unique<unsigned char[]>(size);
MS_EXCEPTION_IF_NULL(message_addr);
message = std::move(message_addr);
#endif
MS_EXCEPTION_IF_NULL(message);
uint64_t src_size = size;

94
mindspore/ccsrc/minddata/dataset/CMakeLists.txt
View File

@ -74,9 +74,7 @@ add_subdirectory(core)
add_subdirectory(kernels)
add_subdirectory(engine)
add_subdirectory(api)
if(NOT APPLE)
add_subdirectory(audio)
endif()
add_subdirectory(audio)
add_subdirectory(text)
add_subdirectory(callback)
add_subdirectory(plugin)
@ -91,16 +89,12 @@ add_dependencies(engine-datasetops-source-sampler core)
add_dependencies(engine-datasetops core)
add_dependencies(engine-datasetops-mapop core)
add_dependencies(engine-opt core)
if(NOT APPLE)
add_dependencies(engine-gnn core)
endif()
add_dependencies(engine-gnn core)
add_dependencies(engine core)
add_dependencies(callback core)
if(NOT APPLE)
add_dependencies(audio-kernels core)
add_dependencies(audio-ir core)
add_dependencies(audio-ir-kernels core)
endif()
add_dependencies(audio-kernels core)
add_dependencies(audio-ir core)
add_dependencies(audio-ir-kernels core)
add_dependencies(text core)
add_dependencies(text-kernels core)
add_dependencies(text-ir core)
@ -161,10 +155,14 @@ set(submodules
$<TARGET_OBJECTS:engine-datasetops-source>
$<TARGET_OBJECTS:engine-datasetops-source-sampler>
$<TARGET_OBJECTS:engine-datasetops-mapop>
$<TARGET_OBJECTS:engine-gnn>
$<TARGET_OBJECTS:engine-datasetops>
$<TARGET_OBJECTS:engine-opt>
$<TARGET_OBJECTS:engine-cache-client>
$<TARGET_OBJECTS:engine>
$<TARGET_OBJECTS:audio-kernels>
$<TARGET_OBJECTS:audio-ir>
$<TARGET_OBJECTS:audio-ir-kernels>
$<TARGET_OBJECTS:text>
$<TARGET_OBJECTS:text-kernels>
$<TARGET_OBJECTS:text-ir>
@ -175,15 +173,6 @@ set(submodules
$<TARGET_OBJECTS:md_plugin>
)
if(NOT APPLE)
set(submodules
${submodules}
$<TARGET_OBJECTS:engine-gnn>
$<TARGET_OBJECTS:audio-kernels>
$<TARGET_OBJECTS:audio-ir>
$<TARGET_OBJECTS:audio-ir-kernels>)
endif()
if(ENABLE_ACL)
set(submodules
${submodules}
@ -223,61 +212,6 @@ endif()
################# Link with external libraries ########################
target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar)
set(targetso $<TARGET_FILE:_c_dataengine>)
set(target_tiny "libtinyxml")
set(target_icuuc "libicuuc")
set(target_icudata "libicudata")
set(target_icui18n "libicui18n")
function(changerpath targetso linkso)
set(some-file "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/${linkso}.txt")
set(some-file1 "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/${linkso}1.txt")
set(some-file2 "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/${linkso}2.txt")
add_custom_command(
OUTPUT
${some-file}
COMMAND
otool -L ${targetso} | grep ${linkso} > ${some-file}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset"
VERBATIM
)
add_custom_command(
OUTPUT
${some-file1}
COMMAND
cat ${some-file} | cut -d " " -f 1 | sed -E "s/^.//g" > ${some-file1}
DEPENDS
${some-file}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset"
VERBATIM
)
add_custom_command(
OUTPUT
${some-file2}
COMMAND
awk -F "/" "{print $NF}" ${some-file1} > ${some-file2}
DEPENDS
${some-file1}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset"
VERBATIM
)
add_custom_target(
link_${linkso} ALL
COMMAND install_name_tool -change `cat ${some-file1}` @rpath/`cat ${some-file2}` ${targetso}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset"
DEPENDS ${targetso} ${some-file1} ${some-file2}
COMMENT "install tool name")
endfunction()
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
changerpath(${targetso} ${target_tiny})
changerpath(${targetso} ${target_icuuc})
add_dependencies(link_${target_icuuc} link_${target_tiny})
changerpath(${targetso} ${target_icudata})
add_dependencies(link_${target_icudata} link_${target_icuuc})
changerpath(${targetso} ${target_icui18n})
add_dependencies(link_${target_icui18n} link_${target_icudata})
endif()
if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
if(ENABLE_PYTHON)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY})
@ -320,9 +254,7 @@ else()
if(${ENABLE_IBVERBS} STREQUAL "ON")
target_link_libraries(_c_dataengine PRIVATE ibverbs rdmacm)
endif()
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(_c_dataengine PRIVATE ps_cache)
endif()
target_link_libraries(_c_dataengine PRIVATE ps_cache)
endif()
endif()
@ -349,3 +281,9 @@ endif()
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
set_target_properties(_c_dataengine PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})
endif()
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
include(${CMAKE_SOURCE_DIR}/cmake/change_rpath.cmake)
set(LINK_SOS libtinyxml libicuuc libicudata libicui18n)
changerpath($<TARGET_FILE:_c_dataengine> _c_dataengine "${LINK_SOS}")
endif()

77
mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt
View File

@ -1,55 +1,42 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
if(ENABLE_PYTHON)
set(API_PYTHON_SRC_FILES
python/bindings/dataset/audio/bindings.cc
python/bindings/dataset/audio/kernels/ir/bindings.cc
python/bindings/dataset/callback/bindings.cc
python/bindings/dataset/core/bindings.cc
python/bindings/dataset/engine/cache/bindings.cc
python/bindings/dataset/engine/datasetops/bindings.cc
python/bindings/dataset/engine/gnn/bindings.cc
python/bindings/dataset/engine/ir/consumer/bindings.cc
python/bindings/dataset/engine/ir/datasetops/bindings.cc
python/bindings/dataset/engine/ir/datasetops/source/bindings.cc
python/bindings/dataset/engine/ir/datasetops/source/samplers/bindings.cc
python/bindings/dataset/engine/ir/execute/bindings.cc
python/bindings/dataset/engine/ir/schema/bindings.cc
python/bindings/dataset/engine/perf/bindings.cc
python/bindings/dataset/kernels/bindings.cc
python/bindings/dataset/kernels/ir/bindings.cc
python/bindings/dataset/kernels/ir/image/bindings.cc
python/bindings/dataset/text/bindings.cc
python/bindings/dataset/text/kernels/ir/bindings.cc
python/bindings/mindrecord/include/bindings.cc
python/pybind_conversion.cc
python/pybind_register.cc
)
if(APPLE)
list(REMOVE_ITEM API_PYTHON_SRC_FILES
add_library(APItoPython OBJECT
python/bindings/dataset/audio/bindings.cc
python/bindings/dataset/audio/kernels/ir/bindings.cc
python/bindings/dataset/callback/bindings.cc
python/bindings/dataset/core/bindings.cc
python/bindings/dataset/engine/cache/bindings.cc
python/bindings/dataset/engine/datasetops/bindings.cc
python/bindings/dataset/engine/gnn/bindings.cc
python/bindings/dataset/engine/ir/consumer/bindings.cc
python/bindings/dataset/engine/ir/datasetops/bindings.cc
python/bindings/dataset/engine/ir/datasetops/source/bindings.cc
python/bindings/dataset/engine/ir/datasetops/source/samplers/bindings.cc
python/bindings/dataset/engine/ir/execute/bindings.cc
python/bindings/dataset/engine/ir/schema/bindings.cc
python/bindings/dataset/engine/perf/bindings.cc
python/bindings/dataset/kernels/bindings.cc
python/bindings/dataset/kernels/ir/bindings.cc
python/bindings/dataset/kernels/ir/image/bindings.cc
python/bindings/dataset/text/bindings.cc
python/bindings/dataset/text/kernels/ir/bindings.cc
python/bindings/mindrecord/include/bindings.cc
python/pybind_conversion.cc
python/pybind_register.cc
)
endif()
add_library(APItoPython OBJECT ${API_PYTHON_SRC_FILES})
target_include_directories(APItoPython PRIVATE ${pybind11_INCLUDE_DIRS})
endif()
set(API_CPP_SRC_FILES
audio.cc
config.cc
data_helper.cc
datasets.cc
execute.cc
iterator.cc
transforms.cc
samplers.cc
text.cc
vision.cc
)
if(APPLE)
list(REMOVE_ITEM API_CPP_SRC_FILES audio.cc)
endif()
add_library(cpp-API OBJECT ${API_CPP_SRC_FILES})
add_library(cpp-API OBJECT
audio.cc
config.cc
data_helper.cc
datasets.cc
execute.cc
iterator.cc
transforms.cc
samplers.cc
text.cc
vision.cc
)

2
mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.h
View File

@ -937,7 +937,7 @@ Status Flanger(const std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *out
cur_channel_phase[j] = static_cast<int>(j * lfo_length * channel_phase + 0.5);
// through the current channel phase and lfo arrays to get the delay
auto iter_lfo = lfo->begin<float>();
delay_tensor[j] = *(iter_lfo + (lfo_pos + cur_channel_phase[j]) % lfo_length);
delay_tensor[j] = *(iter_lfo + static_cast<ptrdiff_t>((lfo_pos + cur_channel_phase[j]) % lfo_length));
// the frac delay is obtained by using the frac function
frac_delay[j] = delay_tensor[j] - static_cast<int>(delay_tensor[j]);
delay_tensor[j] = floor(delay_tensor[j]);

8
mindspore/ccsrc/minddata/dataset/engine/CMakeLists.txt
View File

@ -1,8 +1,6 @@
add_subdirectory(datasetops)
add_subdirectory(opt)
if(NOT APPLE)
add_subdirectory(gnn)
endif()
add_subdirectory(gnn)
add_subdirectory(ir)
message("ENABLE_SECURITY = ${ENABLE_SECURITY}")
@ -49,12 +47,10 @@ add_dependencies(engine
engine-datasetops
engine-datasetops-source
engine-opt
engine-gnn
engine-cache-client
engine-datasetops-mapop
)
if(NOT APPLE)
add_dependencies(engine engine-gnn)
endif()
if(NOT ENABLE_SECURITY)
add_dependencies(engine engine-perf)

4
mindspore/ccsrc/minddata/mindrecord/CMakeLists.txt
View File

@ -48,4 +48,6 @@ else()
endif()
endif()
set_target_properties(_c_mindrecord PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
set_target_properties(_c_mindrecord PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})
endif()

10
mindspore/ccsrc/pipeline/jit/action.cc
View File

@ -46,7 +46,7 @@
#include "frontend/optimizer/py_pass_manager.h"
#include "utils/ms_context.h"
#include "vm/transform.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/parameter_server.h"
#include "ps/scheduler.h"
#include "ps/worker.h"
@ -623,7 +623,7 @@ bool OptInlineAction(const ResourcePtr &res) {
bool GeOptimizeAction(const ResourcePtr &res) { return OptimizeAction(res, kGePasses); }
bool VmOptimizeAction(const ResourcePtr &res) {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_ps_mode()) {
kVmPasses.push_back({"server_communication_op_fusion", ps::Util::FuseServerCommOps});
}
@ -823,7 +823,7 @@ bool ExecuteAction(const ResourcePtr &res) {
return true;
}
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
bool StartPSWorkerAction(const ResourcePtr &) {
ps::Worker::GetInstance().Run();
return true;
@ -1163,7 +1163,7 @@ std::vector<ActionItem> VmPipeline() {
(void)actions.emplace_back(std::make_pair("eliminate_forward_cnode", EliminateForwardCNode));
(void)actions.emplace_back(std::make_pair("validate", ValidateAction));
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_worker()) {
std::string server_mode = ps::PSContext::instance()->server_mode();
if (server_mode == ps::kServerModeFL || server_mode == ps::kServerModeHybrid) {
@ -1201,7 +1201,7 @@ std::vector<ActionItem> MindIRPipeline() {
(void)actions.emplace_back(std::make_pair("execute", ExecuteAction));
return actions;
}
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
std::vector<ActionItem> ServerPipeline() {
auto actions = CommonPipeline();
(void)actions.emplace_back(std::make_pair("optimize", VmOptimizeAction));

2
mindspore/ccsrc/pipeline/jit/init.cc
View File

@ -36,7 +36,7 @@
#else
#include "runtime/device/gpu/distribution/collective_fake_init.h"
#endif
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/util.h"
#endif
#include "ps/ps_context.h"

6
mindspore/ccsrc/pipeline/jit/pass.cc
View File

@ -49,7 +49,7 @@
#include "frontend/optimizer/irpass/gradient_eliminate.h"
#include "frontend/optimizer/irpass/parameter_eliminate.h"
#include "frontend/optimizer/irpass/updatestate_eliminate.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/util.h"
#include "ps/ps_context.h"
#endif
@ -218,7 +218,7 @@ namespace {
bool ReAutoMonadWrapper(const FuncGraphPtr &root, const opt::OptimizerPtr &) { return ReAutoMonad(root); }
bool parallel_mode() {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_server() || ps::PSContext::instance()->is_scheduler()) {
return false;
}
@ -603,7 +603,7 @@ bool SliceRecomputeActivationPass(const ResourcePtr &res) {
bool AddCacheEmbeddingPass(const ResourcePtr &res) {
MS_EXCEPTION_IF_NULL(res);
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_ps_mode()) {
return true;
}

8
mindspore/ccsrc/pipeline/jit/pipeline.cc
View File

@ -66,7 +66,7 @@
#endif
#endif
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/constants.h"
#include "ps/util.h"
#include "ps/worker.h"
@ -881,7 +881,7 @@ std::vector<ActionItem> GetPipeline(const ResourcePtr &resource, const std::stri
std::string backend = MsContext::GetInstance()->backend_policy();
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
const std::string &server_mode = ps::PSContext::instance()->server_mode();
if ((server_mode == ps::kServerModeFL || server_mode == ps::kServerModeHybrid) &&
ps::PSContext::instance()->is_server()) {
@ -1442,7 +1442,7 @@ bool InitExecDataset(const std::string &queue_name, int64_t iter_num, int64_t ba
bool InitExecDatasetVm(const std::string &queue_name, int64_t size, int64_t batch_size,
const std::vector<TypePtr> &types, const std::vector<std::vector<int64_t>> &shapes,
const std::vector<int64_t> &input_indexes, bool need_run) {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if ((ps::PSContext::instance()->is_ps_mode()) && (!ps::PSContext::instance()->is_worker())) {
return true;
}
@ -1661,7 +1661,7 @@ void FinalizeBackend() {
void ClearResAtexit() {
MS_LOG(DEBUG) << "Pipeline clear all resource";
RecordExitStatus();
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_ps_mode() && ps::PSContext::instance()->is_worker()) {
if (ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps::ps_cache_instance.Finalize();

2
mindspore/ccsrc/ps/CMakeLists.txt
View File

@ -1,6 +1,6 @@
file(GLOB_RECURSE _PS_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
if(NOT ENABLE_CPU OR WIN32 OR APPLE)
if(NOT ENABLE_CPU OR WIN32)
list(REMOVE_ITEM _PS_SRC_FILES "optimizer_info_builder.cc")
list(REMOVE_ITEM _PS_SRC_FILES "optimizer_info.cc")
list(REMOVE_ITEM _PS_SRC_FILES "scheduler.cc")

6
mindspore/ccsrc/ps/constants.h
View File

@ -192,9 +192,13 @@ const std::map<std::string, size_t> kCiphers = {{"ECDHE-RSA-AES128-GCM-SHA256",
{"!PSK", 27},
{"kEDH+AESGCM", 28}};
#ifdef __APPLE__
using DataPtr = std::shared_ptr<unsigned char>;
#else
using DataPtr = std::shared_ptr<unsigned char[]>;
#endif
using VectorPtr = std::shared_ptr<std::vector<unsigned char>>;
using Key = uint64_t;
using Key = size_t;
using Keys = std::vector<Key>;
using Values = std::vector<float>;
using ValuesPtr = std::shared_ptr<Values>;

4
mindspore/ccsrc/ps/core/abstract_node.cc
View File

@ -956,7 +956,11 @@ void AbstractNode::ProcessSendData(const std::shared_ptr<TcpConnection> &conn, c
MS_EXCEPTION_IF_NULL(conn);
MS_EXCEPTION_IF_NULL(meta);
MS_EXCEPTION_IF_NULL(data);
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[size], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[size]);
#endif
if (size > 0) {
size_t dest_size = size;
size_t src_size = size;

4
mindspore/ccsrc/ps/core/abstract_node.h
View File

@ -61,7 +61,11 @@ class AbstractNode : public Node {
const std::shared_ptr<MessageMeta> &meta, const Protos &protos,
const void *data, size_t size);
#ifdef __APPLE__
using DataPtr = std::shared_ptr<unsigned char>;
#else
using DataPtr = std::shared_ptr<unsigned char[]>;
#endif
using VectorPtr = std::shared_ptr<std::vector<unsigned char>>;
using RequestHandler =
std::function<void(const std::shared_ptr<TcpConnection> &conn, const std::shared_ptr<MessageMeta> &meta,

4
mindspore/ccsrc/ps/core/communicator/tcp_communicator.h
View File

@ -84,7 +84,11 @@ class TcpCommunicator : public CommunicatorBase {
bool SendPbRequest(const T &pb_msg, const uint32_t &rank_id, TcpUserCommand command,
std::shared_ptr<std::vector<unsigned char>> *output = nullptr) {
const std::string &msg_str = pb_msg.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> msg(new unsigned char[msg_str.size()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> msg(new unsigned char[msg_str.size()]);
#endif
MS_ERROR_IF_NULL_W_RET_VAL(msg, false);
size_t dest_size = msg_str.size();
size_t src_size = msg_str.size();

2
mindspore/ccsrc/ps/core/scheduler_recovery.h
View File

@ -49,7 +49,7 @@ class SchedulerRecovery : public RecoveryBase {
private:
// The node_ will only be instantiated with worker/server node.
SchedulerNode *const node_;
SchedulerNode *const node_ = nullptr;
};
} // namespace core
} // namespace ps

5
mindspore/ccsrc/ps/ps_cache/CMakeLists.txt
View File

@ -1,7 +1,6 @@
if(ENABLE_CPU AND NOT WIN32 AND NOT APPLE)
if(ENABLE_CPU AND NOT WIN32)
file(GLOB_RECURSE _PS_CACHE_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "ps_data/*.cc")
set_property(SOURCE ${_PS_CACHE_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PS)
add_library(ps_cache SHARED ${_PS_CACHE_SRC_FILES})
target_link_libraries(ps_cache mindspore::pybind11_module)
endif()

5
mindspore/ccsrc/ps/ps_cache/ps_cache_manager.cc
View File

@ -231,9 +231,14 @@ void PsCacheManager::AllocMemForHashTable() {
device_address.addr = addr;
auto &host_address = item.second.host_address;
#ifdef __APPLE__
host_address =
std::shared_ptr<float>(new float[host_vocab_cache_size_ * embedding_size], std::default_delete<float[]>());
#else
std::unique_ptr<float[]> host_hash_table_addr = std::make_unique<float[]>(host_vocab_cache_size_ * embedding_size);
MS_EXCEPTION_IF_NULL(host_hash_table_addr);
host_address = std::move(host_hash_table_addr);
#endif
MS_EXCEPTION_IF_NULL(host_address);
max_embedding_size = (embedding_size > max_embedding_size) ? embedding_size : max_embedding_size;

4
mindspore/ccsrc/ps/ps_cache/ps_cache_manager.h
View File

@ -49,7 +49,11 @@ struct HashTableInfo {
size_t embedding_size{0};
size_t vocab_size{0};
Address device_address{nullptr, 0};
#ifdef __APPLE__
std::shared_ptr<float> host_address{nullptr};
#else
std::shared_ptr<float[]> host_address{nullptr};
#endif
ParamInitInfo param_init_info_;
};

18
mindspore/ccsrc/ps/ps_context.cc
View File

@ -18,7 +18,7 @@
#include "utils/log_adapter.h"
#include "utils/ms_utils.h"
#include "backend/kernel_compiler/kernel.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/ps_cache/ps_cache_manager.h"
#include "ps/ps_cache/ps_data/ps_data_prefetch.h"
#endif
@ -88,7 +88,7 @@ void PSContext::Reset() {
is_worker_ = false;
is_pserver_ = false;
is_sched_ = false;
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps_cache_instance.Finalize();
set_cache_enable(false);
@ -144,44 +144,44 @@ uint32_t PSContext::ps_rank_id() const { return rank_id_; }
void PSContext::InsertHashTableSize(const std::string &param_name, size_t cache_vocab_size, size_t embedding_size,
size_t vocab_size) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
ps_cache_instance.InsertHashTableSize(param_name, cache_vocab_size, embedding_size, vocab_size);
#endif
}
void PSContext::ReInsertHashTableSize(const std::string &new_param_name, const std::string &cur_param_name,
size_t cache_vocab_size, size_t embedding_size) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
ps_cache_instance.ReInsertHashTableSize(new_param_name, cur_param_name, cache_vocab_size, embedding_size);
#endif
}
void PSContext::InsertWeightInitInfo(const std::string &param_name, size_t global_seed, size_t op_seed) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
ps_cache_instance.InsertWeightInitInfo(param_name, global_seed, op_seed);
#endif
}
void PSContext::InsertAccumuInitInfo(const std::string &param_name, float init_val) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
ps_cache_instance.InsertAccumuInitInfo(param_name, init_val);
#endif
}
void PSContext::CloneHashTable(const std::string &dest_param_name, const std::string &src_param_name) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
ps_cache_instance.CloneHashTable(dest_param_name, src_param_name);
#endif
}
void PSContext::set_cache_enable(bool cache_enable) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
PsDataPrefetch::GetInstance().set_cache_enable(cache_enable);
#endif
}
void PSContext::set_rank_id(uint32_t rank_id) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
ps_cache_instance.set_rank_id(rank_id);
#endif
}

29
mindspore/ccsrc/ps/worker.cc
View File

@ -227,7 +227,11 @@ bool Worker::InitPSEmbeddingTable(const size_t &key, const std::vector<size_t> &
std::string kv_data = embedding_table_meta.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
#endif
size_t dest_size = kv_data.length();
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
if (ret != 0) {
@ -293,8 +297,11 @@ bool Worker::DoPSEmbeddingLookup(const Key &key, const std::vector<int> &lookup_
if (messages.at(i).first) {
rank_ids.push_back(i);
std::string kv_data = messages.at(i).second.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
#endif
size_t dest_size = kv_data.length();
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
if (ret != 0) {
@ -382,7 +389,11 @@ bool Worker::UpdateEmbeddingTable(const std::vector<Key> &keys, const std::vecto
rank_ids.push_back(i);
std::string kv_data = messages.at(i).second.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
#endif
size_t dest_size = kv_data.length();
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
if (ret != 0) {
@ -403,7 +414,11 @@ void Worker::Finalize() {
kvs.add_keys(0);
kvs.add_values(0.0f);
std::string kv_data = kvs.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
#endif
size_t dest_size = kv_data.length();
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
if (ret != 0) {
@ -644,7 +659,11 @@ void Worker::PushData(const std::vector<Key> &keys, const std::vector<float> &va
SendForPush(cmd, kvs, worker_init_embedding_partitioner_, {});
} else {
std::string kv_data = kvs.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
#endif
size_t dest_size = kv_data.length();
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
if (ret != 0) {
@ -950,7 +969,11 @@ void Worker::SendForPush(int cmd, const KVMessage &send, const KVPartitioner &pa
rank_ids.push_back(i);
std::string kv_data = messages.at(i).second.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
#endif
size_t dest_size = kv_data.length();
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
if (ret != 0) {
@ -977,7 +1000,11 @@ void Worker::SendForPull(int cmd, const KVMessage &send, const KVPartitioner &pa
rank_ids.push_back(i);
std::string kv_data = messages.at(i).second.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
#endif
size_t dest_size = kv_data.length();
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
if (ret != 0) {

8
mindspore/ccsrc/runtime/device/kernel_runtime.cc
View File

@ -33,7 +33,7 @@
#include "frontend/parallel/context.h"
#include "debug/env_config_parser.h"
#include "pipeline/pynative/pynative_profiling.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/ps_cache/ps_cache_manager.h"
#endif
@ -576,7 +576,7 @@ void KernelRuntime::AssignStaticMemoryInput(const session::KernelGraph &graph) {
}
add_need_alloc_nodes(input_node);
}
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
bool ps_cache_check = false;
#endif
for (auto &item : need_alloc_nodes) {
@ -590,7 +590,7 @@ void KernelRuntime::AssignStaticMemoryInput(const session::KernelGraph &graph) {
continue;
}
DeviceAddressPtr device_address = nullptr;
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
const std::string &param_name = item->fullname_with_scope();
if (ps::ps_cache_instance.IsHashTable(param_name)) {
MS_LOG(INFO) << "Parameter(" << param_name << ")"
@ -1598,7 +1598,7 @@ void KernelRuntime::ClearGraphRuntimeResource(uint32_t graph_id) {
MS_LOG(INFO) << "Clear graph:" << graph_id << " runtime resource";
}
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
void KernelRuntime::GetFirstPSEmbeddingCache(const session::KernelGraph &graph,
AnfNodePtr *const first_cache_input_index,
size_t *const first_cache_size) {

2
mindspore/ccsrc/runtime/device/kernel_runtime.h
View File

@ -173,7 +173,7 @@ class KernelRuntime {
void RunOpAssignOutputNodeMemory(const ValuePtr &pre_output_value, const session::KernelGraph &graph);
void AssignValueNodeTensor(const ValueNodePtr &value_node, const ValuePtr &node_value, size_t output_idx);
DeviceAddressPtr PreAssignCNodeMemory(const AnfNodePtr &anf_node, size_t index) const;
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
void GetFirstPSEmbeddingCache(const session::KernelGraph &graph, AnfNodePtr *const first_cache_input_index,
size_t *const first_cache_size);
void CheckIfSupportPSEmbeddingCache(const session::KernelGraph &graph);

6
mindspore/ccsrc/runtime/device/kernel_runtime_manager.cc
View File

@ -16,7 +16,7 @@
#include "runtime/device/kernel_runtime_manager.h"
#include "utils/log_adapter.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/ps_cache/ps_cache_manager.h"
#endif
#include "backend/session/pynative_task_manager.h"
@ -26,7 +26,7 @@ namespace device {
void KernelRuntimeManager::ClearRuntimeResource() {
// Just remove PyNative tasks before runtime resource release.
session::PynativeTaskManager::GetInstance().Reset();
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_worker() && ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps::ps_cache_instance.SyncEmbeddingTable();
}
@ -132,7 +132,7 @@ void KernelRuntimeManager::ReleaseKernelRuntime(const std::string &device_name,
if (runtime == nullptr) {
return;
}
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_worker() && ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps::ps_cache_instance.SyncEmbeddingTable();
}

2
mindspore/ccsrc/runtime/framework/actor/super_kernel_actor.cc
View File

@ -67,7 +67,7 @@ void SuperKernelActor::Run(OpContext<DeviceTensor> *const context) {
MS_LOG(INFO) << "Super kernel actor(" << GetAID().Name()
<< ") launches graph: " << std::to_string(graph_->graph_id());
if (!CopyInputData(context)) {
std::string error_info = "Copy the input data failed, graph id: " + graph_->graph_id();
std::string error_info = "Copy the input data failed, graph id: " + std::to_string(graph_->graph_id());
SET_OPCONTEXT_FAIL_RET_WITH_ERROR((*context), error_info);
}

2
mindspore/ccsrc/vm/transform.cc
View File

@ -632,7 +632,7 @@ void SetMindRTEnable() {
return;
#endif
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_ps_mode()) {
return;
}

4
mindspore/dataset/__init__.py
View File

@ -31,15 +31,13 @@ Common imported modules in corresponding API examples are as follows:
from mindspore.dataset.transforms import c_transforms
"""
import platform
from .core import config
from .engine import *
from .engine.cache_client import DatasetCache
from .engine.datasets import *
from .engine.graphdata import GraphData
from .engine.samplers import *
from .engine.serializer_deserializer import compare, deserialize, serialize, show
if platform.system().lower() != "darwin":
from .engine.graphdata import GraphData
__all__ = []
__all__.extend(engine.__all__)

8
mindspore/dataset/engine/__init__.py
View File

@ -22,23 +22,19 @@ high performance and parse data precisely. It also provides the following
operations for users to preprocess data: shuffle, batch, repeat, map, and zip.
"""
import platform
from ..callback import DSCallback, WaitedDSCallback
from ..core import config
from .cache_client import DatasetCache
from .datasets import *
from .graphdata import GraphData, SamplingStrategy, OutputFormat
from .iterators import *
from .samplers import *
from .serializer_deserializer import compare, deserialize, serialize, show
if platform.system().lower() != "darwin":
from .graphdata import GraphData, SamplingStrategy, OutputFormat
__all__ = ["CelebADataset", "Cifar100Dataset", "Cifar10Dataset", "CLUEDataset", "CocoDataset", "CSVDataset",
"GeneratorDataset", "ImageFolderDataset", "ManifestDataset", "MindDataset", "MnistDataset",
"GeneratorDataset", "GraphData", "ImageFolderDataset", "ManifestDataset", "MindDataset", "MnistDataset",
"NumpySlicesDataset", "PaddedDataset", "TextFileDataset", "TFRecordDataset", "VOCDataset",
"DistributedSampler", "PKSampler", "RandomSampler", "SequentialSampler", "SubsetRandomSampler",
"WeightedRandomSampler", "SubsetSampler",
"DatasetCache", "DSCallback", "Schema", "WaitedDSCallback", "compare", "deserialize",
"serialize", "show", "zip"]
if platform.system().lower() != "darwin":
__all__.append("GraphData")

34
third_party/patch/grpc/grpc.patch001 vendored
View File

@ -1,15 +1,21 @@
diff -Npur grpc/..rej grpc-patch/..rej
--- grpc/..rej 1970-01-01 08:00:00.000000000 +0800
+++ grpc-patch/..rej 2021-04-22 21:00:17.343178600 +0800
@@ -0,0 +1,22 @@
@@ -0,0 +1,28 @@
+--- CMakeLists.txt 2020-02-27 03:12:33.000000000 +0800
++++ CMakeLists.txt 2021-04-07 21:27:12.317207600 +0800
+@@ -12992,7 +12992,7 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
+@@ -12992,7 +12992,13 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
+ add_executable(grpc_cpp_plugin
+ src/compiler/cpp_plugin.cc
+ )
+-
++set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
++if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
++ set(CMAKE_MACOSX_RPATH 1)
++ set(CMAKE_INSTALL_RPATH "@executable_path/../lib")
++ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}")
++else()
++ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
++endif()
+ target_include_directories(grpc_cpp_plugin
+ PRIVATE
+ ${CMAKE_CURRENT_SOURCE_DIR}
@ -27,15 +33,21 @@ diff -Npur grpc/..rej grpc-patch/..rej
diff -Npur grpc/.rej grpc-patch/.rej
--- grpc/.rej 1970-01-01 08:00:00.000000000 +0800
+++ grpc-patch/.rej 2021-04-22 21:03:38.192349100 +0800
@@ -0,0 +1,22 @@
@@ -0,0 +1,28 @@
+--- grpc/CMakeLists.txt 2020-02-27 03:12:33.000000000 +0800
++++ grpc-patch/CMakeLists.txt 2021-04-07 21:27:12.317207600 +0800
+@@ -12992,7 +12992,7 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
+@@ -12992,7 +12992,13 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
+ add_executable(grpc_cpp_plugin
+ src/compiler/cpp_plugin.cc
+ )
+-
++set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
++if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
++ set(CMAKE_MACOSX_RPATH 1)
++ set(CMAKE_INSTALL_RPATH "@executable_path/../lib")
++ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}")
++else()
++ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
++endif()
+ target_include_directories(grpc_cpp_plugin
+ PRIVATE
+ ${CMAKE_CURRENT_SOURCE_DIR}
@ -107,12 +119,18 @@ diff -Npur grpc/CMakeLists.txt grpc-patch/CMakeLists.txt
if(WIN32 AND MSVC)
set_target_properties(upb PROPERTIES COMPILE_PDB_NAME "upb"
COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}"
@@ -12992,7 +13004,7 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
@@ -12992,7 +13004,13 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
add_executable(grpc_cpp_plugin
src/compiler/cpp_plugin.cc
)
-
+set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
+if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
+ set(CMAKE_MACOSX_RPATH 1)
+ set(CMAKE_INSTALL_RPATH "@executable_path/../lib")
+ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}")
+else()
+ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
+endif()
target_include_directories(grpc_cpp_plugin
PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}

5
third_party/patch/libevent/libevent.patch001 vendored
View File

@ -10,3 +10,8 @@ diff -Npur libevent/CMakeLists.txt libevent-modify/CMakeLists.txt
endif()
if (NOT EVENT__DISABLE_THREAD_SUPPORT)
diff -Npur libevent/cmake/AddEventLibrary.cmake libevent-modify/cmake/AddEventLibrary.cmake
--- libevent/cmake/AddEventLibrary.cmake 2020-07-05 20:02:46.000000000 +0800
+++ libevent-modify/cmake/AddEventLibrary.cmake 2021-04-19 16:36:57.982307500 +0800
@@ -153,1 +153,0 @@
- INSTALL_NAME_DIR "${CMAKE_INSTALL_PREFIX}/lib"