fix mindspore mac compile

fix debugger, mindarmour, audio, gnn, grpc, ps/fl.
This commit is contained in:
xulei 2021-11-25 10:38:43 +08:00
parent 745c1eaff8
commit 78b37afc30
43 changed files with 317 additions and 215 deletions

View File

@ -20,7 +20,7 @@ endif()
if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(CMAKE_OSX_SYSROOT "")
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Winconsistent-missing-override -Wuser-defined-warnings \
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Winconsistent-missing-override -Wno-user-defined-warnings \
-Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare \
-Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move \
-Wno-inconsistent-missing-override -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2")

54
cmake/change_rpath.cmake Normal file
View File

@ -0,0 +1,54 @@
function(changerpath target_so target_name link_sos)
set(depend_so "")
foreach(link_so ${link_sos})
set(some-file "${CMAKE_SOURCE_DIR}/build/${target_name}_${link_so}.txt")
set(some-file1 "${CMAKE_SOURCE_DIR}/build/${target_name}_${link_so}1.txt")
set(some-file2 "${CMAKE_SOURCE_DIR}/build/${target_name}_${link_so}2.txt")
add_custom_command(
OUTPUT
${some-file}
COMMAND
otool -L ${target_so} | tail -n +2 | grep ${link_so} | head -n1 > ${some-file}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
VERBATIM
)
add_custom_command(
OUTPUT
${some-file1}
COMMAND
cat ${some-file} | cut -d " " -f 1 | sed -E "s/^.//g" > ${some-file1}
DEPENDS
${some-file}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
VERBATIM
)
add_custom_command(
OUTPUT
${some-file2}
COMMAND
awk -F "/" "{print $NF}" ${some-file1} > ${some-file2}
DEPENDS
${some-file1}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
VERBATIM
)
add_custom_target(
link_${target_name}_${link_so} ALL
COMMAND install_name_tool -change `cat ${some-file1}` @rpath/`cat ${some-file2}` ${target_so}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
DEPENDS ${target_so} ${some-file1} ${some-file2}
COMMENT "install tool name"
)
add_custom_command(
TARGET link_${target_name}_${link_so}
POST_BUILD
COMMAND rm ${some-file} ${some-file1} ${some-file2}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
VERBATIM
)
if(depend_so)
add_dependencies(link_${target_name}_${link_so} link_${target_name}_${depend_so})
endif()
set(depend_so ${link_so})
endforeach()
endfunction()

View File

@ -28,7 +28,7 @@ else()
URL ${REQ_URL}
MD5 ${MD5}
PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/icu4c/icu4c.patch01
CONFIGURE_COMMAND ./icu4c/source/runConfigureICU MacOSX --enable-rpath --disable-tests
CONFIGURE_COMMAND ./icu4c/source/runConfigureICU MacOSX --disable-tests
--disable-samples --disable-icuio --disable-extras
ICU_DATA_FILTER_FILE=${CMAKE_BINARY_DIR}/icu4c_filter.json
)
@ -45,6 +45,12 @@ else()
)
endif()
include_directories(${icu4c_INC})
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
include(${CMAKE_SOURCE_DIR}/cmake/change_rpath.cmake)
changerpath($<TARGET_FILE:icu4c::${LIB_ICU_COMMON}> ${LIB_ICU_COMMON} "libicuuc;libicudata")
changerpath($<TARGET_FILE:icu4c::${LIB_ICU_DATA}> ${LIB_ICU_DATA} "libicudata")
changerpath($<TARGET_FILE:icu4c::${LIB_ICU_I18N}> ${LIB_ICU_I18N} "libicuuc;libicudata;libicui18n")
endif()
add_library(mindspore::icuuc ALIAS icu4c::${LIB_ICU_COMMON})
add_library(mindspore::icudata ALIAS icu4c::${LIB_ICU_DATA})
add_library(mindspore::icui18n ALIAS icu4c::${LIB_ICU_I18N})

View File

@ -29,11 +29,6 @@ option(ENABLE_SYM_FILE "enable sym file" OFF)
option(BUILD_DEV_MODE "MindSpore build nightly dev mode" OFF)
option(ENABLE_FAST_HASH_TABLE "Enable use fast hash table instead of std ones" ON)
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(ENABLE_DEBUGGER OFF)
endif()
if(NOT ENABLE_D AND NOT ENABLE_TESTCASES AND NOT ENABLE_ACL AND NOT ENABLE_GE)
set(ENABLE_GLIBCXX ON)
endif()
@ -143,7 +138,7 @@ endif()
if(ENABLE_DEBUGGER OR ENABLE_TESTCASES)
set(MS_BUILD_GRPC ON)
endif()
if(ENABLE_MINDDATA AND NOT CMAKE_SYSTEM_NAME MATCHES "Windows" AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
if(ENABLE_MINDDATA AND NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
set(MS_BUILD_GRPC ON)
endif()
@ -155,7 +150,7 @@ if(ENABLE_ACL AND NOT ENABLE_D)
set(MODE_ASCEND_ACL ON)
endif()
if(ENABLE_CPU AND NOT WIN32 AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
if(ENABLE_CPU AND NOT WIN32)
add_compile_definitions(ENABLE_ARMOUR)
endif()

View File

@ -40,9 +40,6 @@ set(INSTALL_BASE_DIR ".")
set(INSTALL_BIN_DIR "bin")
set(INSTALL_CFG_DIR "config")
set(INSTALL_LIB_DIR "lib")
message("offline debugger does not support macosx system temporarily")
# set package files
install(
TARGETS _c_expression
@ -50,6 +47,14 @@ install(
COMPONENT mindspore
)
if(ENABLE_DEBUGGER)
install(
TARGETS _mindspore_offline_debug
DESTINATION ${INSTALL_BASE_DIR}
COMPONENT mindspore
)
endif()
install(
TARGETS mindspore_shared_lib
DESTINATION ${INSTALL_LIB_DIR}
@ -71,6 +76,25 @@ if(USE_GLOG)
)
endif()
install(FILES ${libevent_LIBPATH}/libevent-2.1.7.dylib
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
install(FILES ${libevent_LIBPATH}/libevent_core-2.1.7.dylib
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
install(FILES ${libevent_LIBPATH}/libevent_extra-2.1.7.dylib
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
install(FILES ${libevent_LIBPATH}/libevent_openssl-2.1.7.dylib
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
install(FILES ${libevent_LIBPATH}/libevent_pthreads-2.1.7.dylib
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
if(ENABLE_CPU AND NOT WIN32)
install(
TARGETS ps_cache
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
endif()
if(MS_BUILD_GRPC)
file(GLOB_RECURSE GRPC_LIB_LIST
${grpc_LIBPATH}/libmindspore*
@ -218,6 +242,14 @@ if(EXISTS ${CMAKE_SOURCE_DIR}/mindspore/dataset)
)
endif()
if(EXISTS ${CMAKE_SOURCE_DIR}/mindspore/offline_debug)
install(
DIRECTORY ${CMAKE_SOURCE_DIR}/mindspore/offline_debug
DESTINATION ${INSTALL_PY_DIR}
COMPONENT mindspore
)
endif()
## Public header files
install(
DIRECTORY ${CMAKE_SOURCE_DIR}/include

View File

@ -374,9 +374,12 @@ if(CMAKE_SYSTEM_NAME MATCHES "Windows")
target_link_libraries(mindspore mindspore_gvar)
target_link_libraries(_c_expression PRIVATE -Wl,--whole-archive mindspore mindspore_core -Wl,--no-whole-archive)
elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(mindspore mindspore::pybind11_module)
target_link_libraries(mindspore mindspore_gvar)
target_link_libraries(_c_expression PRIVATE -Wl,-all_load mindspore mindspore_core -Wl,-noall_load)
target_link_libraries(mindspore proto_input mindspore::protobuf
mindspore::event mindspore::event_pthreads mindspore::event_openssl mindspore::eigen mindspore::json)
target_link_libraries(mindspore mindspore::event_core ps_cache)
target_link_libraries(_c_expression PRIVATE -Wl,-all_load mindspore mindspore_core proto_input -Wl,-noall_load)
target_link_libraries(_c_expression PRIVATE mindspore::pybind11_module)
target_link_libraries(_c_expression PRIVATE mindspore_gvar)
else()
if(ENABLE_CPU AND NOT WIN32)
target_link_libraries(mindspore proto_input mindspore::protobuf

View File

@ -44,7 +44,7 @@ if(ENABLE_CPU)
endif()
endif()
if(NOT ENABLE_CPU OR WIN32 OR APPLE)
if(NOT ENABLE_CPU OR WIN32)
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/apply_momentum_ps_kernel.cc")
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/embedding_look_up_proxy_kernel.cc")
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/embedding_look_up_ps_kernel.cc")

View File

@ -96,7 +96,8 @@ void GetKeysKernel::BuildGetKeysReq(const std::shared_ptr<fl::FBBuilder> &fbb) {
MS_LOG(INFO) << "BuildGetKeysReq successfully.";
}
bool GetKeysKernel::SavePublicKeyList(auto remote_public_key) {
bool GetKeysKernel::SavePublicKeyList(
const flatbuffers::Vector<flatbuffers::Offset<mindspore::schema::ClientPublicKeys>> *remote_public_key) {
if (remote_public_key == nullptr) {
MS_LOG(EXCEPTION) << "Input remote_pubic_key is nullptr.";
}

View File

@ -41,7 +41,8 @@ class GetKeysKernel : public CPUKernel {
private:
void BuildGetKeysReq(const std::shared_ptr<fl::FBBuilder> &fbb);
bool SavePublicKeyList(auto remote_public_key);
bool SavePublicKeyList(
const flatbuffers::Vector<flatbuffers::Offset<mindspore::schema::ClientPublicKeys>> *remote_public_key);
uint32_t rank_id_;
uint32_t server_num_;

View File

@ -33,10 +33,6 @@ if("${ENABLE_HIDDEN}" STREQUAL "OFF")
string(REPLACE " -fvisibility=hidden" " -fvisibility=default" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
endif()
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-private-field")
endif()
if(ENABLE_DEBUGGER)
list(APPEND _DEBUG_SRC_LIST
"${CMAKE_CURRENT_SOURCE_DIR}/debugger/debugger.cc"
@ -56,7 +52,7 @@ if(NOT ENABLE_SECURITY)
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/dump_utils.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/npy_header.cc"
)
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows" AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
list(APPEND _DEBUG_SRC_LIST
"${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/e2e_dump.cc"

View File

@ -37,10 +37,19 @@
#include "nlohmann/json.hpp"
#include "debug/debugger/tensor_summary.h"
#include "utils/file_utils.h"
#include "linux/limits.h"
#include "climits"
#ifdef ONLINE_DBG_MODE
namespace mindspore {
#endif
namespace {
#ifdef __APPLE__
constexpr int kStrErrorNone = 0;
#else
constexpr char *kStrErrorNone = nullptr;
#endif
} // namespace
DebugServices::DebugServices() { tensor_loader_ = std::make_shared<TensorLoader>(); }
DebugServices::DebugServices(const DebugServices &other) {
@ -632,7 +641,7 @@ void DebugServices::ReadTensorFromNpy(const std::string &tensor_name, const std:
const int kMaxFilenameLength = 128;
char err_info[kMaxFilenameLength];
auto ret = strerror_r(errno, err_info, sizeof(err_info));
if (ret != nullptr) {
if (ret != kStrErrorNone) {
MS_LOG(ERROR) << " ErrInfo:" << ret;
}
return;
@ -1059,7 +1068,7 @@ void DebugServices::ReadGraphRunIter(std::string file_path, std::tuple<uint32_t,
MS_LOG(ERROR) << "Failed to open file (In ReadGraphRunIter) " << file_path << " Errno:" << errno;
const int kMaxFilenameLength = NAME_MAX;
char err_info[kMaxFilenameLength];
if (strerror_r(errno, err_info, sizeof(err_info)) != nullptr) {
if (strerror_r(errno, err_info, sizeof(err_info)) != kStrErrorNone) {
MS_LOG(ERROR) << " ErrInfo:" << strerror_r(errno, err_info, sizeof(err_info));
}
@ -1190,7 +1199,7 @@ void DebugServices::ReadFileAndAddToTensor(const bool found, const std::vector<s
std::vector<std::shared_ptr<TensorData>> *result_list) {
std::string time_stamp = "";
std::string type_name = "";
uint64_t data_size = 0;
size_t data_size = 0;
std::vector<int64_t> shape;
std::vector<char> *buffer = nullptr;
if (found) {

View File

@ -32,7 +32,11 @@ namespace mindspore {
#endif
class TensorLoader {
public:
#ifndef __APPLE__
TensorLoader() : iter_num_(-1), mem_total_(0), mem_usage_(0) {}
#else
TensorLoader() : mem_total_(0), mem_usage_(0) {}
#endif
~TensorLoader() { EmptyTensor(); }
@ -227,7 +231,9 @@ class TensorLoader {
// the pair is (device_id, iteration)
std::map<std::string, std::shared_ptr<TensorData>> tensor_list_map_;
std::map<std::string, std::shared_ptr<TensorData>> prev_tensor_list_map_;
#ifndef __APPLE__
uint32_t iter_num_;
#endif
std::mutex lock_;
std::mutex mem_lock_;
uint64_t mem_total_;

View File

@ -1,6 +1,6 @@
file(GLOB_RECURSE _FL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
if(NOT ENABLE_CPU OR WIN32 OR APPLE)
if(NOT ENABLE_CPU OR WIN32)
list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/apply_momentum_kernel.cc")
list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/aggregation_kernel_factory.cc")
list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/dense_grad_accum_kernel.cc")
@ -49,6 +49,10 @@ if(NOT ENABLE_CPU OR WIN32 OR APPLE)
list(REMOVE_ITEM _FL_SRC_FILES "armour/cipher/cipher_unmask.cc")
endif()
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-tautological-pointer-compare")
endif()
list(LENGTH _FL_SRC_FILES fl_file_num)
if(NOT fl_file_num EQUAL 0)
set_property(SOURCE ${_FL_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_FL)

View File

@ -104,10 +104,14 @@ bool FLWorker::SendToServer(uint32_t server_rank, const void *data, size_t size,
std::this_thread::yield();
}
#ifdef __APPLE__
std::shared_ptr<unsigned char> message(new unsigned char[size], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> message;
std::unique_ptr<unsigned char[]> message_addr = std::make_unique<unsigned char[]>(size);
MS_EXCEPTION_IF_NULL(message_addr);
message = std::move(message_addr);
#endif
MS_EXCEPTION_IF_NULL(message);
uint64_t src_size = size;

View File

@ -74,9 +74,7 @@ add_subdirectory(core)
add_subdirectory(kernels)
add_subdirectory(engine)
add_subdirectory(api)
if(NOT APPLE)
add_subdirectory(audio)
endif()
add_subdirectory(audio)
add_subdirectory(text)
add_subdirectory(callback)
add_subdirectory(plugin)
@ -91,16 +89,12 @@ add_dependencies(engine-datasetops-source-sampler core)
add_dependencies(engine-datasetops core)
add_dependencies(engine-datasetops-mapop core)
add_dependencies(engine-opt core)
if(NOT APPLE)
add_dependencies(engine-gnn core)
endif()
add_dependencies(engine-gnn core)
add_dependencies(engine core)
add_dependencies(callback core)
if(NOT APPLE)
add_dependencies(audio-kernels core)
add_dependencies(audio-ir core)
add_dependencies(audio-ir-kernels core)
endif()
add_dependencies(audio-kernels core)
add_dependencies(audio-ir core)
add_dependencies(audio-ir-kernels core)
add_dependencies(text core)
add_dependencies(text-kernels core)
add_dependencies(text-ir core)
@ -161,10 +155,14 @@ set(submodules
$<TARGET_OBJECTS:engine-datasetops-source>
$<TARGET_OBJECTS:engine-datasetops-source-sampler>
$<TARGET_OBJECTS:engine-datasetops-mapop>
$<TARGET_OBJECTS:engine-gnn>
$<TARGET_OBJECTS:engine-datasetops>
$<TARGET_OBJECTS:engine-opt>
$<TARGET_OBJECTS:engine-cache-client>
$<TARGET_OBJECTS:engine>
$<TARGET_OBJECTS:audio-kernels>
$<TARGET_OBJECTS:audio-ir>
$<TARGET_OBJECTS:audio-ir-kernels>
$<TARGET_OBJECTS:text>
$<TARGET_OBJECTS:text-kernels>
$<TARGET_OBJECTS:text-ir>
@ -175,15 +173,6 @@ set(submodules
$<TARGET_OBJECTS:md_plugin>
)
if(NOT APPLE)
set(submodules
${submodules}
$<TARGET_OBJECTS:engine-gnn>
$<TARGET_OBJECTS:audio-kernels>
$<TARGET_OBJECTS:audio-ir>
$<TARGET_OBJECTS:audio-ir-kernels>)
endif()
if(ENABLE_ACL)
set(submodules
${submodules}
@ -223,61 +212,6 @@ endif()
################# Link with external libraries ########################
target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar)
set(targetso $<TARGET_FILE:_c_dataengine>)
set(target_tiny "libtinyxml")
set(target_icuuc "libicuuc")
set(target_icudata "libicudata")
set(target_icui18n "libicui18n")
function(changerpath targetso linkso)
set(some-file "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/${linkso}.txt")
set(some-file1 "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/${linkso}1.txt")
set(some-file2 "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/${linkso}2.txt")
add_custom_command(
OUTPUT
${some-file}
COMMAND
otool -L ${targetso} | grep ${linkso} > ${some-file}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset"
VERBATIM
)
add_custom_command(
OUTPUT
${some-file1}
COMMAND
cat ${some-file} | cut -d " " -f 1 | sed -E "s/^.//g" > ${some-file1}
DEPENDS
${some-file}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset"
VERBATIM
)
add_custom_command(
OUTPUT
${some-file2}
COMMAND
awk -F "/" "{print $NF}" ${some-file1} > ${some-file2}
DEPENDS
${some-file1}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset"
VERBATIM
)
add_custom_target(
link_${linkso} ALL
COMMAND install_name_tool -change `cat ${some-file1}` @rpath/`cat ${some-file2}` ${targetso}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset"
DEPENDS ${targetso} ${some-file1} ${some-file2}
COMMENT "install tool name")
endfunction()
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
changerpath(${targetso} ${target_tiny})
changerpath(${targetso} ${target_icuuc})
add_dependencies(link_${target_icuuc} link_${target_tiny})
changerpath(${targetso} ${target_icudata})
add_dependencies(link_${target_icudata} link_${target_icuuc})
changerpath(${targetso} ${target_icui18n})
add_dependencies(link_${target_icui18n} link_${target_icudata})
endif()
if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
if(ENABLE_PYTHON)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY})
@ -320,9 +254,7 @@ else()
if(${ENABLE_IBVERBS} STREQUAL "ON")
target_link_libraries(_c_dataengine PRIVATE ibverbs rdmacm)
endif()
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(_c_dataengine PRIVATE ps_cache)
endif()
target_link_libraries(_c_dataengine PRIVATE ps_cache)
endif()
endif()
@ -349,3 +281,9 @@ endif()
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
set_target_properties(_c_dataengine PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})
endif()
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
include(${CMAKE_SOURCE_DIR}/cmake/change_rpath.cmake)
set(LINK_SOS libtinyxml libicuuc libicudata libicui18n)
changerpath($<TARGET_FILE:_c_dataengine> _c_dataengine "${LINK_SOS}")
endif()

View File

@ -1,55 +1,42 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
if(ENABLE_PYTHON)
set(API_PYTHON_SRC_FILES
python/bindings/dataset/audio/bindings.cc
python/bindings/dataset/audio/kernels/ir/bindings.cc
python/bindings/dataset/callback/bindings.cc
python/bindings/dataset/core/bindings.cc
python/bindings/dataset/engine/cache/bindings.cc
python/bindings/dataset/engine/datasetops/bindings.cc
python/bindings/dataset/engine/gnn/bindings.cc
python/bindings/dataset/engine/ir/consumer/bindings.cc
python/bindings/dataset/engine/ir/datasetops/bindings.cc
python/bindings/dataset/engine/ir/datasetops/source/bindings.cc
python/bindings/dataset/engine/ir/datasetops/source/samplers/bindings.cc
python/bindings/dataset/engine/ir/execute/bindings.cc
python/bindings/dataset/engine/ir/schema/bindings.cc
python/bindings/dataset/engine/perf/bindings.cc
python/bindings/dataset/kernels/bindings.cc
python/bindings/dataset/kernels/ir/bindings.cc
python/bindings/dataset/kernels/ir/image/bindings.cc
python/bindings/dataset/text/bindings.cc
python/bindings/dataset/text/kernels/ir/bindings.cc
python/bindings/mindrecord/include/bindings.cc
python/pybind_conversion.cc
python/pybind_register.cc
)
if(APPLE)
list(REMOVE_ITEM API_PYTHON_SRC_FILES
add_library(APItoPython OBJECT
python/bindings/dataset/audio/bindings.cc
python/bindings/dataset/audio/kernels/ir/bindings.cc
python/bindings/dataset/callback/bindings.cc
python/bindings/dataset/core/bindings.cc
python/bindings/dataset/engine/cache/bindings.cc
python/bindings/dataset/engine/datasetops/bindings.cc
python/bindings/dataset/engine/gnn/bindings.cc
python/bindings/dataset/engine/ir/consumer/bindings.cc
python/bindings/dataset/engine/ir/datasetops/bindings.cc
python/bindings/dataset/engine/ir/datasetops/source/bindings.cc
python/bindings/dataset/engine/ir/datasetops/source/samplers/bindings.cc
python/bindings/dataset/engine/ir/execute/bindings.cc
python/bindings/dataset/engine/ir/schema/bindings.cc
python/bindings/dataset/engine/perf/bindings.cc
python/bindings/dataset/kernels/bindings.cc
python/bindings/dataset/kernels/ir/bindings.cc
python/bindings/dataset/kernels/ir/image/bindings.cc
python/bindings/dataset/text/bindings.cc
python/bindings/dataset/text/kernels/ir/bindings.cc
python/bindings/mindrecord/include/bindings.cc
python/pybind_conversion.cc
python/pybind_register.cc
)
endif()
add_library(APItoPython OBJECT ${API_PYTHON_SRC_FILES})
target_include_directories(APItoPython PRIVATE ${pybind11_INCLUDE_DIRS})
endif()
set(API_CPP_SRC_FILES
audio.cc
config.cc
data_helper.cc
datasets.cc
execute.cc
iterator.cc
transforms.cc
samplers.cc
text.cc
vision.cc
)
if(APPLE)
list(REMOVE_ITEM API_CPP_SRC_FILES audio.cc)
endif()
add_library(cpp-API OBJECT ${API_CPP_SRC_FILES})
add_library(cpp-API OBJECT
audio.cc
config.cc
data_helper.cc
datasets.cc
execute.cc
iterator.cc
transforms.cc
samplers.cc
text.cc
vision.cc
)

View File

@ -937,7 +937,7 @@ Status Flanger(const std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *out
cur_channel_phase[j] = static_cast<int>(j * lfo_length * channel_phase + 0.5);
// through the current channel phase and lfo arrays to get the delay
auto iter_lfo = lfo->begin<float>();
delay_tensor[j] = *(iter_lfo + (lfo_pos + cur_channel_phase[j]) % lfo_length);
delay_tensor[j] = *(iter_lfo + static_cast<ptrdiff_t>((lfo_pos + cur_channel_phase[j]) % lfo_length));
// the frac delay is obtained by using the frac function
frac_delay[j] = delay_tensor[j] - static_cast<int>(delay_tensor[j]);
delay_tensor[j] = floor(delay_tensor[j]);

View File

@ -1,8 +1,6 @@
add_subdirectory(datasetops)
add_subdirectory(opt)
if(NOT APPLE)
add_subdirectory(gnn)
endif()
add_subdirectory(gnn)
add_subdirectory(ir)
message("ENABLE_SECURITY = ${ENABLE_SECURITY}")
@ -49,12 +47,10 @@ add_dependencies(engine
engine-datasetops
engine-datasetops-source
engine-opt
engine-gnn
engine-cache-client
engine-datasetops-mapop
)
if(NOT APPLE)
add_dependencies(engine engine-gnn)
endif()
if(NOT ENABLE_SECURITY)
add_dependencies(engine engine-perf)

View File

@ -48,4 +48,6 @@ else()
endif()
endif()
set_target_properties(_c_mindrecord PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
set_target_properties(_c_mindrecord PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})
endif()

View File

@ -46,7 +46,7 @@
#include "frontend/optimizer/py_pass_manager.h"
#include "utils/ms_context.h"
#include "vm/transform.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/parameter_server.h"
#include "ps/scheduler.h"
#include "ps/worker.h"
@ -623,7 +623,7 @@ bool OptInlineAction(const ResourcePtr &res) {
bool GeOptimizeAction(const ResourcePtr &res) { return OptimizeAction(res, kGePasses); }
bool VmOptimizeAction(const ResourcePtr &res) {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_ps_mode()) {
kVmPasses.push_back({"server_communication_op_fusion", ps::Util::FuseServerCommOps});
}
@ -823,7 +823,7 @@ bool ExecuteAction(const ResourcePtr &res) {
return true;
}
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
bool StartPSWorkerAction(const ResourcePtr &) {
ps::Worker::GetInstance().Run();
return true;
@ -1163,7 +1163,7 @@ std::vector<ActionItem> VmPipeline() {
(void)actions.emplace_back(std::make_pair("eliminate_forward_cnode", EliminateForwardCNode));
(void)actions.emplace_back(std::make_pair("validate", ValidateAction));
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_worker()) {
std::string server_mode = ps::PSContext::instance()->server_mode();
if (server_mode == ps::kServerModeFL || server_mode == ps::kServerModeHybrid) {
@ -1201,7 +1201,7 @@ std::vector<ActionItem> MindIRPipeline() {
(void)actions.emplace_back(std::make_pair("execute", ExecuteAction));
return actions;
}
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
std::vector<ActionItem> ServerPipeline() {
auto actions = CommonPipeline();
(void)actions.emplace_back(std::make_pair("optimize", VmOptimizeAction));

View File

@ -36,7 +36,7 @@
#else
#include "runtime/device/gpu/distribution/collective_fake_init.h"
#endif
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/util.h"
#endif
#include "ps/ps_context.h"

View File

@ -49,7 +49,7 @@
#include "frontend/optimizer/irpass/gradient_eliminate.h"
#include "frontend/optimizer/irpass/parameter_eliminate.h"
#include "frontend/optimizer/irpass/updatestate_eliminate.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/util.h"
#include "ps/ps_context.h"
#endif
@ -218,7 +218,7 @@ namespace {
bool ReAutoMonadWrapper(const FuncGraphPtr &root, const opt::OptimizerPtr &) { return ReAutoMonad(root); }
bool parallel_mode() {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_server() || ps::PSContext::instance()->is_scheduler()) {
return false;
}
@ -603,7 +603,7 @@ bool SliceRecomputeActivationPass(const ResourcePtr &res) {
bool AddCacheEmbeddingPass(const ResourcePtr &res) {
MS_EXCEPTION_IF_NULL(res);
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_ps_mode()) {
return true;
}

View File

@ -66,7 +66,7 @@
#endif
#endif
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/constants.h"
#include "ps/util.h"
#include "ps/worker.h"
@ -881,7 +881,7 @@ std::vector<ActionItem> GetPipeline(const ResourcePtr &resource, const std::stri
std::string backend = MsContext::GetInstance()->backend_policy();
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
const std::string &server_mode = ps::PSContext::instance()->server_mode();
if ((server_mode == ps::kServerModeFL || server_mode == ps::kServerModeHybrid) &&
ps::PSContext::instance()->is_server()) {
@ -1442,7 +1442,7 @@ bool InitExecDataset(const std::string &queue_name, int64_t iter_num, int64_t ba
bool InitExecDatasetVm(const std::string &queue_name, int64_t size, int64_t batch_size,
const std::vector<TypePtr> &types, const std::vector<std::vector<int64_t>> &shapes,
const std::vector<int64_t> &input_indexes, bool need_run) {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if ((ps::PSContext::instance()->is_ps_mode()) && (!ps::PSContext::instance()->is_worker())) {
return true;
}
@ -1661,7 +1661,7 @@ void FinalizeBackend() {
void ClearResAtexit() {
MS_LOG(DEBUG) << "Pipeline clear all resource";
RecordExitStatus();
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_ps_mode() && ps::PSContext::instance()->is_worker()) {
if (ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps::ps_cache_instance.Finalize();

View File

@ -1,6 +1,6 @@
file(GLOB_RECURSE _PS_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
if(NOT ENABLE_CPU OR WIN32 OR APPLE)
if(NOT ENABLE_CPU OR WIN32)
list(REMOVE_ITEM _PS_SRC_FILES "optimizer_info_builder.cc")
list(REMOVE_ITEM _PS_SRC_FILES "optimizer_info.cc")
list(REMOVE_ITEM _PS_SRC_FILES "scheduler.cc")

View File

@ -192,9 +192,13 @@ const std::map<std::string, size_t> kCiphers = {{"ECDHE-RSA-AES128-GCM-SHA256",
{"!PSK", 27},
{"kEDH+AESGCM", 28}};
#ifdef __APPLE__
using DataPtr = std::shared_ptr<unsigned char>;
#else
using DataPtr = std::shared_ptr<unsigned char[]>;
#endif
using VectorPtr = std::shared_ptr<std::vector<unsigned char>>;
using Key = uint64_t;
using Key = size_t;
using Keys = std::vector<Key>;
using Values = std::vector<float>;
using ValuesPtr = std::shared_ptr<Values>;

View File

@ -956,7 +956,11 @@ void AbstractNode::ProcessSendData(const std::shared_ptr<TcpConnection> &conn, c
MS_EXCEPTION_IF_NULL(conn);
MS_EXCEPTION_IF_NULL(meta);
MS_EXCEPTION_IF_NULL(data);
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[size], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[size]);
#endif
if (size > 0) {
size_t dest_size = size;
size_t src_size = size;

View File

@ -61,7 +61,11 @@ class AbstractNode : public Node {
const std::shared_ptr<MessageMeta> &meta, const Protos &protos,
const void *data, size_t size);
#ifdef __APPLE__
using DataPtr = std::shared_ptr<unsigned char>;
#else
using DataPtr = std::shared_ptr<unsigned char[]>;
#endif
using VectorPtr = std::shared_ptr<std::vector<unsigned char>>;
using RequestHandler =
std::function<void(const std::shared_ptr<TcpConnection> &conn, const std::shared_ptr<MessageMeta> &meta,

View File

@ -84,7 +84,11 @@ class TcpCommunicator : public CommunicatorBase {
bool SendPbRequest(const T &pb_msg, const uint32_t &rank_id, TcpUserCommand command,
std::shared_ptr<std::vector<unsigned char>> *output = nullptr) {
const std::string &msg_str = pb_msg.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> msg(new unsigned char[msg_str.size()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> msg(new unsigned char[msg_str.size()]);
#endif
MS_ERROR_IF_NULL_W_RET_VAL(msg, false);
size_t dest_size = msg_str.size();
size_t src_size = msg_str.size();

View File

@ -49,7 +49,7 @@ class SchedulerRecovery : public RecoveryBase {
private:
// The node_ will only be instantiated with worker/server node.
SchedulerNode *const node_;
SchedulerNode *const node_ = nullptr;
};
} // namespace core
} // namespace ps

View File

@ -1,7 +1,6 @@
if(ENABLE_CPU AND NOT WIN32 AND NOT APPLE)
if(ENABLE_CPU AND NOT WIN32)
file(GLOB_RECURSE _PS_CACHE_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "ps_data/*.cc")
set_property(SOURCE ${_PS_CACHE_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PS)
add_library(ps_cache SHARED ${_PS_CACHE_SRC_FILES})
target_link_libraries(ps_cache mindspore::pybind11_module)
endif()

View File

@ -231,9 +231,14 @@ void PsCacheManager::AllocMemForHashTable() {
device_address.addr = addr;
auto &host_address = item.second.host_address;
#ifdef __APPLE__
host_address =
std::shared_ptr<float>(new float[host_vocab_cache_size_ * embedding_size], std::default_delete<float[]>());
#else
std::unique_ptr<float[]> host_hash_table_addr = std::make_unique<float[]>(host_vocab_cache_size_ * embedding_size);
MS_EXCEPTION_IF_NULL(host_hash_table_addr);
host_address = std::move(host_hash_table_addr);
#endif
MS_EXCEPTION_IF_NULL(host_address);
max_embedding_size = (embedding_size > max_embedding_size) ? embedding_size : max_embedding_size;

View File

@ -49,7 +49,11 @@ struct HashTableInfo {
size_t embedding_size{0};
size_t vocab_size{0};
Address device_address{nullptr, 0};
#ifdef __APPLE__
std::shared_ptr<float> host_address{nullptr};
#else
std::shared_ptr<float[]> host_address{nullptr};
#endif
ParamInitInfo param_init_info_;
};

View File

@ -18,7 +18,7 @@
#include "utils/log_adapter.h"
#include "utils/ms_utils.h"
#include "backend/kernel_compiler/kernel.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/ps_cache/ps_cache_manager.h"
#include "ps/ps_cache/ps_data/ps_data_prefetch.h"
#endif
@ -88,7 +88,7 @@ void PSContext::Reset() {
is_worker_ = false;
is_pserver_ = false;
is_sched_ = false;
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps_cache_instance.Finalize();
set_cache_enable(false);
@ -144,44 +144,44 @@ uint32_t PSContext::ps_rank_id() const { return rank_id_; }
void PSContext::InsertHashTableSize(const std::string &param_name, size_t cache_vocab_size, size_t embedding_size,
size_t vocab_size) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
ps_cache_instance.InsertHashTableSize(param_name, cache_vocab_size, embedding_size, vocab_size);
#endif
}
void PSContext::ReInsertHashTableSize(const std::string &new_param_name, const std::string &cur_param_name,
size_t cache_vocab_size, size_t embedding_size) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
ps_cache_instance.ReInsertHashTableSize(new_param_name, cur_param_name, cache_vocab_size, embedding_size);
#endif
}
void PSContext::InsertWeightInitInfo(const std::string &param_name, size_t global_seed, size_t op_seed) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
ps_cache_instance.InsertWeightInitInfo(param_name, global_seed, op_seed);
#endif
}
void PSContext::InsertAccumuInitInfo(const std::string &param_name, float init_val) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
ps_cache_instance.InsertAccumuInitInfo(param_name, init_val);
#endif
}
void PSContext::CloneHashTable(const std::string &dest_param_name, const std::string &src_param_name) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
ps_cache_instance.CloneHashTable(dest_param_name, src_param_name);
#endif
}
void PSContext::set_cache_enable(bool cache_enable) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
PsDataPrefetch::GetInstance().set_cache_enable(cache_enable);
#endif
}
void PSContext::set_rank_id(uint32_t rank_id) const {
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
ps_cache_instance.set_rank_id(rank_id);
#endif
}

View File

@ -227,7 +227,11 @@ bool Worker::InitPSEmbeddingTable(const size_t &key, const std::vector<size_t> &
std::string kv_data = embedding_table_meta.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
#endif
size_t dest_size = kv_data.length();
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
if (ret != 0) {
@ -293,8 +297,11 @@ bool Worker::DoPSEmbeddingLookup(const Key &key, const std::vector<int> &lookup_
if (messages.at(i).first) {
rank_ids.push_back(i);
std::string kv_data = messages.at(i).second.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
#endif
size_t dest_size = kv_data.length();
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
if (ret != 0) {
@ -382,7 +389,11 @@ bool Worker::UpdateEmbeddingTable(const std::vector<Key> &keys, const std::vecto
rank_ids.push_back(i);
std::string kv_data = messages.at(i).second.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
#endif
size_t dest_size = kv_data.length();
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
if (ret != 0) {
@ -403,7 +414,11 @@ void Worker::Finalize() {
kvs.add_keys(0);
kvs.add_values(0.0f);
std::string kv_data = kvs.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
#endif
size_t dest_size = kv_data.length();
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
if (ret != 0) {
@ -644,7 +659,11 @@ void Worker::PushData(const std::vector<Key> &keys, const std::vector<float> &va
SendForPush(cmd, kvs, worker_init_embedding_partitioner_, {});
} else {
std::string kv_data = kvs.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
#endif
size_t dest_size = kv_data.length();
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
if (ret != 0) {
@ -950,7 +969,11 @@ void Worker::SendForPush(int cmd, const KVMessage &send, const KVPartitioner &pa
rank_ids.push_back(i);
std::string kv_data = messages.at(i).second.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
#endif
size_t dest_size = kv_data.length();
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
if (ret != 0) {
@ -977,7 +1000,11 @@ void Worker::SendForPull(int cmd, const KVMessage &send, const KVPartitioner &pa
rank_ids.push_back(i);
std::string kv_data = messages.at(i).second.SerializeAsString();
#ifdef __APPLE__
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
#else
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
#endif
size_t dest_size = kv_data.length();
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
if (ret != 0) {

View File

@ -33,7 +33,7 @@
#include "frontend/parallel/context.h"
#include "debug/env_config_parser.h"
#include "pipeline/pynative/pynative_profiling.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/ps_cache/ps_cache_manager.h"
#endif
@ -576,7 +576,7 @@ void KernelRuntime::AssignStaticMemoryInput(const session::KernelGraph &graph) {
}
add_need_alloc_nodes(input_node);
}
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
bool ps_cache_check = false;
#endif
for (auto &item : need_alloc_nodes) {
@ -590,7 +590,7 @@ void KernelRuntime::AssignStaticMemoryInput(const session::KernelGraph &graph) {
continue;
}
DeviceAddressPtr device_address = nullptr;
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
const std::string &param_name = item->fullname_with_scope();
if (ps::ps_cache_instance.IsHashTable(param_name)) {
MS_LOG(INFO) << "Parameter(" << param_name << ")"
@ -1598,7 +1598,7 @@ void KernelRuntime::ClearGraphRuntimeResource(uint32_t graph_id) {
MS_LOG(INFO) << "Clear graph:" << graph_id << " runtime resource";
}
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
void KernelRuntime::GetFirstPSEmbeddingCache(const session::KernelGraph &graph,
AnfNodePtr *const first_cache_input_index,
size_t *const first_cache_size) {

View File

@ -173,7 +173,7 @@ class KernelRuntime {
void RunOpAssignOutputNodeMemory(const ValuePtr &pre_output_value, const session::KernelGraph &graph);
void AssignValueNodeTensor(const ValueNodePtr &value_node, const ValuePtr &node_value, size_t output_idx);
DeviceAddressPtr PreAssignCNodeMemory(const AnfNodePtr &anf_node, size_t index) const;
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
void GetFirstPSEmbeddingCache(const session::KernelGraph &graph, AnfNodePtr *const first_cache_input_index,
size_t *const first_cache_size);
void CheckIfSupportPSEmbeddingCache(const session::KernelGraph &graph);

View File

@ -16,7 +16,7 @@
#include "runtime/device/kernel_runtime_manager.h"
#include "utils/log_adapter.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/ps_cache/ps_cache_manager.h"
#endif
#include "backend/session/pynative_task_manager.h"
@ -26,7 +26,7 @@ namespace device {
void KernelRuntimeManager::ClearRuntimeResource() {
// Just remove PyNative tasks before runtime resource release.
session::PynativeTaskManager::GetInstance().Reset();
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_worker() && ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps::ps_cache_instance.SyncEmbeddingTable();
}
@ -132,7 +132,7 @@ void KernelRuntimeManager::ReleaseKernelRuntime(const std::string &device_name,
if (runtime == nullptr) {
return;
}
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_worker() && ps::PsDataPrefetch::GetInstance().cache_enable()) {
ps::ps_cache_instance.SyncEmbeddingTable();
}

View File

@ -67,7 +67,7 @@ void SuperKernelActor::Run(OpContext<DeviceTensor> *const context) {
MS_LOG(INFO) << "Super kernel actor(" << GetAID().Name()
<< ") launches graph: " << std::to_string(graph_->graph_id());
if (!CopyInputData(context)) {
std::string error_info = "Copy the input data failed, graph id: " + graph_->graph_id();
std::string error_info = "Copy the input data failed, graph id: " + std::to_string(graph_->graph_id());
SET_OPCONTEXT_FAIL_RET_WITH_ERROR((*context), error_info);
}

View File

@ -632,7 +632,7 @@ void SetMindRTEnable() {
return;
#endif
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (ps::PSContext::instance()->is_ps_mode()) {
return;
}

View File

@ -31,15 +31,13 @@ Common imported modules in corresponding API examples are as follows:
from mindspore.dataset.transforms import c_transforms
"""
import platform
from .core import config
from .engine import *
from .engine.cache_client import DatasetCache
from .engine.datasets import *
from .engine.graphdata import GraphData
from .engine.samplers import *
from .engine.serializer_deserializer import compare, deserialize, serialize, show
if platform.system().lower() != "darwin":
from .engine.graphdata import GraphData
__all__ = []
__all__.extend(engine.__all__)

View File

@ -22,23 +22,19 @@ high performance and parse data precisely. It also provides the following
operations for users to preprocess data: shuffle, batch, repeat, map, and zip.
"""
import platform
from ..callback import DSCallback, WaitedDSCallback
from ..core import config
from .cache_client import DatasetCache
from .datasets import *
from .graphdata import GraphData, SamplingStrategy, OutputFormat
from .iterators import *
from .samplers import *
from .serializer_deserializer import compare, deserialize, serialize, show
if platform.system().lower() != "darwin":
from .graphdata import GraphData, SamplingStrategy, OutputFormat
__all__ = ["CelebADataset", "Cifar100Dataset", "Cifar10Dataset", "CLUEDataset", "CocoDataset", "CSVDataset",
"GeneratorDataset", "ImageFolderDataset", "ManifestDataset", "MindDataset", "MnistDataset",
"GeneratorDataset", "GraphData", "ImageFolderDataset", "ManifestDataset", "MindDataset", "MnistDataset",
"NumpySlicesDataset", "PaddedDataset", "TextFileDataset", "TFRecordDataset", "VOCDataset",
"DistributedSampler", "PKSampler", "RandomSampler", "SequentialSampler", "SubsetRandomSampler",
"WeightedRandomSampler", "SubsetSampler",
"DatasetCache", "DSCallback", "Schema", "WaitedDSCallback", "compare", "deserialize",
"serialize", "show", "zip"]
if platform.system().lower() != "darwin":
__all__.append("GraphData")

View File

@ -1,15 +1,21 @@
diff -Npur grpc/..rej grpc-patch/..rej
--- grpc/..rej 1970-01-01 08:00:00.000000000 +0800
+++ grpc-patch/..rej 2021-04-22 21:00:17.343178600 +0800
@@ -0,0 +1,22 @@
@@ -0,0 +1,28 @@
+--- CMakeLists.txt 2020-02-27 03:12:33.000000000 +0800
++++ CMakeLists.txt 2021-04-07 21:27:12.317207600 +0800
+@@ -12992,7 +12992,7 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
+@@ -12992,7 +12992,13 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
+ add_executable(grpc_cpp_plugin
+ src/compiler/cpp_plugin.cc
+ )
+-
++set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
++if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
++ set(CMAKE_MACOSX_RPATH 1)
++ set(CMAKE_INSTALL_RPATH "@executable_path/../lib")
++ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}")
++else()
++ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
++endif()
+ target_include_directories(grpc_cpp_plugin
+ PRIVATE
+ ${CMAKE_CURRENT_SOURCE_DIR}
@ -27,15 +33,21 @@ diff -Npur grpc/..rej grpc-patch/..rej
diff -Npur grpc/.rej grpc-patch/.rej
--- grpc/.rej 1970-01-01 08:00:00.000000000 +0800
+++ grpc-patch/.rej 2021-04-22 21:03:38.192349100 +0800
@@ -0,0 +1,22 @@
@@ -0,0 +1,28 @@
+--- grpc/CMakeLists.txt 2020-02-27 03:12:33.000000000 +0800
++++ grpc-patch/CMakeLists.txt 2021-04-07 21:27:12.317207600 +0800
+@@ -12992,7 +12992,7 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
+@@ -12992,7 +12992,13 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
+ add_executable(grpc_cpp_plugin
+ src/compiler/cpp_plugin.cc
+ )
+-
++set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
++if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
++ set(CMAKE_MACOSX_RPATH 1)
++ set(CMAKE_INSTALL_RPATH "@executable_path/../lib")
++ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}")
++else()
++ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
++endif()
+ target_include_directories(grpc_cpp_plugin
+ PRIVATE
+ ${CMAKE_CURRENT_SOURCE_DIR}
@ -107,12 +119,18 @@ diff -Npur grpc/CMakeLists.txt grpc-patch/CMakeLists.txt
if(WIN32 AND MSVC)
set_target_properties(upb PROPERTIES COMPILE_PDB_NAME "upb"
COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}"
@@ -12992,7 +13004,7 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
@@ -12992,7 +13004,13 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
add_executable(grpc_cpp_plugin
src/compiler/cpp_plugin.cc
)
-
+set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
+if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
+ set(CMAKE_MACOSX_RPATH 1)
+ set(CMAKE_INSTALL_RPATH "@executable_path/../lib")
+ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}")
+else()
+ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
+endif()
target_include_directories(grpc_cpp_plugin
PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}

View File

@ -10,3 +10,8 @@ diff -Npur libevent/CMakeLists.txt libevent-modify/CMakeLists.txt
endif()
if (NOT EVENT__DISABLE_THREAD_SUPPORT)
diff -Npur libevent/cmake/AddEventLibrary.cmake libevent-modify/cmake/AddEventLibrary.cmake
--- libevent/cmake/AddEventLibrary.cmake 2020-07-05 20:02:46.000000000 +0800
+++ libevent-modify/cmake/AddEventLibrary.cmake 2021-04-19 16:36:57.982307500 +0800
@@ -153,1 +153,0 @@
- INSTALL_NAME_DIR "${CMAKE_INSTALL_PREFIX}/lib"