forked from mindspore-Ecosystem/mindspore
fix mindspore mac compile
fix debugger, mindarmour, audio, gnn, grpc, ps/fl.
This commit is contained in:
parent
745c1eaff8
commit
78b37afc30
|
@ -20,7 +20,7 @@ endif()
|
|||
|
||||
if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
|
||||
set(CMAKE_OSX_SYSROOT "")
|
||||
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Winconsistent-missing-override -Wuser-defined-warnings \
|
||||
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Winconsistent-missing-override -Wno-user-defined-warnings \
|
||||
-Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare \
|
||||
-Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move \
|
||||
-Wno-inconsistent-missing-override -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2")
|
||||
|
|
|
@ -0,0 +1,54 @@
|
|||
function(changerpath target_so target_name link_sos)
|
||||
set(depend_so "")
|
||||
foreach(link_so ${link_sos})
|
||||
set(some-file "${CMAKE_SOURCE_DIR}/build/${target_name}_${link_so}.txt")
|
||||
set(some-file1 "${CMAKE_SOURCE_DIR}/build/${target_name}_${link_so}1.txt")
|
||||
set(some-file2 "${CMAKE_SOURCE_DIR}/build/${target_name}_${link_so}2.txt")
|
||||
add_custom_command(
|
||||
OUTPUT
|
||||
${some-file}
|
||||
COMMAND
|
||||
otool -L ${target_so} | tail -n +2 | grep ${link_so} | head -n1 > ${some-file}
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
|
||||
VERBATIM
|
||||
)
|
||||
add_custom_command(
|
||||
OUTPUT
|
||||
${some-file1}
|
||||
COMMAND
|
||||
cat ${some-file} | cut -d " " -f 1 | sed -E "s/^.//g" > ${some-file1}
|
||||
DEPENDS
|
||||
${some-file}
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
|
||||
VERBATIM
|
||||
)
|
||||
add_custom_command(
|
||||
OUTPUT
|
||||
${some-file2}
|
||||
COMMAND
|
||||
awk -F "/" "{print $NF}" ${some-file1} > ${some-file2}
|
||||
DEPENDS
|
||||
${some-file1}
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
|
||||
VERBATIM
|
||||
)
|
||||
add_custom_target(
|
||||
link_${target_name}_${link_so} ALL
|
||||
COMMAND install_name_tool -change `cat ${some-file1}` @rpath/`cat ${some-file2}` ${target_so}
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
|
||||
DEPENDS ${target_so} ${some-file1} ${some-file2}
|
||||
COMMENT "install tool name"
|
||||
)
|
||||
add_custom_command(
|
||||
TARGET link_${target_name}_${link_so}
|
||||
POST_BUILD
|
||||
COMMAND rm ${some-file} ${some-file1} ${some-file2}
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/build"
|
||||
VERBATIM
|
||||
)
|
||||
if(depend_so)
|
||||
add_dependencies(link_${target_name}_${link_so} link_${target_name}_${depend_so})
|
||||
endif()
|
||||
set(depend_so ${link_so})
|
||||
endforeach()
|
||||
endfunction()
|
|
@ -28,7 +28,7 @@ else()
|
|||
URL ${REQ_URL}
|
||||
MD5 ${MD5}
|
||||
PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/icu4c/icu4c.patch01
|
||||
CONFIGURE_COMMAND ./icu4c/source/runConfigureICU MacOSX --enable-rpath --disable-tests
|
||||
CONFIGURE_COMMAND ./icu4c/source/runConfigureICU MacOSX --disable-tests
|
||||
--disable-samples --disable-icuio --disable-extras
|
||||
ICU_DATA_FILTER_FILE=${CMAKE_BINARY_DIR}/icu4c_filter.json
|
||||
)
|
||||
|
@ -45,6 +45,12 @@ else()
|
|||
)
|
||||
endif()
|
||||
include_directories(${icu4c_INC})
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
include(${CMAKE_SOURCE_DIR}/cmake/change_rpath.cmake)
|
||||
changerpath($<TARGET_FILE:icu4c::${LIB_ICU_COMMON}> ${LIB_ICU_COMMON} "libicuuc;libicudata")
|
||||
changerpath($<TARGET_FILE:icu4c::${LIB_ICU_DATA}> ${LIB_ICU_DATA} "libicudata")
|
||||
changerpath($<TARGET_FILE:icu4c::${LIB_ICU_I18N}> ${LIB_ICU_I18N} "libicuuc;libicudata;libicui18n")
|
||||
endif()
|
||||
add_library(mindspore::icuuc ALIAS icu4c::${LIB_ICU_COMMON})
|
||||
add_library(mindspore::icudata ALIAS icu4c::${LIB_ICU_DATA})
|
||||
add_library(mindspore::icui18n ALIAS icu4c::${LIB_ICU_I18N})
|
||||
|
|
|
@ -29,11 +29,6 @@ option(ENABLE_SYM_FILE "enable sym file" OFF)
|
|||
option(BUILD_DEV_MODE "MindSpore build nightly dev mode" OFF)
|
||||
option(ENABLE_FAST_HASH_TABLE "Enable use fast hash table instead of std ones" ON)
|
||||
|
||||
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
set(ENABLE_DEBUGGER OFF)
|
||||
endif()
|
||||
|
||||
if(NOT ENABLE_D AND NOT ENABLE_TESTCASES AND NOT ENABLE_ACL AND NOT ENABLE_GE)
|
||||
set(ENABLE_GLIBCXX ON)
|
||||
endif()
|
||||
|
@ -143,7 +138,7 @@ endif()
|
|||
if(ENABLE_DEBUGGER OR ENABLE_TESTCASES)
|
||||
set(MS_BUILD_GRPC ON)
|
||||
endif()
|
||||
if(ENABLE_MINDDATA AND NOT CMAKE_SYSTEM_NAME MATCHES "Windows" AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
if(ENABLE_MINDDATA AND NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
|
||||
set(MS_BUILD_GRPC ON)
|
||||
endif()
|
||||
|
||||
|
@ -155,7 +150,7 @@ if(ENABLE_ACL AND NOT ENABLE_D)
|
|||
set(MODE_ASCEND_ACL ON)
|
||||
endif()
|
||||
|
||||
if(ENABLE_CPU AND NOT WIN32 AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
if(ENABLE_CPU AND NOT WIN32)
|
||||
add_compile_definitions(ENABLE_ARMOUR)
|
||||
endif()
|
||||
|
||||
|
|
|
@ -40,9 +40,6 @@ set(INSTALL_BASE_DIR ".")
|
|||
set(INSTALL_BIN_DIR "bin")
|
||||
set(INSTALL_CFG_DIR "config")
|
||||
set(INSTALL_LIB_DIR "lib")
|
||||
|
||||
message("offline debugger does not support macosx system temporarily")
|
||||
|
||||
# set package files
|
||||
install(
|
||||
TARGETS _c_expression
|
||||
|
@ -50,6 +47,14 @@ install(
|
|||
COMPONENT mindspore
|
||||
)
|
||||
|
||||
if(ENABLE_DEBUGGER)
|
||||
install(
|
||||
TARGETS _mindspore_offline_debug
|
||||
DESTINATION ${INSTALL_BASE_DIR}
|
||||
COMPONENT mindspore
|
||||
)
|
||||
endif()
|
||||
|
||||
install(
|
||||
TARGETS mindspore_shared_lib
|
||||
DESTINATION ${INSTALL_LIB_DIR}
|
||||
|
@ -71,6 +76,25 @@ if(USE_GLOG)
|
|||
)
|
||||
endif()
|
||||
|
||||
install(FILES ${libevent_LIBPATH}/libevent-2.1.7.dylib
|
||||
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
|
||||
install(FILES ${libevent_LIBPATH}/libevent_core-2.1.7.dylib
|
||||
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
|
||||
install(FILES ${libevent_LIBPATH}/libevent_extra-2.1.7.dylib
|
||||
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
|
||||
install(FILES ${libevent_LIBPATH}/libevent_openssl-2.1.7.dylib
|
||||
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
|
||||
install(FILES ${libevent_LIBPATH}/libevent_pthreads-2.1.7.dylib
|
||||
DESTINATION ${INSTALL_LIB_DIR} COMPONENT mindspore)
|
||||
|
||||
if(ENABLE_CPU AND NOT WIN32)
|
||||
install(
|
||||
TARGETS ps_cache
|
||||
DESTINATION ${INSTALL_LIB_DIR}
|
||||
COMPONENT mindspore
|
||||
)
|
||||
endif()
|
||||
|
||||
if(MS_BUILD_GRPC)
|
||||
file(GLOB_RECURSE GRPC_LIB_LIST
|
||||
${grpc_LIBPATH}/libmindspore*
|
||||
|
@ -218,6 +242,14 @@ if(EXISTS ${CMAKE_SOURCE_DIR}/mindspore/dataset)
|
|||
)
|
||||
endif()
|
||||
|
||||
if(EXISTS ${CMAKE_SOURCE_DIR}/mindspore/offline_debug)
|
||||
install(
|
||||
DIRECTORY ${CMAKE_SOURCE_DIR}/mindspore/offline_debug
|
||||
DESTINATION ${INSTALL_PY_DIR}
|
||||
COMPONENT mindspore
|
||||
)
|
||||
endif()
|
||||
|
||||
## Public header files
|
||||
install(
|
||||
DIRECTORY ${CMAKE_SOURCE_DIR}/include
|
||||
|
|
|
@ -374,9 +374,12 @@ if(CMAKE_SYSTEM_NAME MATCHES "Windows")
|
|||
target_link_libraries(mindspore mindspore_gvar)
|
||||
target_link_libraries(_c_expression PRIVATE -Wl,--whole-archive mindspore mindspore_core -Wl,--no-whole-archive)
|
||||
elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
target_link_libraries(mindspore mindspore::pybind11_module)
|
||||
target_link_libraries(mindspore mindspore_gvar)
|
||||
target_link_libraries(_c_expression PRIVATE -Wl,-all_load mindspore mindspore_core -Wl,-noall_load)
|
||||
target_link_libraries(mindspore proto_input mindspore::protobuf
|
||||
mindspore::event mindspore::event_pthreads mindspore::event_openssl mindspore::eigen mindspore::json)
|
||||
target_link_libraries(mindspore mindspore::event_core ps_cache)
|
||||
target_link_libraries(_c_expression PRIVATE -Wl,-all_load mindspore mindspore_core proto_input -Wl,-noall_load)
|
||||
target_link_libraries(_c_expression PRIVATE mindspore::pybind11_module)
|
||||
target_link_libraries(_c_expression PRIVATE mindspore_gvar)
|
||||
else()
|
||||
if(ENABLE_CPU AND NOT WIN32)
|
||||
target_link_libraries(mindspore proto_input mindspore::protobuf
|
||||
|
|
|
@ -44,7 +44,7 @@ if(ENABLE_CPU)
|
|||
endif()
|
||||
endif()
|
||||
|
||||
if(NOT ENABLE_CPU OR WIN32 OR APPLE)
|
||||
if(NOT ENABLE_CPU OR WIN32)
|
||||
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/apply_momentum_ps_kernel.cc")
|
||||
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/embedding_look_up_proxy_kernel.cc")
|
||||
list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/embedding_look_up_ps_kernel.cc")
|
||||
|
|
|
@ -96,7 +96,8 @@ void GetKeysKernel::BuildGetKeysReq(const std::shared_ptr<fl::FBBuilder> &fbb) {
|
|||
MS_LOG(INFO) << "BuildGetKeysReq successfully.";
|
||||
}
|
||||
|
||||
bool GetKeysKernel::SavePublicKeyList(auto remote_public_key) {
|
||||
bool GetKeysKernel::SavePublicKeyList(
|
||||
const flatbuffers::Vector<flatbuffers::Offset<mindspore::schema::ClientPublicKeys>> *remote_public_key) {
|
||||
if (remote_public_key == nullptr) {
|
||||
MS_LOG(EXCEPTION) << "Input remote_pubic_key is nullptr.";
|
||||
}
|
||||
|
|
|
@ -41,7 +41,8 @@ class GetKeysKernel : public CPUKernel {
|
|||
|
||||
private:
|
||||
void BuildGetKeysReq(const std::shared_ptr<fl::FBBuilder> &fbb);
|
||||
bool SavePublicKeyList(auto remote_public_key);
|
||||
bool SavePublicKeyList(
|
||||
const flatbuffers::Vector<flatbuffers::Offset<mindspore::schema::ClientPublicKeys>> *remote_public_key);
|
||||
|
||||
uint32_t rank_id_;
|
||||
uint32_t server_num_;
|
||||
|
|
|
@ -33,10 +33,6 @@ if("${ENABLE_HIDDEN}" STREQUAL "OFF")
|
|||
string(REPLACE " -fvisibility=hidden" " -fvisibility=default" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
endif()
|
||||
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-private-field")
|
||||
endif()
|
||||
|
||||
if(ENABLE_DEBUGGER)
|
||||
list(APPEND _DEBUG_SRC_LIST
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/debugger/debugger.cc"
|
||||
|
@ -56,7 +52,7 @@ if(NOT ENABLE_SECURITY)
|
|||
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/dump_utils.cc"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/npy_header.cc"
|
||||
)
|
||||
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows" AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
|
||||
list(APPEND _DEBUG_SRC_LIST
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/e2e_dump.cc"
|
||||
|
|
|
@ -37,10 +37,19 @@
|
|||
#include "nlohmann/json.hpp"
|
||||
#include "debug/debugger/tensor_summary.h"
|
||||
#include "utils/file_utils.h"
|
||||
#include "linux/limits.h"
|
||||
#include "climits"
|
||||
#ifdef ONLINE_DBG_MODE
|
||||
namespace mindspore {
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
#ifdef __APPLE__
|
||||
constexpr int kStrErrorNone = 0;
|
||||
#else
|
||||
constexpr char *kStrErrorNone = nullptr;
|
||||
#endif
|
||||
} // namespace
|
||||
|
||||
DebugServices::DebugServices() { tensor_loader_ = std::make_shared<TensorLoader>(); }
|
||||
|
||||
DebugServices::DebugServices(const DebugServices &other) {
|
||||
|
@ -632,7 +641,7 @@ void DebugServices::ReadTensorFromNpy(const std::string &tensor_name, const std:
|
|||
const int kMaxFilenameLength = 128;
|
||||
char err_info[kMaxFilenameLength];
|
||||
auto ret = strerror_r(errno, err_info, sizeof(err_info));
|
||||
if (ret != nullptr) {
|
||||
if (ret != kStrErrorNone) {
|
||||
MS_LOG(ERROR) << " ErrInfo:" << ret;
|
||||
}
|
||||
return;
|
||||
|
@ -1059,7 +1068,7 @@ void DebugServices::ReadGraphRunIter(std::string file_path, std::tuple<uint32_t,
|
|||
MS_LOG(ERROR) << "Failed to open file (In ReadGraphRunIter) " << file_path << " Errno:" << errno;
|
||||
const int kMaxFilenameLength = NAME_MAX;
|
||||
char err_info[kMaxFilenameLength];
|
||||
if (strerror_r(errno, err_info, sizeof(err_info)) != nullptr) {
|
||||
if (strerror_r(errno, err_info, sizeof(err_info)) != kStrErrorNone) {
|
||||
MS_LOG(ERROR) << " ErrInfo:" << strerror_r(errno, err_info, sizeof(err_info));
|
||||
}
|
||||
|
||||
|
@ -1190,7 +1199,7 @@ void DebugServices::ReadFileAndAddToTensor(const bool found, const std::vector<s
|
|||
std::vector<std::shared_ptr<TensorData>> *result_list) {
|
||||
std::string time_stamp = "";
|
||||
std::string type_name = "";
|
||||
uint64_t data_size = 0;
|
||||
size_t data_size = 0;
|
||||
std::vector<int64_t> shape;
|
||||
std::vector<char> *buffer = nullptr;
|
||||
if (found) {
|
||||
|
|
|
@ -32,7 +32,11 @@ namespace mindspore {
|
|||
#endif
|
||||
class TensorLoader {
|
||||
public:
|
||||
#ifndef __APPLE__
|
||||
TensorLoader() : iter_num_(-1), mem_total_(0), mem_usage_(0) {}
|
||||
#else
|
||||
TensorLoader() : mem_total_(0), mem_usage_(0) {}
|
||||
#endif
|
||||
|
||||
~TensorLoader() { EmptyTensor(); }
|
||||
|
||||
|
@ -227,7 +231,9 @@ class TensorLoader {
|
|||
// the pair is (device_id, iteration)
|
||||
std::map<std::string, std::shared_ptr<TensorData>> tensor_list_map_;
|
||||
std::map<std::string, std::shared_ptr<TensorData>> prev_tensor_list_map_;
|
||||
#ifndef __APPLE__
|
||||
uint32_t iter_num_;
|
||||
#endif
|
||||
std::mutex lock_;
|
||||
std::mutex mem_lock_;
|
||||
uint64_t mem_total_;
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
file(GLOB_RECURSE _FL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
|
||||
|
||||
if(NOT ENABLE_CPU OR WIN32 OR APPLE)
|
||||
if(NOT ENABLE_CPU OR WIN32)
|
||||
list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/apply_momentum_kernel.cc")
|
||||
list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/aggregation_kernel_factory.cc")
|
||||
list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/dense_grad_accum_kernel.cc")
|
||||
|
@ -49,6 +49,10 @@ if(NOT ENABLE_CPU OR WIN32 OR APPLE)
|
|||
list(REMOVE_ITEM _FL_SRC_FILES "armour/cipher/cipher_unmask.cc")
|
||||
endif()
|
||||
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-tautological-pointer-compare")
|
||||
endif()
|
||||
|
||||
list(LENGTH _FL_SRC_FILES fl_file_num)
|
||||
if(NOT fl_file_num EQUAL 0)
|
||||
set_property(SOURCE ${_FL_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_FL)
|
||||
|
|
|
@ -104,10 +104,14 @@ bool FLWorker::SendToServer(uint32_t server_rank, const void *data, size_t size,
|
|||
std::this_thread::yield();
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
std::shared_ptr<unsigned char> message(new unsigned char[size], std::default_delete<unsigned char[]>());
|
||||
#else
|
||||
std::shared_ptr<unsigned char[]> message;
|
||||
std::unique_ptr<unsigned char[]> message_addr = std::make_unique<unsigned char[]>(size);
|
||||
MS_EXCEPTION_IF_NULL(message_addr);
|
||||
message = std::move(message_addr);
|
||||
#endif
|
||||
MS_EXCEPTION_IF_NULL(message);
|
||||
|
||||
uint64_t src_size = size;
|
||||
|
|
|
@ -74,9 +74,7 @@ add_subdirectory(core)
|
|||
add_subdirectory(kernels)
|
||||
add_subdirectory(engine)
|
||||
add_subdirectory(api)
|
||||
if(NOT APPLE)
|
||||
add_subdirectory(audio)
|
||||
endif()
|
||||
add_subdirectory(text)
|
||||
add_subdirectory(callback)
|
||||
add_subdirectory(plugin)
|
||||
|
@ -91,16 +89,12 @@ add_dependencies(engine-datasetops-source-sampler core)
|
|||
add_dependencies(engine-datasetops core)
|
||||
add_dependencies(engine-datasetops-mapop core)
|
||||
add_dependencies(engine-opt core)
|
||||
if(NOT APPLE)
|
||||
add_dependencies(engine-gnn core)
|
||||
endif()
|
||||
add_dependencies(engine core)
|
||||
add_dependencies(callback core)
|
||||
if(NOT APPLE)
|
||||
add_dependencies(audio-kernels core)
|
||||
add_dependencies(audio-ir core)
|
||||
add_dependencies(audio-ir-kernels core)
|
||||
endif()
|
||||
add_dependencies(text core)
|
||||
add_dependencies(text-kernels core)
|
||||
add_dependencies(text-ir core)
|
||||
|
@ -161,10 +155,14 @@ set(submodules
|
|||
$<TARGET_OBJECTS:engine-datasetops-source>
|
||||
$<TARGET_OBJECTS:engine-datasetops-source-sampler>
|
||||
$<TARGET_OBJECTS:engine-datasetops-mapop>
|
||||
$<TARGET_OBJECTS:engine-gnn>
|
||||
$<TARGET_OBJECTS:engine-datasetops>
|
||||
$<TARGET_OBJECTS:engine-opt>
|
||||
$<TARGET_OBJECTS:engine-cache-client>
|
||||
$<TARGET_OBJECTS:engine>
|
||||
$<TARGET_OBJECTS:audio-kernels>
|
||||
$<TARGET_OBJECTS:audio-ir>
|
||||
$<TARGET_OBJECTS:audio-ir-kernels>
|
||||
$<TARGET_OBJECTS:text>
|
||||
$<TARGET_OBJECTS:text-kernels>
|
||||
$<TARGET_OBJECTS:text-ir>
|
||||
|
@ -175,15 +173,6 @@ set(submodules
|
|||
$<TARGET_OBJECTS:md_plugin>
|
||||
)
|
||||
|
||||
if(NOT APPLE)
|
||||
set(submodules
|
||||
${submodules}
|
||||
$<TARGET_OBJECTS:engine-gnn>
|
||||
$<TARGET_OBJECTS:audio-kernels>
|
||||
$<TARGET_OBJECTS:audio-ir>
|
||||
$<TARGET_OBJECTS:audio-ir-kernels>)
|
||||
endif()
|
||||
|
||||
if(ENABLE_ACL)
|
||||
set(submodules
|
||||
${submodules}
|
||||
|
@ -223,61 +212,6 @@ endif()
|
|||
################# Link with external libraries ########################
|
||||
target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar)
|
||||
|
||||
set(targetso $<TARGET_FILE:_c_dataengine>)
|
||||
set(target_tiny "libtinyxml")
|
||||
set(target_icuuc "libicuuc")
|
||||
set(target_icudata "libicudata")
|
||||
set(target_icui18n "libicui18n")
|
||||
function(changerpath targetso linkso)
|
||||
set(some-file "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/${linkso}.txt")
|
||||
set(some-file1 "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/${linkso}1.txt")
|
||||
set(some-file2 "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/${linkso}2.txt")
|
||||
add_custom_command(
|
||||
OUTPUT
|
||||
${some-file}
|
||||
COMMAND
|
||||
otool -L ${targetso} | grep ${linkso} > ${some-file}
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset"
|
||||
VERBATIM
|
||||
)
|
||||
add_custom_command(
|
||||
OUTPUT
|
||||
${some-file1}
|
||||
COMMAND
|
||||
cat ${some-file} | cut -d " " -f 1 | sed -E "s/^.//g" > ${some-file1}
|
||||
DEPENDS
|
||||
${some-file}
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset"
|
||||
VERBATIM
|
||||
)
|
||||
add_custom_command(
|
||||
OUTPUT
|
||||
${some-file2}
|
||||
COMMAND
|
||||
awk -F "/" "{print $NF}" ${some-file1} > ${some-file2}
|
||||
DEPENDS
|
||||
${some-file1}
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset"
|
||||
VERBATIM
|
||||
)
|
||||
add_custom_target(
|
||||
link_${linkso} ALL
|
||||
COMMAND install_name_tool -change `cat ${some-file1}` @rpath/`cat ${some-file2}` ${targetso}
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset"
|
||||
DEPENDS ${targetso} ${some-file1} ${some-file2}
|
||||
COMMENT "install tool name")
|
||||
endfunction()
|
||||
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
changerpath(${targetso} ${target_tiny})
|
||||
changerpath(${targetso} ${target_icuuc})
|
||||
add_dependencies(link_${target_icuuc} link_${target_tiny})
|
||||
changerpath(${targetso} ${target_icudata})
|
||||
add_dependencies(link_${target_icudata} link_${target_icuuc})
|
||||
changerpath(${targetso} ${target_icui18n})
|
||||
add_dependencies(link_${target_icui18n} link_${target_icudata})
|
||||
endif()
|
||||
|
||||
if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
|
||||
if(ENABLE_PYTHON)
|
||||
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY})
|
||||
|
@ -320,11 +254,9 @@ else()
|
|||
if(${ENABLE_IBVERBS} STREQUAL "ON")
|
||||
target_link_libraries(_c_dataengine PRIVATE ibverbs rdmacm)
|
||||
endif()
|
||||
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
target_link_libraries(_c_dataengine PRIVATE ps_cache)
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
target_link_libraries(_c_dataengine PRIVATE mindspore_core mindspore_shared_lib)
|
||||
|
||||
|
@ -349,3 +281,9 @@ endif()
|
|||
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
set_target_properties(_c_dataengine PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})
|
||||
endif()
|
||||
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
include(${CMAKE_SOURCE_DIR}/cmake/change_rpath.cmake)
|
||||
set(LINK_SOS libtinyxml libicuuc libicudata libicui18n)
|
||||
changerpath($<TARGET_FILE:_c_dataengine> _c_dataengine "${LINK_SOS}")
|
||||
endif()
|
|
@ -1,7 +1,7 @@
|
|||
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
|
||||
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
|
||||
if(ENABLE_PYTHON)
|
||||
set(API_PYTHON_SRC_FILES
|
||||
add_library(APItoPython OBJECT
|
||||
python/bindings/dataset/audio/bindings.cc
|
||||
python/bindings/dataset/audio/kernels/ir/bindings.cc
|
||||
python/bindings/dataset/callback/bindings.cc
|
||||
|
@ -25,18 +25,10 @@ if(ENABLE_PYTHON)
|
|||
python/pybind_conversion.cc
|
||||
python/pybind_register.cc
|
||||
)
|
||||
if(APPLE)
|
||||
list(REMOVE_ITEM API_PYTHON_SRC_FILES
|
||||
python/bindings/dataset/audio/bindings.cc
|
||||
python/bindings/dataset/audio/kernels/ir/bindings.cc
|
||||
python/bindings/dataset/engine/gnn/bindings.cc
|
||||
)
|
||||
endif()
|
||||
add_library(APItoPython OBJECT ${API_PYTHON_SRC_FILES})
|
||||
target_include_directories(APItoPython PRIVATE ${pybind11_INCLUDE_DIRS})
|
||||
endif()
|
||||
|
||||
set(API_CPP_SRC_FILES
|
||||
add_library(cpp-API OBJECT
|
||||
audio.cc
|
||||
config.cc
|
||||
data_helper.cc
|
||||
|
@ -48,8 +40,3 @@ set(API_CPP_SRC_FILES
|
|||
text.cc
|
||||
vision.cc
|
||||
)
|
||||
|
||||
if(APPLE)
|
||||
list(REMOVE_ITEM API_CPP_SRC_FILES audio.cc)
|
||||
endif()
|
||||
add_library(cpp-API OBJECT ${API_CPP_SRC_FILES})
|
|
@ -937,7 +937,7 @@ Status Flanger(const std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *out
|
|||
cur_channel_phase[j] = static_cast<int>(j * lfo_length * channel_phase + 0.5);
|
||||
// through the current channel phase and lfo arrays to get the delay
|
||||
auto iter_lfo = lfo->begin<float>();
|
||||
delay_tensor[j] = *(iter_lfo + (lfo_pos + cur_channel_phase[j]) % lfo_length);
|
||||
delay_tensor[j] = *(iter_lfo + static_cast<ptrdiff_t>((lfo_pos + cur_channel_phase[j]) % lfo_length));
|
||||
// the frac delay is obtained by using the frac function
|
||||
frac_delay[j] = delay_tensor[j] - static_cast<int>(delay_tensor[j]);
|
||||
delay_tensor[j] = floor(delay_tensor[j]);
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
add_subdirectory(datasetops)
|
||||
add_subdirectory(opt)
|
||||
if(NOT APPLE)
|
||||
add_subdirectory(gnn)
|
||||
endif()
|
||||
add_subdirectory(ir)
|
||||
|
||||
message("ENABLE_SECURITY = ${ENABLE_SECURITY}")
|
||||
|
@ -49,12 +47,10 @@ add_dependencies(engine
|
|||
engine-datasetops
|
||||
engine-datasetops-source
|
||||
engine-opt
|
||||
engine-gnn
|
||||
engine-cache-client
|
||||
engine-datasetops-mapop
|
||||
)
|
||||
if(NOT APPLE)
|
||||
add_dependencies(engine engine-gnn)
|
||||
endif()
|
||||
|
||||
if(NOT ENABLE_SECURITY)
|
||||
add_dependencies(engine engine-perf)
|
||||
|
|
|
@ -48,4 +48,6 @@ else()
|
|||
endif()
|
||||
endif()
|
||||
|
||||
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
set_target_properties(_c_mindrecord PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})
|
||||
endif()
|
||||
|
|
|
@ -46,7 +46,7 @@
|
|||
#include "frontend/optimizer/py_pass_manager.h"
|
||||
#include "utils/ms_context.h"
|
||||
#include "vm/transform.h"
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
#include "ps/parameter_server.h"
|
||||
#include "ps/scheduler.h"
|
||||
#include "ps/worker.h"
|
||||
|
@ -623,7 +623,7 @@ bool OptInlineAction(const ResourcePtr &res) {
|
|||
bool GeOptimizeAction(const ResourcePtr &res) { return OptimizeAction(res, kGePasses); }
|
||||
|
||||
bool VmOptimizeAction(const ResourcePtr &res) {
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
if (ps::PSContext::instance()->is_ps_mode()) {
|
||||
kVmPasses.push_back({"server_communication_op_fusion", ps::Util::FuseServerCommOps});
|
||||
}
|
||||
|
@ -823,7 +823,7 @@ bool ExecuteAction(const ResourcePtr &res) {
|
|||
return true;
|
||||
}
|
||||
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
bool StartPSWorkerAction(const ResourcePtr &) {
|
||||
ps::Worker::GetInstance().Run();
|
||||
return true;
|
||||
|
@ -1163,7 +1163,7 @@ std::vector<ActionItem> VmPipeline() {
|
|||
(void)actions.emplace_back(std::make_pair("eliminate_forward_cnode", EliminateForwardCNode));
|
||||
|
||||
(void)actions.emplace_back(std::make_pair("validate", ValidateAction));
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
if (ps::PSContext::instance()->is_worker()) {
|
||||
std::string server_mode = ps::PSContext::instance()->server_mode();
|
||||
if (server_mode == ps::kServerModeFL || server_mode == ps::kServerModeHybrid) {
|
||||
|
@ -1201,7 +1201,7 @@ std::vector<ActionItem> MindIRPipeline() {
|
|||
(void)actions.emplace_back(std::make_pair("execute", ExecuteAction));
|
||||
return actions;
|
||||
}
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
std::vector<ActionItem> ServerPipeline() {
|
||||
auto actions = CommonPipeline();
|
||||
(void)actions.emplace_back(std::make_pair("optimize", VmOptimizeAction));
|
||||
|
|
|
@ -36,7 +36,7 @@
|
|||
#else
|
||||
#include "runtime/device/gpu/distribution/collective_fake_init.h"
|
||||
#endif
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
#include "ps/util.h"
|
||||
#endif
|
||||
#include "ps/ps_context.h"
|
||||
|
|
|
@ -49,7 +49,7 @@
|
|||
#include "frontend/optimizer/irpass/gradient_eliminate.h"
|
||||
#include "frontend/optimizer/irpass/parameter_eliminate.h"
|
||||
#include "frontend/optimizer/irpass/updatestate_eliminate.h"
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
#include "ps/util.h"
|
||||
#include "ps/ps_context.h"
|
||||
#endif
|
||||
|
@ -218,7 +218,7 @@ namespace {
|
|||
bool ReAutoMonadWrapper(const FuncGraphPtr &root, const opt::OptimizerPtr &) { return ReAutoMonad(root); }
|
||||
|
||||
bool parallel_mode() {
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
if (ps::PSContext::instance()->is_server() || ps::PSContext::instance()->is_scheduler()) {
|
||||
return false;
|
||||
}
|
||||
|
@ -603,7 +603,7 @@ bool SliceRecomputeActivationPass(const ResourcePtr &res) {
|
|||
|
||||
bool AddCacheEmbeddingPass(const ResourcePtr &res) {
|
||||
MS_EXCEPTION_IF_NULL(res);
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
if (ps::PSContext::instance()->is_ps_mode()) {
|
||||
return true;
|
||||
}
|
||||
|
|
|
@ -66,7 +66,7 @@
|
|||
#endif
|
||||
#endif
|
||||
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
#include "ps/constants.h"
|
||||
#include "ps/util.h"
|
||||
#include "ps/worker.h"
|
||||
|
@ -881,7 +881,7 @@ std::vector<ActionItem> GetPipeline(const ResourcePtr &resource, const std::stri
|
|||
|
||||
std::string backend = MsContext::GetInstance()->backend_policy();
|
||||
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
const std::string &server_mode = ps::PSContext::instance()->server_mode();
|
||||
if ((server_mode == ps::kServerModeFL || server_mode == ps::kServerModeHybrid) &&
|
||||
ps::PSContext::instance()->is_server()) {
|
||||
|
@ -1442,7 +1442,7 @@ bool InitExecDataset(const std::string &queue_name, int64_t iter_num, int64_t ba
|
|||
bool InitExecDatasetVm(const std::string &queue_name, int64_t size, int64_t batch_size,
|
||||
const std::vector<TypePtr> &types, const std::vector<std::vector<int64_t>> &shapes,
|
||||
const std::vector<int64_t> &input_indexes, bool need_run) {
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
if ((ps::PSContext::instance()->is_ps_mode()) && (!ps::PSContext::instance()->is_worker())) {
|
||||
return true;
|
||||
}
|
||||
|
@ -1661,7 +1661,7 @@ void FinalizeBackend() {
|
|||
void ClearResAtexit() {
|
||||
MS_LOG(DEBUG) << "Pipeline clear all resource";
|
||||
RecordExitStatus();
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
if (ps::PSContext::instance()->is_ps_mode() && ps::PSContext::instance()->is_worker()) {
|
||||
if (ps::PsDataPrefetch::GetInstance().cache_enable()) {
|
||||
ps::ps_cache_instance.Finalize();
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
file(GLOB_RECURSE _PS_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
|
||||
|
||||
if(NOT ENABLE_CPU OR WIN32 OR APPLE)
|
||||
if(NOT ENABLE_CPU OR WIN32)
|
||||
list(REMOVE_ITEM _PS_SRC_FILES "optimizer_info_builder.cc")
|
||||
list(REMOVE_ITEM _PS_SRC_FILES "optimizer_info.cc")
|
||||
list(REMOVE_ITEM _PS_SRC_FILES "scheduler.cc")
|
||||
|
|
|
@ -192,9 +192,13 @@ const std::map<std::string, size_t> kCiphers = {{"ECDHE-RSA-AES128-GCM-SHA256",
|
|||
{"!PSK", 27},
|
||||
{"kEDH+AESGCM", 28}};
|
||||
|
||||
#ifdef __APPLE__
|
||||
using DataPtr = std::shared_ptr<unsigned char>;
|
||||
#else
|
||||
using DataPtr = std::shared_ptr<unsigned char[]>;
|
||||
#endif
|
||||
using VectorPtr = std::shared_ptr<std::vector<unsigned char>>;
|
||||
using Key = uint64_t;
|
||||
using Key = size_t;
|
||||
using Keys = std::vector<Key>;
|
||||
using Values = std::vector<float>;
|
||||
using ValuesPtr = std::shared_ptr<Values>;
|
||||
|
|
|
@ -956,7 +956,11 @@ void AbstractNode::ProcessSendData(const std::shared_ptr<TcpConnection> &conn, c
|
|||
MS_EXCEPTION_IF_NULL(conn);
|
||||
MS_EXCEPTION_IF_NULL(meta);
|
||||
MS_EXCEPTION_IF_NULL(data);
|
||||
#ifdef __APPLE__
|
||||
std::shared_ptr<unsigned char> res(new unsigned char[size], std::default_delete<unsigned char[]>());
|
||||
#else
|
||||
std::shared_ptr<unsigned char[]> res(new unsigned char[size]);
|
||||
#endif
|
||||
if (size > 0) {
|
||||
size_t dest_size = size;
|
||||
size_t src_size = size;
|
||||
|
|
|
@ -61,7 +61,11 @@ class AbstractNode : public Node {
|
|||
const std::shared_ptr<MessageMeta> &meta, const Protos &protos,
|
||||
const void *data, size_t size);
|
||||
|
||||
#ifdef __APPLE__
|
||||
using DataPtr = std::shared_ptr<unsigned char>;
|
||||
#else
|
||||
using DataPtr = std::shared_ptr<unsigned char[]>;
|
||||
#endif
|
||||
using VectorPtr = std::shared_ptr<std::vector<unsigned char>>;
|
||||
using RequestHandler =
|
||||
std::function<void(const std::shared_ptr<TcpConnection> &conn, const std::shared_ptr<MessageMeta> &meta,
|
||||
|
|
|
@ -84,7 +84,11 @@ class TcpCommunicator : public CommunicatorBase {
|
|||
bool SendPbRequest(const T &pb_msg, const uint32_t &rank_id, TcpUserCommand command,
|
||||
std::shared_ptr<std::vector<unsigned char>> *output = nullptr) {
|
||||
const std::string &msg_str = pb_msg.SerializeAsString();
|
||||
#ifdef __APPLE__
|
||||
std::shared_ptr<unsigned char> msg(new unsigned char[msg_str.size()], std::default_delete<unsigned char[]>());
|
||||
#else
|
||||
std::shared_ptr<unsigned char[]> msg(new unsigned char[msg_str.size()]);
|
||||
#endif
|
||||
MS_ERROR_IF_NULL_W_RET_VAL(msg, false);
|
||||
size_t dest_size = msg_str.size();
|
||||
size_t src_size = msg_str.size();
|
||||
|
|
|
@ -49,7 +49,7 @@ class SchedulerRecovery : public RecoveryBase {
|
|||
|
||||
private:
|
||||
// The node_ will only be instantiated with worker/server node.
|
||||
SchedulerNode *const node_;
|
||||
SchedulerNode *const node_ = nullptr;
|
||||
};
|
||||
} // namespace core
|
||||
} // namespace ps
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
if(ENABLE_CPU AND NOT WIN32 AND NOT APPLE)
|
||||
if(ENABLE_CPU AND NOT WIN32)
|
||||
file(GLOB_RECURSE _PS_CACHE_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "ps_data/*.cc")
|
||||
set_property(SOURCE ${_PS_CACHE_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PS)
|
||||
add_library(ps_cache SHARED ${_PS_CACHE_SRC_FILES})
|
||||
target_link_libraries(ps_cache mindspore::pybind11_module)
|
||||
endif()
|
||||
|
||||
|
||||
|
|
|
@ -231,9 +231,14 @@ void PsCacheManager::AllocMemForHashTable() {
|
|||
device_address.addr = addr;
|
||||
|
||||
auto &host_address = item.second.host_address;
|
||||
#ifdef __APPLE__
|
||||
host_address =
|
||||
std::shared_ptr<float>(new float[host_vocab_cache_size_ * embedding_size], std::default_delete<float[]>());
|
||||
#else
|
||||
std::unique_ptr<float[]> host_hash_table_addr = std::make_unique<float[]>(host_vocab_cache_size_ * embedding_size);
|
||||
MS_EXCEPTION_IF_NULL(host_hash_table_addr);
|
||||
host_address = std::move(host_hash_table_addr);
|
||||
#endif
|
||||
MS_EXCEPTION_IF_NULL(host_address);
|
||||
|
||||
max_embedding_size = (embedding_size > max_embedding_size) ? embedding_size : max_embedding_size;
|
||||
|
|
|
@ -49,7 +49,11 @@ struct HashTableInfo {
|
|||
size_t embedding_size{0};
|
||||
size_t vocab_size{0};
|
||||
Address device_address{nullptr, 0};
|
||||
#ifdef __APPLE__
|
||||
std::shared_ptr<float> host_address{nullptr};
|
||||
#else
|
||||
std::shared_ptr<float[]> host_address{nullptr};
|
||||
#endif
|
||||
ParamInitInfo param_init_info_;
|
||||
};
|
||||
|
||||
|
|
|
@ -18,7 +18,7 @@
|
|||
#include "utils/log_adapter.h"
|
||||
#include "utils/ms_utils.h"
|
||||
#include "backend/kernel_compiler/kernel.h"
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
#include "ps/ps_cache/ps_cache_manager.h"
|
||||
#include "ps/ps_cache/ps_data/ps_data_prefetch.h"
|
||||
#endif
|
||||
|
@ -88,7 +88,7 @@ void PSContext::Reset() {
|
|||
is_worker_ = false;
|
||||
is_pserver_ = false;
|
||||
is_sched_ = false;
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
if (ps::PsDataPrefetch::GetInstance().cache_enable()) {
|
||||
ps_cache_instance.Finalize();
|
||||
set_cache_enable(false);
|
||||
|
@ -144,44 +144,44 @@ uint32_t PSContext::ps_rank_id() const { return rank_id_; }
|
|||
|
||||
void PSContext::InsertHashTableSize(const std::string ¶m_name, size_t cache_vocab_size, size_t embedding_size,
|
||||
size_t vocab_size) const {
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
ps_cache_instance.InsertHashTableSize(param_name, cache_vocab_size, embedding_size, vocab_size);
|
||||
#endif
|
||||
}
|
||||
|
||||
void PSContext::ReInsertHashTableSize(const std::string &new_param_name, const std::string &cur_param_name,
|
||||
size_t cache_vocab_size, size_t embedding_size) const {
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
ps_cache_instance.ReInsertHashTableSize(new_param_name, cur_param_name, cache_vocab_size, embedding_size);
|
||||
#endif
|
||||
}
|
||||
|
||||
void PSContext::InsertWeightInitInfo(const std::string ¶m_name, size_t global_seed, size_t op_seed) const {
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
ps_cache_instance.InsertWeightInitInfo(param_name, global_seed, op_seed);
|
||||
#endif
|
||||
}
|
||||
|
||||
void PSContext::InsertAccumuInitInfo(const std::string ¶m_name, float init_val) const {
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
ps_cache_instance.InsertAccumuInitInfo(param_name, init_val);
|
||||
#endif
|
||||
}
|
||||
|
||||
void PSContext::CloneHashTable(const std::string &dest_param_name, const std::string &src_param_name) const {
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
ps_cache_instance.CloneHashTable(dest_param_name, src_param_name);
|
||||
#endif
|
||||
}
|
||||
|
||||
void PSContext::set_cache_enable(bool cache_enable) const {
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
PsDataPrefetch::GetInstance().set_cache_enable(cache_enable);
|
||||
#endif
|
||||
}
|
||||
|
||||
void PSContext::set_rank_id(uint32_t rank_id) const {
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
ps_cache_instance.set_rank_id(rank_id);
|
||||
#endif
|
||||
}
|
||||
|
|
|
@ -227,7 +227,11 @@ bool Worker::InitPSEmbeddingTable(const size_t &key, const std::vector<size_t> &
|
|||
|
||||
std::string kv_data = embedding_table_meta.SerializeAsString();
|
||||
|
||||
#ifdef __APPLE__
|
||||
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
|
||||
#else
|
||||
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
|
||||
#endif
|
||||
size_t dest_size = kv_data.length();
|
||||
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
|
||||
if (ret != 0) {
|
||||
|
@ -293,8 +297,11 @@ bool Worker::DoPSEmbeddingLookup(const Key &key, const std::vector<int> &lookup_
|
|||
if (messages.at(i).first) {
|
||||
rank_ids.push_back(i);
|
||||
std::string kv_data = messages.at(i).second.SerializeAsString();
|
||||
|
||||
#ifdef __APPLE__
|
||||
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
|
||||
#else
|
||||
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
|
||||
#endif
|
||||
size_t dest_size = kv_data.length();
|
||||
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
|
||||
if (ret != 0) {
|
||||
|
@ -382,7 +389,11 @@ bool Worker::UpdateEmbeddingTable(const std::vector<Key> &keys, const std::vecto
|
|||
rank_ids.push_back(i);
|
||||
std::string kv_data = messages.at(i).second.SerializeAsString();
|
||||
|
||||
#ifdef __APPLE__
|
||||
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
|
||||
#else
|
||||
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
|
||||
#endif
|
||||
size_t dest_size = kv_data.length();
|
||||
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
|
||||
if (ret != 0) {
|
||||
|
@ -403,7 +414,11 @@ void Worker::Finalize() {
|
|||
kvs.add_keys(0);
|
||||
kvs.add_values(0.0f);
|
||||
std::string kv_data = kvs.SerializeAsString();
|
||||
#ifdef __APPLE__
|
||||
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
|
||||
#else
|
||||
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
|
||||
#endif
|
||||
size_t dest_size = kv_data.length();
|
||||
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
|
||||
if (ret != 0) {
|
||||
|
@ -644,7 +659,11 @@ void Worker::PushData(const std::vector<Key> &keys, const std::vector<float> &va
|
|||
SendForPush(cmd, kvs, worker_init_embedding_partitioner_, {});
|
||||
} else {
|
||||
std::string kv_data = kvs.SerializeAsString();
|
||||
#ifdef __APPLE__
|
||||
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
|
||||
#else
|
||||
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
|
||||
#endif
|
||||
size_t dest_size = kv_data.length();
|
||||
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
|
||||
if (ret != 0) {
|
||||
|
@ -950,7 +969,11 @@ void Worker::SendForPush(int cmd, const KVMessage &send, const KVPartitioner &pa
|
|||
rank_ids.push_back(i);
|
||||
std::string kv_data = messages.at(i).second.SerializeAsString();
|
||||
|
||||
#ifdef __APPLE__
|
||||
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
|
||||
#else
|
||||
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
|
||||
#endif
|
||||
size_t dest_size = kv_data.length();
|
||||
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
|
||||
if (ret != 0) {
|
||||
|
@ -977,7 +1000,11 @@ void Worker::SendForPull(int cmd, const KVMessage &send, const KVPartitioner &pa
|
|||
rank_ids.push_back(i);
|
||||
std::string kv_data = messages.at(i).second.SerializeAsString();
|
||||
|
||||
#ifdef __APPLE__
|
||||
std::shared_ptr<unsigned char> res(new unsigned char[kv_data.length()], std::default_delete<unsigned char[]>());
|
||||
#else
|
||||
std::shared_ptr<unsigned char[]> res(new unsigned char[kv_data.length()]);
|
||||
#endif
|
||||
size_t dest_size = kv_data.length();
|
||||
int ret = memcpy_s(res.get(), dest_size, kv_data.data(), kv_data.length());
|
||||
if (ret != 0) {
|
||||
|
|
|
@ -33,7 +33,7 @@
|
|||
#include "frontend/parallel/context.h"
|
||||
#include "debug/env_config_parser.h"
|
||||
#include "pipeline/pynative/pynative_profiling.h"
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
#include "ps/ps_cache/ps_cache_manager.h"
|
||||
#endif
|
||||
|
||||
|
@ -576,7 +576,7 @@ void KernelRuntime::AssignStaticMemoryInput(const session::KernelGraph &graph) {
|
|||
}
|
||||
add_need_alloc_nodes(input_node);
|
||||
}
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
bool ps_cache_check = false;
|
||||
#endif
|
||||
for (auto &item : need_alloc_nodes) {
|
||||
|
@ -590,7 +590,7 @@ void KernelRuntime::AssignStaticMemoryInput(const session::KernelGraph &graph) {
|
|||
continue;
|
||||
}
|
||||
DeviceAddressPtr device_address = nullptr;
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
const std::string ¶m_name = item->fullname_with_scope();
|
||||
if (ps::ps_cache_instance.IsHashTable(param_name)) {
|
||||
MS_LOG(INFO) << "Parameter(" << param_name << ")"
|
||||
|
@ -1598,7 +1598,7 @@ void KernelRuntime::ClearGraphRuntimeResource(uint32_t graph_id) {
|
|||
MS_LOG(INFO) << "Clear graph:" << graph_id << " runtime resource";
|
||||
}
|
||||
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
void KernelRuntime::GetFirstPSEmbeddingCache(const session::KernelGraph &graph,
|
||||
AnfNodePtr *const first_cache_input_index,
|
||||
size_t *const first_cache_size) {
|
||||
|
|
|
@ -173,7 +173,7 @@ class KernelRuntime {
|
|||
void RunOpAssignOutputNodeMemory(const ValuePtr &pre_output_value, const session::KernelGraph &graph);
|
||||
void AssignValueNodeTensor(const ValueNodePtr &value_node, const ValuePtr &node_value, size_t output_idx);
|
||||
DeviceAddressPtr PreAssignCNodeMemory(const AnfNodePtr &anf_node, size_t index) const;
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__)))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
void GetFirstPSEmbeddingCache(const session::KernelGraph &graph, AnfNodePtr *const first_cache_input_index,
|
||||
size_t *const first_cache_size);
|
||||
void CheckIfSupportPSEmbeddingCache(const session::KernelGraph &graph);
|
||||
|
|
|
@ -16,7 +16,7 @@
|
|||
|
||||
#include "runtime/device/kernel_runtime_manager.h"
|
||||
#include "utils/log_adapter.h"
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
#include "ps/ps_cache/ps_cache_manager.h"
|
||||
#endif
|
||||
#include "backend/session/pynative_task_manager.h"
|
||||
|
@ -26,7 +26,7 @@ namespace device {
|
|||
void KernelRuntimeManager::ClearRuntimeResource() {
|
||||
// Just remove PyNative tasks before runtime resource release.
|
||||
session::PynativeTaskManager::GetInstance().Reset();
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
if (ps::PSContext::instance()->is_worker() && ps::PsDataPrefetch::GetInstance().cache_enable()) {
|
||||
ps::ps_cache_instance.SyncEmbeddingTable();
|
||||
}
|
||||
|
@ -132,7 +132,7 @@ void KernelRuntimeManager::ReleaseKernelRuntime(const std::string &device_name,
|
|||
if (runtime == nullptr) {
|
||||
return;
|
||||
}
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
if (ps::PSContext::instance()->is_worker() && ps::PsDataPrefetch::GetInstance().cache_enable()) {
|
||||
ps::ps_cache_instance.SyncEmbeddingTable();
|
||||
}
|
||||
|
|
|
@ -67,7 +67,7 @@ void SuperKernelActor::Run(OpContext<DeviceTensor> *const context) {
|
|||
MS_LOG(INFO) << "Super kernel actor(" << GetAID().Name()
|
||||
<< ") launches graph: " << std::to_string(graph_->graph_id());
|
||||
if (!CopyInputData(context)) {
|
||||
std::string error_info = "Copy the input data failed, graph id: " + graph_->graph_id();
|
||||
std::string error_info = "Copy the input data failed, graph id: " + std::to_string(graph_->graph_id());
|
||||
SET_OPCONTEXT_FAIL_RET_WITH_ERROR((*context), error_info);
|
||||
}
|
||||
|
||||
|
|
|
@ -632,7 +632,7 @@ void SetMindRTEnable() {
|
|||
return;
|
||||
#endif
|
||||
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__))
|
||||
#if ((defined ENABLE_CPU) && (!defined _WIN32))
|
||||
if (ps::PSContext::instance()->is_ps_mode()) {
|
||||
return;
|
||||
}
|
||||
|
|
|
@ -31,15 +31,13 @@ Common imported modules in corresponding API examples are as follows:
|
|||
from mindspore.dataset.transforms import c_transforms
|
||||
"""
|
||||
|
||||
import platform
|
||||
from .core import config
|
||||
from .engine import *
|
||||
from .engine.cache_client import DatasetCache
|
||||
from .engine.datasets import *
|
||||
from .engine.graphdata import GraphData
|
||||
from .engine.samplers import *
|
||||
from .engine.serializer_deserializer import compare, deserialize, serialize, show
|
||||
if platform.system().lower() != "darwin":
|
||||
from .engine.graphdata import GraphData
|
||||
|
||||
__all__ = []
|
||||
__all__.extend(engine.__all__)
|
||||
|
|
|
@ -22,23 +22,19 @@ high performance and parse data precisely. It also provides the following
|
|||
operations for users to preprocess data: shuffle, batch, repeat, map, and zip.
|
||||
"""
|
||||
|
||||
import platform
|
||||
from ..callback import DSCallback, WaitedDSCallback
|
||||
from ..core import config
|
||||
from .cache_client import DatasetCache
|
||||
from .datasets import *
|
||||
from .graphdata import GraphData, SamplingStrategy, OutputFormat
|
||||
from .iterators import *
|
||||
from .samplers import *
|
||||
from .serializer_deserializer import compare, deserialize, serialize, show
|
||||
if platform.system().lower() != "darwin":
|
||||
from .graphdata import GraphData, SamplingStrategy, OutputFormat
|
||||
|
||||
__all__ = ["CelebADataset", "Cifar100Dataset", "Cifar10Dataset", "CLUEDataset", "CocoDataset", "CSVDataset",
|
||||
"GeneratorDataset", "ImageFolderDataset", "ManifestDataset", "MindDataset", "MnistDataset",
|
||||
"GeneratorDataset", "GraphData", "ImageFolderDataset", "ManifestDataset", "MindDataset", "MnistDataset",
|
||||
"NumpySlicesDataset", "PaddedDataset", "TextFileDataset", "TFRecordDataset", "VOCDataset",
|
||||
"DistributedSampler", "PKSampler", "RandomSampler", "SequentialSampler", "SubsetRandomSampler",
|
||||
"WeightedRandomSampler", "SubsetSampler",
|
||||
"DatasetCache", "DSCallback", "Schema", "WaitedDSCallback", "compare", "deserialize",
|
||||
"serialize", "show", "zip"]
|
||||
if platform.system().lower() != "darwin":
|
||||
__all__.append("GraphData")
|
||||
|
|
|
@ -1,15 +1,21 @@
|
|||
diff -Npur grpc/..rej grpc-patch/..rej
|
||||
--- grpc/..rej 1970-01-01 08:00:00.000000000 +0800
|
||||
+++ grpc-patch/..rej 2021-04-22 21:00:17.343178600 +0800
|
||||
@@ -0,0 +1,22 @@
|
||||
@@ -0,0 +1,28 @@
|
||||
+--- CMakeLists.txt 2020-02-27 03:12:33.000000000 +0800
|
||||
++++ CMakeLists.txt 2021-04-07 21:27:12.317207600 +0800
|
||||
+@@ -12992,7 +12992,7 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
|
||||
+@@ -12992,7 +12992,13 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
|
||||
+ add_executable(grpc_cpp_plugin
|
||||
+ src/compiler/cpp_plugin.cc
|
||||
+ )
|
||||
+-
|
||||
++if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
++ set(CMAKE_MACOSX_RPATH 1)
|
||||
++ set(CMAKE_INSTALL_RPATH "@executable_path/../lib")
|
||||
++ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}")
|
||||
++else()
|
||||
++ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
|
||||
++endif()
|
||||
+ target_include_directories(grpc_cpp_plugin
|
||||
+ PRIVATE
|
||||
+ ${CMAKE_CURRENT_SOURCE_DIR}
|
||||
|
@ -27,15 +33,21 @@ diff -Npur grpc/..rej grpc-patch/..rej
|
|||
diff -Npur grpc/.rej grpc-patch/.rej
|
||||
--- grpc/.rej 1970-01-01 08:00:00.000000000 +0800
|
||||
+++ grpc-patch/.rej 2021-04-22 21:03:38.192349100 +0800
|
||||
@@ -0,0 +1,22 @@
|
||||
@@ -0,0 +1,28 @@
|
||||
+--- grpc/CMakeLists.txt 2020-02-27 03:12:33.000000000 +0800
|
||||
++++ grpc-patch/CMakeLists.txt 2021-04-07 21:27:12.317207600 +0800
|
||||
+@@ -12992,7 +12992,7 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
|
||||
+@@ -12992,7 +12992,13 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
|
||||
+ add_executable(grpc_cpp_plugin
|
||||
+ src/compiler/cpp_plugin.cc
|
||||
+ )
|
||||
+-
|
||||
++if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
++ set(CMAKE_MACOSX_RPATH 1)
|
||||
++ set(CMAKE_INSTALL_RPATH "@executable_path/../lib")
|
||||
++ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}")
|
||||
++else()
|
||||
++ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
|
||||
++endif()
|
||||
+ target_include_directories(grpc_cpp_plugin
|
||||
+ PRIVATE
|
||||
+ ${CMAKE_CURRENT_SOURCE_DIR}
|
||||
|
@ -107,12 +119,18 @@ diff -Npur grpc/CMakeLists.txt grpc-patch/CMakeLists.txt
|
|||
if(WIN32 AND MSVC)
|
||||
set_target_properties(upb PROPERTIES COMPILE_PDB_NAME "upb"
|
||||
COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}"
|
||||
@@ -12992,7 +13004,7 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
|
||||
@@ -12992,7 +13004,13 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
|
||||
add_executable(grpc_cpp_plugin
|
||||
src/compiler/cpp_plugin.cc
|
||||
)
|
||||
-
|
||||
+if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
+ set(CMAKE_MACOSX_RPATH 1)
|
||||
+ set(CMAKE_INSTALL_RPATH "@executable_path/../lib")
|
||||
+ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}")
|
||||
+else()
|
||||
+ set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
|
||||
+endif()
|
||||
target_include_directories(grpc_cpp_plugin
|
||||
PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR}
|
||||
|
|
|
@ -10,3 +10,8 @@ diff -Npur libevent/CMakeLists.txt libevent-modify/CMakeLists.txt
|
|||
endif()
|
||||
|
||||
if (NOT EVENT__DISABLE_THREAD_SUPPORT)
|
||||
diff -Npur libevent/cmake/AddEventLibrary.cmake libevent-modify/cmake/AddEventLibrary.cmake
|
||||
--- libevent/cmake/AddEventLibrary.cmake 2020-07-05 20:02:46.000000000 +0800
|
||||
+++ libevent-modify/cmake/AddEventLibrary.cmake 2021-04-19 16:36:57.982307500 +0800
|
||||
@@ -153,1 +153,0 @@
|
||||
- INSTALL_NAME_DIR "${CMAKE_INSTALL_PREFIX}/lib"
|
||||
|
|
Loading…
Reference in New Issue