Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

mac build

This commit is contained in:
xiaoyisd 2020-12-04 17:12:51 +08:00
parent ec3983b77d
commit 424999267a
49 changed files with 340 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@ -13,7 +13,8 @@ if (NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
endif ()
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Werror -Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare -Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move -Wno-inconsistent-missing-override -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2")
set(CMAKE_OSX_SYSROOT "")
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Winconsistent-missing-override -Wuser-defined-warnings -Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare -Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move -Wno-inconsistent-missing-override -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2")
else()
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Wl,--allow-shlib-undefined -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2")
endif()

33
cmake/external_libs/icu4c.cmake
View File

@ -5,14 +5,14 @@ set(LIB_ICU_I18N icui18n)
if (ENABLE_GITEE)
set(REQ_URL "https://gitee.com/mirrors/icu/repository/archive/release-67-1.tar.gz")
set(MD5 "72415ffd1af3acf19f9aa3fa82c7b5bc")
else()
else ()
set(REQ_URL "https://github.com/unicode-org/icu/archive/release-67-1.tar.gz")
set(MD5 "fd525fb47d8827b0b7da78b51dd2d93f")
endif ()
if (CMAKE_SYSTEM_NAME MATCHES "Windows")
message("icu4c thirdparty do not support windows currently.")
else()
else ()
set(JSON_FILE "{ \n\
\"strategy\": \"additive\",\n\
\"featureFilters\": {\n\
@ -21,17 +21,28 @@ else()
}\
")
file(WRITE ${CMAKE_BINARY_DIR}/icu4c_filter.json ${JSON_FILE})
mindspore_add_pkg(icu4c
VER 67.1
LIBS ${LIB_ICU_COMMON} ${LIB_ICU_DATA} ${LIB_ICU_I18N}
URL ${REQ_URL}
MD5 ${MD5}
PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/icu4c/icu4c.patch01
CONFIGURE_COMMAND ./icu4c/source/runConfigureICU Linux --enable-rpath --disable-tests --disable-samples --disable-icuio --disable-extras ICU_DATA_FILTER_FILE=${CMAKE_BINARY_DIR}/icu4c_filter.json
)
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
mindspore_add_pkg(icu4c
VER 67.1
LIBS ${LIB_ICU_COMMON} ${LIB_ICU_DATA} ${LIB_ICU_I18N}
URL ${REQ_URL}
MD5 ${MD5}
PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/icu4c/icu4c.patch01
CONFIGURE_COMMAND ./icu4c/source/runConfigureICU MacOSX --enable-rpath --disable-tests --disable-samples --disable-icuio --disable-extras ICU_DATA_FILTER_FILE=${CMAKE_BINARY_DIR}/icu4c_filter.json
)
else ()
mindspore_add_pkg(icu4c
VER 67.1
LIBS ${LIB_ICU_COMMON} ${LIB_ICU_DATA} ${LIB_ICU_I18N}
URL ${REQ_URL}
MD5 ${MD5}
PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/icu4c/icu4c.patch01
CONFIGURE_COMMAND ./icu4c/source/runConfigureICU Linux --enable-rpath --disable-tests --disable-samples --disable-icuio --disable-extras ICU_DATA_FILTER_FILE=${CMAKE_BINARY_DIR}/icu4c_filter.json
)
endif ()
include_directories(${icu4c_INC})
add_library(mindspore::icuuc ALIAS icu4c::${LIB_ICU_COMMON})
add_library(mindspore::icudata ALIAS icu4c::${LIB_ICU_DATA})
add_library(mindspore::icui18n ALIAS icu4c::${LIB_ICU_I18N})
add_definitions(-D ENABLE_ICU4C)
endif()
endif ()

4
cmake/external_libs/libevent.cmake
View File

@ -1,5 +1,7 @@
set(libevent_CFLAGS "-fstack-protector-all -D_FORTIFY_SOURCE=2 -O2")
set(libevent_LDFLAGS "-Wl,-z,now")
if (NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(libevent_LDFLAGS "-Wl,-z,now")
endif ()
if (ENABLE_GITEE)
set(REQ_URL "https://gitee.com/mirrors/libevent/repository/archive/release-2.1.12-stable.tar.gz")

2
cmake/external_libs/sqlite.cmake
View File

@ -24,8 +24,8 @@ else ()
set(sqlite_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
else()
set(sqlite_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
endif()
set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
mindspore_add_pkg(sqlite
VER 3.32.2
LIBS sqlite3

8
cmake/external_libs/tinyxml2.cmake
View File

@ -4,15 +4,15 @@ set(tinyxml2_CFLAGS "-fstack-protector -D_FORTIFY_SOURCE=2 -O2")
if (ENABLE_GITEE)
set(REQ_URL "https://gitee.com/mirrors/tinyxml2/repository/archive/8.0.0.tar.gz")
set(MD5 "6a70cea637d0b17179e8bfd77860f811")
else()
else ()
set(REQ_URL "https://github.com/leethomason/tinyxml2/archive/8.0.0.tar.gz")
set(MD5 "5dc535c8b34ee621fe2128f072d275b5")
endif ()
if (NOT WIN32)
set(tinyxml2_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
endif()
if (NOT WIN32 AND NOT APPLE)
set(tinyxml2_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
endif ()
mindspore_add_pkg(tinyxml2
VER 8.0.0

25
mindspore/ccsrc/CMakeLists.txt
View File

@ -2,7 +2,10 @@
include_directories(${CMAKE_SOURCE_DIR}/mindspore/core)
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
include_directories(${CMAKE_BINARY_DIR})
link_directories(${CMAKE_SOURCE_DIR}/build/mindspore/graphengine)
if (NOT(CMAKE_SYSTEM_NAME MATCHES "Darwin"))
link_directories(${CMAKE_SOURCE_DIR}/build/mindspore/graphengine)
endif ()
if (CMAKE_SYSTEM_NAME MATCHES "Windows")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF")
@ -252,9 +255,11 @@ if (ENABLE_D)
target_link_libraries(mindspore -Wl,--start-group proto_input mindspore::protobuf -Wl,--end-group)
elseif (CMAKE_SYSTEM_NAME MATCHES "Windows")
target_link_libraries(mindspore -Wl,--start-group proto_input mindspore::protobuf mindspore::sentencepiece -Wl,--end-group)
elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(mindspore -Wl proto_input mindspore::protobuf mindspore::sentencepiece -Wl)
else ()
target_link_libraries(mindspore -Wl,--start-group proto_input mindspore::protobuf -Wl,--end-group)
endif()
endif ()
# set c_expression building
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
@ -296,6 +301,10 @@ if (CMAKE_SYSTEM_NAME MATCHES "Windows")
target_link_libraries(mindspore mindspore::pybind11_module)
target_link_libraries(mindspore mindspore_gvar)
target_link_libraries(_c_expression PRIVATE -Wl,--whole-archive mindspore -Wl,--no-whole-archive)
elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(mindspore mindspore::pybind11_module)
target_link_libraries(mindspore mindspore_gvar)
target_link_libraries(_c_expression PRIVATE -Wl,-force_load mindspore -Wl,-noall_load)
else ()
if (ENABLE_CPU AND (ENABLE_D OR ENABLE_GPU))
target_link_libraries(mindspore mindspore::pslite proto_input mindspore::protobuf mindspore::event mindspore::event_pthreads ${zeromq_DIRPATH}/zmq_install/lib/libzmq.a)
@ -326,7 +335,7 @@ if (ENABLE_GPU)
${CUDA_PATH}/lib64/libcusolver.so)
if (ENABLE_MPI)
set_target_properties(_ms_mpi PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})
endif()
endif ()
endif ()
if (ENABLE_CPU)
@ -349,8 +358,14 @@ add_library(inference SHARED
)
set_target_properties(inference PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})
target_link_libraries(inference PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,--whole-archive mindspore proto_input -Wl,--no-whole-archive mindspore_gvar)
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(inference PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,-force_load mindspore proto_input -Wl,-noall_load mindspore_gvar)
else()
target_link_libraries(inference PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,--whole-archive mindspore proto_input -Wl,--no-whole-archive mindspore_gvar)
endif()
if (ENABLE_D)
find_library(adump_server libadump_server.a ${ASCEND_RUNTIME_PATH} ${ASCEND_TOOLKIT_RUNTIME_PATH})

4
mindspore/ccsrc/backend/kernel_compiler/CMakeLists.txt
View File

@ -5,6 +5,10 @@ file(GLOB_RECURSE KERNEL_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"oplib/*.cc"
)
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor")
endif ()
if (ENABLE_D)
file(GLOB_RECURSE D_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"kernel_query.cc"

2
mindspore/ccsrc/backend/kernel_compiler/cpu/sigmoid_cross_entropy_with_logits_cpu_kernel.cc
View File

@ -22,7 +22,7 @@ namespace kernel {
void SigmoidCrossEntropyWithLogitsCPUKernel::InitKernel(const CNodePtr &kernel_node) {
CheckParam(kernel_node);
dtype_ = AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, 0);
std::vector<uint64_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
std::vector<size_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
for (const uint64_t &d : x_shape) {
tensor_size_ *= d;
}

2
mindspore/ccsrc/backend/kernel_compiler/cpu/sigmoid_cross_entropy_with_logits_grad_cpu_kernel.cc
View File

@ -22,7 +22,7 @@ namespace kernel {
void SigmoidCrossEntropyWithLogitsGradCPUKernel::InitKernel(const CNodePtr &kernel_node) {
CheckParam(kernel_node);
dtype_ = AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, 0);
std::vector<uint64_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
std::vector<size_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
for (const uint64_t &d : x_shape) {
tensor_size_ *= d;
}

2
mindspore/ccsrc/backend/kernel_compiler/cpu/smooth_l1_loss_cpu_kernel.cc
View File

@ -23,7 +23,7 @@ void SmoothL1LossCPUKernel::InitKernel(const CNodePtr &kernel_node) {
beta_ = AnfAlgo::GetNodeAttr<float>(kernel_node, "beta");
CheckParam(kernel_node);
dtype_ = AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, 0);
std::vector<uint64_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
std::vector<size_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
for (const uint64_t &d : x_shape) {
tensor_size_ *= d;
}

2
mindspore/ccsrc/backend/kernel_compiler/cpu/smooth_l1_loss_grad_cpu_kernel.cc
View File

@ -23,7 +23,7 @@ void SmoothL1LossGradCPUKernel::InitKernel(const CNodePtr &kernel_node) {
beta_ = AnfAlgo::GetNodeAttr<float>(kernel_node, "beta");
CheckParam(kernel_node);
dtype_ = AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, 0);
std::vector<uint64_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
std::vector<size_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
for (const uint64_t &d : x_shape) {
tensor_size_ *= d;
}

4
mindspore/ccsrc/backend/optimizer/CMakeLists.txt
View File

@ -21,5 +21,9 @@ if (ENABLE_GPU)
list(APPEND _PREACTIVATE_SRC_LIST ${_GPU_SRC_LIST})
endif ()
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -Wno-user-defined-warnings -Wno-inconsistent-missing-override -Wno-overloaded-virtual -Wno-unused-const-variable -Wno-pessimizing-move")
endif ()
set_property(SOURCE ${_PREACTIVATE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PRE_ACT)
add_library(_mindspore_backend_optimizer_obj OBJECT ${_PREACTIVATE_SRC_LIST})

12
mindspore/ccsrc/backend/session/CMakeLists.txt
View File

@ -8,17 +8,17 @@ file(GLOB_RECURSE _SESSION_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"anf_runtime_algorithm.cc"
)
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-overloaded-virtual")
endif ()
if (ENABLE_GPU)
file(GLOB_RECURSE _GPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"gpu_session.cc"
)
file(GLOB_RECURSE _GPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "gpu_session.cc")
list(APPEND _SESSION_SRC_LIST ${_GPU_SRC_LIST})
endif ()
if (ENABLE_CPU)
file(GLOB_RECURSE _CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"cpu_session.cc"
)
file(GLOB_RECURSE _CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "cpu_session.cc")
list(APPEND _SESSION_SRC_LIST ${_CPU_SRC_LIST})
endif ()

2
mindspore/ccsrc/backend/session/executor.cc
View File

@ -257,7 +257,7 @@ GraphId Executor::CompileGraph(const SessionPtr &session, const GraphSegmentPtr
GraphId Executor::CompileGraph(const SessionPtr &session, NotNull<FuncGraphPtr> func_graph) {
auto task = std::make_shared<CompileGraphTask>();
task->session_ = session;
task->func_graph_ = func_graph;
task->func_graph_ = func_graph.get();
SyncRunTask(task);
return task->graph_id_;
}

4
mindspore/ccsrc/backend/session/kernel_graph.cc
View File

@ -472,7 +472,11 @@ void KernelGraph::SetKernelInfoForNode(const AnfNodePtr &node) const {
if (kOpAssignKernelNameList.find(AnfAlgo::GetCNodeName(node)) != kOpAssignKernelNameList.end()) {
ResetAssignInputFeaatureMapFlag(node->cast<CNodePtr>());
}
#ifdef __APPLE__
std::vector<int> feature_map_input_indexs;
#else
std::vector<size_t> feature_map_input_indexs;
#endif
kernel_info->set_feature_map_flag(false);
for (size_t index = 0; index < AnfAlgo::GetInputTensorNum(node); ++index) {
if (AnfAlgo::IsFeatureMapInput(node, index)) {

4
mindspore/ccsrc/backend/session/session_basic.h
View File

@ -53,7 +53,11 @@ struct OpRunInfo {
bool is_dynamic_shape = false;
bool is_auto_mixed_precision = false;
std::string next_op_name = "";
#ifdef __APPLE__
int next_input_index = 0;
#else
size_t next_input_index = 0;
#endif
};
using OpRunInfoPtr = std::shared_ptr<OpRunInfo>;
class Executor;

9
mindspore/ccsrc/cxx_api/CMakeLists.txt
View File

@ -36,8 +36,13 @@ set(MSLIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/types.cc
add_library(mindspore_shared_lib SHARED ${MSLIB_SRC})
set_target_properties(mindspore_shared_lib PROPERTIES OUTPUT_NAME mindspore)
target_link_libraries(mindspore_shared_lib PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,--whole-archive mindspore -Wl,--no-whole-archive proto_input mindspore_gvar mindspore::protobuf)
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(mindspore_shared_lib PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,-force_load mindspore -Wl,-noall_load proto_input mindspore_gvar mindspore::protobuf)
else ()
target_link_libraries(mindspore_shared_lib PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,--whole-archive mindspore -Wl,--no-whole-archive proto_input mindspore_gvar mindspore::protobuf)
endif ()
if (ENABLE_CPU)
target_link_libraries(mindspore_shared_lib PRIVATE mindspore::dnnl mindspore::mkldnn)

5
mindspore/ccsrc/frontend/optimizer/CMakeLists.txt
View File

@ -1,3 +1,8 @@
file(GLOB_RECURSE _OPTIMIZER_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor")
endif ()
set_property(SOURCE ${_OPTIMIZER_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_OPTIMIZER)
add_library(_mindspore_frontend_optimizer_obj OBJECT ${_OPTIMIZER_SRC_FILES})

4
mindspore/ccsrc/frontend/parallel/CMakeLists.txt
View File

@ -4,5 +4,9 @@ if (ENABLE_DUMP_PROTO)
list(REMOVE_ITEM _PARALLEL_SRC_FILES "parallel/strategy_checkpoint/parallel_strategy_checkpoint.cc")
endif ()
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor")
endif ()
set_property(SOURCE ${_PARALLEL_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PARALLEL)
add_library(_mindspore_frontend_parallel_obj OBJECT ${_PARALLEL_SRC_FILES})

2
mindspore/ccsrc/frontend/parallel/graph_util/node_info.cc
View File

@ -111,7 +111,7 @@ size_t GetLengthOfDataType(const TypePtr &type) {
case kNumberTypeInt:
return sizeof(int64_t);
case kNumberTypeUInt:
return sizeof(unsigned int64_t);
return sizeof(unsigned);
case kNumberTypeFloat:
return sizeof(float);
default:

99
mindspore/ccsrc/minddata/dataset/CMakeLists.txt
View File

@ -3,7 +3,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reorder")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-switch")
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-uninitialized")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-uninitialized -Wno-deprecated-declarations -Wno-delete-non-abstract-non-virtual-dtor -Wno-constant-conversion")
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-maybe-uninitialized")
endif()
@ -49,16 +49,20 @@ include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/kernels
####################### Flags ########################################
# compile flags
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath,$ORIGIN:$ORIGIN/lib")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default")
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-abstract-non-virtual-dtor")
else ()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath,$ORIGIN:$ORIGIN/lib")
endif()
################## Include sub-modules ###############################
add_subdirectory(util)
add_subdirectory(core)
add_subdirectory(kernels)
add_subdirectory(engine)
add_subdirectory(api)
add_subdirectory(text)
add_subdirectory(callback)
######################################################################
add_dependencies(utils core)
@ -75,16 +79,20 @@ add_dependencies(engine-perf core)
add_dependencies(engine-gnn core)
add_dependencies(engine core)
add_dependencies(callback core)
add_dependencies(text core)
add_dependencies(text-kernels core)
add_dependencies(cpp-API core)
add_dependencies(engine-ir-datasetops core)
add_dependencies(engine-ir-datasetops-source core)
add_dependencies(engine-ir-cache core)
if (NOT(${CMAKE_SYSTEM_NAME} MATCHES "Darwin"))
add_subdirectory(text)
add_dependencies(text core)
add_dependencies(text-kernels core)
endif ()
if (ENABLE_PYTHON)
add_dependencies(APItoPython core)
endif()
endif ()
if (ENABLE_TDTQUE)
add_dependencies(engine-tdt core)
endif ()
@ -100,37 +108,42 @@ if (ENABLE_CACHE)
endif ()
################### Create _c_dataengine Library ######################
set(submodules
$<TARGET_OBJECTS:core>
$<TARGET_OBJECTS:callback>
$<TARGET_OBJECTS:utils>
$<TARGET_OBJECTS:kernels>
$<TARGET_OBJECTS:kernels-image>
$<TARGET_OBJECTS:lite-cv>
$<TARGET_OBJECTS:kernels-data>
$<TARGET_OBJECTS:cpp-API>
$<TARGET_OBJECTS:engine-ir-datasetops>
$<TARGET_OBJECTS:core>
$<TARGET_OBJECTS:callback>
$<TARGET_OBJECTS:utils>
$<TARGET_OBJECTS:kernels>
$<TARGET_OBJECTS:kernels-image>
$<TARGET_OBJECTS:lite-cv>
$<TARGET_OBJECTS:kernels-data>
$<TARGET_OBJECTS:cpp-API>
$<TARGET_OBJECTS:engine-ir-datasetops>
$<TARGET_OBJECTS:engine-ir-datasetops-source>
$<TARGET_OBJECTS:engine-ir-cache>
$<TARGET_OBJECTS:kernels-soft-dvpp-image>
$<TARGET_OBJECTS:soft-dvpp-utils>
$<TARGET_OBJECTS:engine-datasetops-source>
$<TARGET_OBJECTS:engine-datasetops-source-sampler>
$<TARGET_OBJECTS:engine-datasetops-mapop>
$<TARGET_OBJECTS:engine-gnn>
$<TARGET_OBJECTS:engine-perf>
$<TARGET_OBJECTS:engine-datasetops>
$<TARGET_OBJECTS:engine-opt>
$<TARGET_OBJECTS:engine-cache-client>
$<TARGET_OBJECTS:engine>
$<TARGET_OBJECTS:text>
$<TARGET_OBJECTS:text-kernels>
)
$<TARGET_OBJECTS:soft-dvpp-utils>
$<TARGET_OBJECTS:engine-datasetops-source>
$<TARGET_OBJECTS:engine-datasetops-source-sampler>
$<TARGET_OBJECTS:engine-datasetops-mapop>
$<TARGET_OBJECTS:engine-gnn>
$<TARGET_OBJECTS:engine-perf>
$<TARGET_OBJECTS:engine-datasetops>
$<TARGET_OBJECTS:engine-opt>
$<TARGET_OBJECTS:engine-cache-client>
$<TARGET_OBJECTS:engine>
)
if (NOT(${CMAKE_SYSTEM_NAME} MATCHES "Darwin"))
set(submodules
${submodules}
$<TARGET_OBJECTS:text>
$<TARGET_OBJECTS:text-kernels>
)
endif ()
if (ENABLE_PYTHON)
set(submodules
${submodules}
$<TARGET_OBJECTS:APItoPython>)
endif()
endif ()
if (ENABLE_TDTQUE)
add_library(_c_dataengine SHARED ${submodules} $<TARGET_OBJECTS:engine-tdt>)
@ -143,7 +156,7 @@ set_target_properties(_c_dataengine PROPERTIES
PREFIX "${PYTHON_MODULE_PREFIX}"
SUFFIX "${PYTHON_MODULE_EXTENSION}"
)
endif()
endif ()
######################################################################
@ -152,18 +165,18 @@ target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar)
if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
if (ENABLE_PYTHON)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY})
else()
else ()
target_link_libraries(_c_dataengine PRIVATE ${SECUREC_LIBRARY})
endif()
else()
endif ()
else ()
set(ICU_LIB mindspore::icuuc mindspore::icudata mindspore::icui18n)
if (ENABLE_PYTHON)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module -ldl ${SECUREC_LIBRARY})
else()
else ()
target_link_libraries(_c_dataengine PRIVATE -ldl ${SECUREC_LIBRARY})
endif()
endif ()
target_link_libraries(_c_dataengine PUBLIC mindspore::sentencepiece)
endif()
endif ()
target_link_libraries(_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::turbojpeg mindspore::opencv_core mindspore::opencv_imgcodecs
mindspore::opencv_imgproc mindspore::tinyxml2 mindspore::sentencepiece_train ${ICU_LIB})
if (ENABLE_GPUQUE)
@ -181,26 +194,26 @@ add_dependencies(_c_dataengine _c_mindrecord)
if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
set(MINDRECORD_LINK_OBJECT ${CMAKE_BINARY_DIR}/mindspore/ccsrc/minddata/mindrecord/CMakeFiles/_c_mindrecord.dir/objects.a)
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord ${MINDRECORD_LINK_OBJECT} mindspore::sqlite)
else()
else ()
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord)
if (ENABLE_CPU AND (ENABLE_D OR ENABLE_GPU))
target_link_libraries(_c_dataengine PRIVATE mindspore::pslite ${zeromq_DIRPATH}/zmq_install/lib/libzmq.a)
if (${ENABLE_IBVERBS} STREQUAL "ON")
target_link_libraries(_c_dataengine PRIVATE ibverbs rdmacm)
endif()
endif()
endif()
endif ()
endif ()
endif ()
if (USE_GLOG)
target_link_libraries(_c_dataengine PRIVATE mindspore::glog)
else()
else ()
if (CMAKE_SYSTEM_NAME MATCHES "Linux")
target_link_options(_c_dataengine PRIVATE -Wl,-init,mindspore_log_init)
elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set_target_properties(_c_dataengine PROPERTIES MACOSX_RPATH ON)
endif ()
endif()
endif ()
if (MS_BUILD_GRPC)
target_link_libraries(_c_dataengine PRIVATE mindspore::grpc++)
endif()
endif ()

96
mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt
View File

@ -1,37 +1,71 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
if (ENABLE_PYTHON)
add_library(APItoPython OBJECT
python/pybind_register.cc
python/pybind_conversion.cc
python/bindings/dataset/include/datasets_bindings.cc
python/bindings/dataset/include/iterator_bindings.cc
python/bindings/dataset/include/execute_binding.cc
python/bindings/dataset/include/schema_bindings.cc
python/bindings/dataset/engine/cache/bindings.cc
python/bindings/dataset/core/bindings.cc
python/bindings/dataset/callback/bindings.cc
python/bindings/dataset/kernels/data/bindings.cc
python/bindings/dataset/kernels/bindings.cc
python/bindings/dataset/engine/datasetops/bindings.cc
python/bindings/dataset/engine/datasetops/source/bindings.cc
python/bindings/dataset/engine/gnn/bindings.cc
python/bindings/dataset/kernels/image/bindings.cc
python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc
python/bindings/dataset/text/bindings.cc
python/bindings/dataset/text/kernels/bindings.cc
python/bindings/mindrecord/include/bindings.cc
)
if (APPLE)
add_library(APItoPython OBJECT
python/pybind_register.cc
python/pybind_conversion.cc
python/bindings/dataset/include/datasets_bindings.cc
python/bindings/dataset/include/iterator_bindings.cc
python/bindings/dataset/include/execute_binding.cc
python/bindings/dataset/include/schema_bindings.cc
python/bindings/dataset/engine/cache/bindings.cc
python/bindings/dataset/core/bindings.cc
python/bindings/dataset/callback/bindings.cc
python/bindings/dataset/kernels/data/bindings.cc
python/bindings/dataset/kernels/bindings.cc
python/bindings/dataset/engine/datasetops/bindings.cc
python/bindings/dataset/engine/datasetops/source/bindings.cc
python/bindings/dataset/engine/gnn/bindings.cc
python/bindings/dataset/kernels/image/bindings.cc
python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc
python/bindings/mindrecord/include/bindings.cc
)
else()
add_library(APItoPython OBJECT
python/pybind_register.cc
python/pybind_conversion.cc
python/bindings/dataset/include/datasets_bindings.cc
python/bindings/dataset/include/iterator_bindings.cc
python/bindings/dataset/include/execute_binding.cc
python/bindings/dataset/include/schema_bindings.cc
python/bindings/dataset/engine/cache/bindings.cc
python/bindings/dataset/core/bindings.cc
python/bindings/dataset/callback/bindings.cc
python/bindings/dataset/kernels/data/bindings.cc
python/bindings/dataset/kernels/bindings.cc
python/bindings/dataset/engine/datasetops/bindings.cc
python/bindings/dataset/engine/datasetops/source/bindings.cc
python/bindings/dataset/engine/gnn/bindings.cc
python/bindings/dataset/kernels/image/bindings.cc
python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc
python/bindings/dataset/text/bindings.cc
python/bindings/dataset/text/kernels/bindings.cc
python/bindings/mindrecord/include/bindings.cc
)
endif()
target_include_directories(APItoPython PRIVATE ${pybind11_INCLUDE_DIRS})
endif ()
add_library(cpp-API OBJECT
config.cc
datasets.cc
execute.cc
iterator.cc
transforms.cc
samplers.cc
text.cc
vision.cc
)
if (APPLE)
add_library(cpp-API OBJECT
config.cc
datasets.cc
execute.cc
iterator.cc
transforms.cc
samplers.cc
vision.cc
)
else()
add_library(cpp-API OBJECT
config.cc
datasets.cc
execute.cc
iterator.cc
transforms.cc
samplers.cc
text.cc
vision.cc
)
endif()

19
mindspore/ccsrc/minddata/dataset/core/tensor.cc
View File

@ -695,13 +695,17 @@ Status Tensor::GetDataAsNumpy(py::array *data) {
}
Status Tensor::GetDataAsNumpyStrings(py::array *data) {
auto itr = begin<std::string_view>();
uint64_t max = 0;
uint64_t max_value = 0;
for (; itr != end<std::string_view>(); itr++) {
max = std::max((*itr).length(), max);
#ifdef __APPLE__
max_value = fmax((*itr).length(), max_value);
#else
max_value = std::max((*itr).length(), max_value);
#endif
}
// if all strings are empty, numpy stores a byte for each string |S1
max = (max == 0 ? 1 : max);
uint64_t total_size = shape_.NumOfElements() * max;
max_value = (max_value == 0 ? 1 : max_value);
uint64_t total_size = shape_.NumOfElements() * max_value;
char *tmp_data = reinterpret_cast<char *>(data_allocator_->allocate(total_size));
if (tmp_data == nullptr) RETURN_STATUS_UNEXPECTED("Cannot create temp array.");
int ret_code = memset_s(tmp_data, total_size, 0, total_size);
@ -711,13 +715,14 @@ Status Tensor::GetDataAsNumpyStrings(py::array *data) {
uint64_t i = 0;
for (; itr != end<std::string_view>(); itr++, i++) {
if (!(*itr).empty()) {
ret_code = memcpy_s(tmp_data + i * max, total_size, (*itr).data(), (*itr).length());
ret_code = memcpy_s(tmp_data + i * max_value, total_size, (*itr).data(), (*itr).length());
CHECK_FAIL_RETURN_UNEXPECTED(ret_code == 0, "Failed to copy string data.");
}
}
auto strides = shape_.Strides();
std::transform(strides.begin(), strides.end(), strides.begin(), [&max](const auto &s) { return s * max; });
*data = py::array(py::dtype("S" + std::to_string(max)), shape_.AsVector(), strides, tmp_data);
std::transform(strides.begin(), strides.end(), strides.begin(),
[&max_value](const auto &s) { return s * max_value; });
*data = py::array(py::dtype("S" + std::to_string(max_value)), shape_.AsVector(), strides, tmp_data);
data_allocator_->deallocate(reinterpret_cast<uchar *>(tmp_data));
return Status::OK();
}

10
mindspore/ccsrc/minddata/dataset/engine/cache/cache_hw.cc vendored
View File

@ -154,6 +154,9 @@ Status CacheServerHW::GetNumaNodeInfo() {
}
Status CacheServerHW::SetAffinity(const Task &tk, numa_id_t numa_node) {
#ifdef __APPLE__
return Status::OK();
#else
auto r = numa_cpuset_.find(numa_node);
if (r != numa_cpuset_.end()) {
auto err = pthread_setaffinity_np(tk.GetNativeHandle(), sizeof(r->second), &r->second);
@ -165,6 +168,7 @@ Status CacheServerHW::SetAffinity(const Task &tk, numa_id_t numa_node) {
RETURN_STATUS_UNEXPECTED("Numa node " + std::to_string(numa_node) + " not found");
}
return Status::OK();
#endif
}
std::vector<cpu_id_t> CacheServerHW::GetCpuList(numa_id_t numa_id) {
@ -182,6 +186,9 @@ std::vector<cpu_id_t> CacheServerHW::GetCpuList(numa_id_t numa_id) {
}
numa_id_t CacheServerHW::GetMyNode() const {
#ifdef __APPLE__
numa_id_t node_id = -1;
#else
numa_id_t node_id = 0;
auto cpu = sched_getcpu();
#ifdef NUMA_ENABLED
@ -197,7 +204,8 @@ numa_id_t CacheServerHW::GetMyNode() const {
}
}
MS_LOG(DEBUG) << "cpu id " << cpu << " found : " << std::boolalpha << found;
#endif
#endif // end NUMA_ENABLED
#endif // end __APPLE__
return node_id;
}

19
mindspore/ccsrc/minddata/dataset/engine/cache/cache_hw.h vendored
View File

@ -18,7 +18,7 @@
#ifdef NUMA_ENABLED
#include <numa.h>
#endif
#endif // NUMA_ENABLED
#include <sched.h>
#include <stdlib.h>
#include <map>
@ -32,6 +32,23 @@
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/util/task.h"
#ifdef __APPLE__
#define SYSCTL_CORE_COUNT "machdep.cpu.core_count"
#include <sys/sysctl.h>
#include <mach/thread_policy.h>
typedef struct cpu_set {
uint32_t count;
} cpu_set_t;
static inline void CPU_ZERO(cpu_set_t *cs) { cs->count = 0; }
static inline void CPU_SET(int num, cpu_set_t *cs) { cs->count |= (1 << num); }
static inline int CPU_ISSET(int num, cpu_set_t *cs) { return (cs->count & (1 << num)); }
#endif // __APPLE__
namespace mindspore {
namespace dataset {
class CacheServerHW {

6
mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc vendored
View File

@ -235,7 +235,7 @@ Status CreateCacheRequest::PostReply() {
// Next is a set of cpu id that we should re-adjust ourselves for better affinity.
auto sz = p->cpu_id()->size();
cc_->cpu_list_.reserve(sz);
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) && !defined(__APPLE__)
std::string c_list;
cpu_set_t cpu_set;
CPU_ZERO(&cpu_set);
@ -243,13 +243,13 @@ Status CreateCacheRequest::PostReply() {
for (auto i = 0; i < sz; ++i) {
auto cpu_id = p->cpu_id()->Get(i);
cc_->cpu_list_.push_back(cpu_id);
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) && !defined(__APPLE__)
c_list += std::to_string(cpu_id) + " ";
CPU_SET(cpu_id, &cpu_set);
#endif
}
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) && !defined(__APPLE__)
if (sz > 0) {
auto err = sched_setaffinity(getpid(), sizeof(cpu_set), &cpu_set);
if (err == -1) {

4
mindspore/ccsrc/minddata/dataset/engine/cache/storage_container.cc vendored
View File

@ -71,6 +71,8 @@ Status StorageContainer::Read(WritableSlice *dest, off64_t offset) const noexcep
RETURN_STATUS_UNEXPECTED(strerror(errno));
}
auto r_sz = read(fd_, dest->GetMutablePointer(), sz);
#elif defined(__APPLE__)
auto r_sz = pread(fd_, dest->GetMutablePointer(), sz, offset);
#else
auto r_sz = pread64(fd_, dest->GetMutablePointer(), sz, offset);
#endif
@ -94,6 +96,8 @@ Status StorageContainer::Write(const ReadableSlice &dest, off64_t offset) const
RETURN_STATUS_UNEXPECTED(strerror(errno));
}
auto r_sz = write(fd_, dest.GetPointer(), sz);
#elif defined(__APPLE__)
auto r_sz = pwrite(fd_, dest.GetPointer(), sz, offset);
#else
auto r_sz = pwrite64(fd_, dest.GetPointer(), sz, offset);
#endif

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc
View File

@ -745,7 +745,13 @@ Status TFReaderOp::LoadBytesList(const ColDescriptor &current_col, const dataeng
}
uint64_t max_size = 0;
for (uint32_t i = 0; i < bytes_list.value_size(); ++i) max_size = std::max(max_size, bytes_list.value(i).size());
for (uint32_t i = 0; i < bytes_list.value_size(); ++i) {
#ifdef __APPLE__
max_size = fmax(max_size, bytes_list.value(i).size());
#else
max_size = std::max(max_size, bytes_list.value(i).size());
#endif
}
int64_t pad_size = max_size;

2
mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc
View File

@ -136,7 +136,7 @@ void ExecutionTree::PrintNode(std::ostream &out, const std::shared_ptr<DatasetOp
Status ExecutionTree::Launch() {
// opencv limit too many threads
#ifndef ENABLE_ANDROID
#if !defined(_WIN32) && !defined(_WIN64)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__)
int32_t thread_num = get_nprocs();
if (thread_num == 0) {
std::string err_msg = "Invalid thread number.";

2
mindspore/ccsrc/minddata/dataset/engine/execution_tree.h
View File

@ -22,7 +22,7 @@
#include <string>
#include <vector>
#ifndef ENABLE_ANDROID
#if !defined(_WIN32) && !defined(_WIN64)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__)
#include <sys/sysinfo.h>
#include <opencv2/imgproc/imgproc.hpp>
#endif

2
mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc
View File

@ -193,7 +193,7 @@ void JpegSetSource(j_decompress_ptr cinfo, const void *data, int64_t datasize) {
(*cinfo->mem->alloc_small)(reinterpret_cast<j_common_ptr>(cinfo), JPOOL_PERMANENT, sizeof(struct jpeg_source_mgr)));
cinfo->src->init_source = JpegInitSource;
cinfo->src->fill_input_buffer = JpegFillInputBuffer;
#if defined(_WIN32) || defined(_WIN64) || defined(ENABLE_ARM32)
#if defined(_WIN32) || defined(_WIN64) || defined(ENABLE_ARM32) || defined(__APPLE__)
cinfo->src->skip_input_data = reinterpret_cast<void (*)(j_decompress_ptr, long)>(JpegSkipInputData);
#else
cinfo->src->skip_input_data = JpegSkipInputData;

3
mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.h
View File

@ -25,6 +25,9 @@
#if defined(_WIN32) || defined(_WIN64)
#undef HAVE_STDDEF_H
#undef HAVE_STDLIB_H
#elif __APPLE__
#include <sys/param.h>
#include <sys/mount.h>
#endif
#include "./jpeglib.h"
#include "./jerror.h"

3
mindspore/ccsrc/minddata/dataset/text/kernels/jieba_tokenizer_op.h
View File

@ -18,7 +18,7 @@
#include <string>
#include <memory>
#if !defined(__APPLE__)
#include "cppjieba/Jieba.hpp"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/kernels/tensor_op.h"
@ -67,3 +67,4 @@ class JiebaTokenizerOp : public TensorOp {
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_TEXT_JIEBA_OP_H_
#endif

4
mindspore/ccsrc/minddata/dataset/util/services.cc
View File

@ -27,6 +27,10 @@
#include "minddata/dataset/util/random.h"
#include "minddata/dataset/util/task_manager.h"
#if defined(__APPLE__)
#define LOGIN_NAME_MAX 256
#endif
namespace mindspore {
namespace dataset {
std::unique_ptr<Services> Services::instance_ = nullptr;

2
mindspore/ccsrc/minddata/dataset/util/sig_handler.cc
View File

@ -16,7 +16,7 @@
#include "minddata/dataset/util/sig_handler.h"
#include <signal.h>
#include <sys/types.h>
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) && !defined(__APPLE__)
#include <ucontext.h>
#endif
#include <unistd.h>

5
mindspore/ccsrc/minddata/dataset/util/slice.h
View File

@ -22,6 +22,11 @@
#include "./securec.h"
#include "minddata/dataset/util/allocator.h"
#include "minddata/dataset/util/status.h"
#ifdef __APPLE__
#define off64_t off_t
#endif
namespace mindspore {
namespace dataset {
/// \brief A ReadableSlice wraps a const pointer in memory and its size.

6
mindspore/ccsrc/minddata/mindrecord/CMakeLists.txt
View File

@ -10,7 +10,11 @@ aux_source_directory(common DIR_LIB_SRCS)
# set(CMAKE_CXX_COMPILER "g++")
# set(CMAKE_CXX_FLAGS "-Wall -fvisibility=hidden")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath,$ORIGIN:$ORIGIN/lib")
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-command-line-argument")
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath,$ORIGIN:$ORIGIN/lib")
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-type")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable")

2
mindspore/ccsrc/minddata/mindrecord/common/shard_utils.cc
View File

@ -164,7 +164,7 @@ bool IsLegalFile(const std::string &path) {
}
std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type) {
#if defined(_WIN32) || defined(_WIN64)
#if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__)
return {SUCCESS, 100};
#else
uint64_t ll_count = 0;

2
mindspore/ccsrc/minddata/mindrecord/include/common/shard_utils.h
View File

@ -21,7 +21,7 @@
#include <limits.h>
#include <stdlib.h>
#include <sys/stat.h>
#if !defined(_WIN32) && !defined(_WIN64)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__)
#include <sys/statfs.h>
#include <sys/wait.h>
#endif

2
mindspore/ccsrc/minddata/mindrecord/include/shard_reader.h
View File

@ -19,7 +19,7 @@
#include <dirent.h>
#include <signal.h>
#if !defined(_WIN32) && !defined(_WIN64)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__)
#include <sys/prctl.h>
#endif
#include <sys/stat.h>

2
mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc
View File

@ -1173,7 +1173,7 @@ TASK_RETURN_CONTENT ShardReader::ConsumerOneTask(int task_id, uint32_t consumer_
MSRStatus ShardReader::ConsumerByRow(int consumer_id) {
// Set thread name
#if !defined(_WIN32) && !defined(_WIN64)
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__)
auto thread_id = kThreadName + std::to_string(consumer_id);
prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0);
#endif

10
mindspore/ccsrc/pipeline/pynative/pynative_execute.cc
View File

@ -1340,6 +1340,15 @@ py::object PynativeExecutor::RunOpInMs(const OpExecInfoPtr &op_exec_info, Pynati
ConstructInputTensor(op_exec_info, &tensors_mask, &input_tensors);
// get graph info for checking it whether existing in the cache
std::string graph_info = GetSingleOpGraphInfo(op_exec_info, input_tensors);
#ifdef __APPLE__
session::OpRunInfo op_run_info = {op_exec_info->op_name,
op_exec_info->py_primitive,
op_exec_info->abstract,
op_exec_info->is_dynamic_shape,
op_exec_info->is_mixed_precision_cast,
op_exec_info->next_op_name,
static_cast<int>(op_exec_info->next_input_index)};
#else
session::OpRunInfo op_run_info = {op_exec_info->op_name,
op_exec_info->py_primitive,
op_exec_info->abstract,
@ -1347,6 +1356,7 @@ py::object PynativeExecutor::RunOpInMs(const OpExecInfoPtr &op_exec_info, Pynati
op_exec_info->is_mixed_precision_cast,
op_exec_info->next_op_name,
op_exec_info->next_input_index};
#endif
VectorRef outputs;
session->RunOp(&op_run_info, graph_info, &input_tensors, &outputs, tensors_mask);
if (op_exec_info->is_dynamic_shape) {

8
mindspore/ccsrc/runtime/device/kernel_runtime.cc
View File

@ -847,7 +847,11 @@ void KernelRuntime::GenAddrCleanLaunchArgs(const CNodePtr &cnode, AddressPtrList
auto pre_node = (cnode->inputs()[1])->cast<CNodePtr>();
// set clean output address
if (AnfAlgo::HasNodeAttr(kAttrAtomicOutputIndexs, pre_node)) {
#if defined(__APPLE__)
auto clean_output_indexes = AnfAlgo::GetNodeAttr<std::vector<int>>(pre_node, kAttrAtomicOutputIndexs);
#else
auto clean_output_indexes = AnfAlgo::GetNodeAttr<std::vector<size_t>>(pre_node, kAttrAtomicOutputIndexs);
#endif
for (auto index : clean_output_indexes) {
auto device_address = AnfAlgo::GetOutputAddr(pre_node, index);
kernel::AddressPtr input = std::make_shared<kernel::Address>();
@ -861,7 +865,11 @@ void KernelRuntime::GenAddrCleanLaunchArgs(const CNodePtr &cnode, AddressPtrList
}
// set clean workspace address
if (AnfAlgo::HasNodeAttr(kAttrAtomicWorkspaceIndexs, pre_node)) {
#if defined(__APPLE__)
auto clean_workspaces_indexes = AnfAlgo::GetNodeAttr<std::vector<int>>(pre_node, kAttrAtomicWorkspaceIndexs);
#else
auto clean_workspaces_indexes = AnfAlgo::GetNodeAttr<std::vector<size_t>>(pre_node, kAttrAtomicWorkspaceIndexs);
#endif
for (const auto &index : clean_workspaces_indexes) {
auto device_address = AnfAlgo::GetWorkspaceAddr(pre_node, index);
kernel::AddressPtr workspace = std::make_shared<kernel::Address>();

5
mindspore/ccsrc/utils/system/sha256.h
View File

@ -21,6 +21,7 @@
#include <iomanip>
#include <fstream>
#include <memory>
#include <vector>
namespace mindspore {
namespace system {
@ -100,9 +101,9 @@ bool ProcessInner(const std::string &message, const int &bias, uint32_t *digest,
w[i] = sigma3(w[i - 2]) + w[i - 7] + sigma2(w[i - 15]) + w[i - 16];
}
std::shared_ptr<uint32_t[]> hash(new uint32_t[digest_size]());
std::vector<uint32_t> hash(digest_size);
size_t mem_size = digest_size * sizeof(uint32_t);
auto ret = memcpy_s(hash.get(), mem_size, digest, mem_size);
auto ret = memcpy_s(hash.data(), mem_size, digest, mem_size);
if (ret != EOK) {
return false;
}

4
mindspore/ccsrc/vm/segment_runner.cc
View File

@ -131,7 +131,7 @@ std::tuple<FuncGraphPtr, AnfNodePtrList, AnfNodePtrList> TransformSegmentToAnfGr
if (!IsValueNode<Primitive>(inps[0]) &&
!(IsValueNode<FuncGraph>(inps[0]) &&
inps[0]->cast<ValueNodePtr>()->value()->cast<FuncGraphPtr>()->has_attr(FUNC_GRAPH_ATTR_GRAPH_KERNEL))) {
MS_LOG(EXCEPTION) << "Input[0] Must be a Primitive valuenode";
MS_LOG(EXCEPTION) << "Input[0] Must be a Primitive ValueNode";
}
auto fn = inps[0];
std::vector<AnfNodePtr> args{fn};
@ -141,7 +141,7 @@ std::tuple<FuncGraphPtr, AnfNodePtrList, AnfNodePtrList> TransformSegmentToAnfGr
} else if (IsPrimitive(fn, prim::kPrimControlDepend) && inps.size() == 3) {
for (size_t i = 1; i < inps.size(); ++i) {
if (inps[i]->isa<CNode>() && std::find(lst.begin(), lst.end(), inps[i]) == lst.end()) {
args.emplace_back(NewValueNode(MakeValue(i)));
args.emplace_back(NewValueNode(MakeValue(static_cast<int>(i))));
} else {
args.emplace_back(RefSubGraphNode(fg, inps[i], &inputs, &eqv));
}

15
mindspore/core/CMakeLists.txt
View File

@ -2,12 +2,9 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR})
include_directories(${CMAKE_BINARY_DIR})
include_directories(${CMAKE_SOURCE_DIR}/mindspore/core)
add_subdirectory(gvar)
message("************build core***************")
message("************ build core ***************")
if (CMAKE_SYSTEM_NAME MATCHES "Windows")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF")
add_compile_definitions(BUILDING_DLL)
endif()
file(GLOB_RECURSE CORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"abstract/*.cc"
@ -17,9 +14,17 @@ file(GLOB_RECURSE CORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"utils/*.cc"
"load_mindir/*.cc"
)
if (CMAKE_SYSTEM_NAME MATCHES "Windows")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF")
add_compile_definitions(BUILDING_DLL)
elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wuser-defined-warnings -Winconsistent-missing-override")
endif ()
set_property(SOURCE ${CORE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_CORE)
add_library(mindspore_core STATIC ${CORE_SRC_LIST})
target_link_libraries(mindspore_core PRIVATE mindspore_gvar)
if (USE_GLOG)
target_link_libraries(mindspore_core PRIVATE mindspore::glog)
endif ()

6
mindspore/core/gvar/CMakeLists.txt
View File

@ -1,12 +1,16 @@
file(GLOB_RECURSE MS_GVAR_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cc)
set_property(SOURCE ${MS_GVAR_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_COMMON)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_FORTIFY_SOURCE=2 -O2")
if (CMAKE_SYSTEM_NAME MATCHES "Windows")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF")
add_compile_definitions(BUILDING_DLL)
endif()
elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wuser-defined-warnings -Winconsistent-missing-override")
endif ()
add_library(mindspore_gvar SHARED ${MS_GVAR_SRC_LIST})
if (APPLE)
set_target_properties(mindspore_gvar PROPERTIES MACOSX_RPATH ON)
endif ()

4
third_party/patch/icu4c/icu4c.patch01 vendored
View File

@ -5,8 +5,8 @@
THE_OS="Linux"
THE_COMP="the clang or else GNU C++"
- RELEASE_CFLAGS='-O3'
+ RELEASE_CFLAGS='-fstack-protector -Wl,-z,now -D_FORTIFY_SOURCE=2 -O3'
+ RELEASE_CFLAGS='-fstack-protector -D_FORTIFY_SOURCE=2 -O3'
- RELEASE_CXXFLAGS='-O3'
+ RELEASE_CXXFLAGS='-fstack-protector -Wl,-z,now -D_FORTIFY_SOURCE=2 -O3'
+ RELEASE_CXXFLAGS='-fstack-protector -D_FORTIFY_SOURCE=2 -O3'
DEBUG_CFLAGS='-g'
DEBUG_CXXFLAGS='-g'

6
third_party/securec/CMakeLists.txt vendored
View File

@ -1,9 +1,9 @@
SET(CMAKE_BUILD_TYPE "Debug")
if (CMAKE_SYSTEM_NAME MATCHES "Windows")
SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -fstack-protector-all")
else()
SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -fstack-protector-all -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='")
endif()
else ()
SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -Wno-nullability-completeness -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -fstack-protector-all -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='")
endif ()
SET(CMAKE_C_FLAGS_RELEASE "$ENV{CFLAGS} -fPIC -O3 -Wall -Wno-deprecated-declarations -fstack-protector-all")
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)