Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

support building on windows

This commit is contained in:
chenjianping 2020-04-15 09:36:48 +00:00
parent d90e121547
commit 1286767d0e
48 changed files with 619 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
CMakeLists.txt
View File

@ -12,6 +12,7 @@ else()
endif()
set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -Wl,--allow-shlib-undefined -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)=' -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/include -std=c++17 -Werror -Wall -Wno-deprecated-declarations -fPIC")
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
@ -29,6 +30,18 @@ find_package(Python3 3.7 COMPONENTS Interpreter Development)
if(Python3_FOUND)
set(PYTHON_INCLUDE_DIRS "${Python3_INCLUDE_DIRS}")
set(PYTHON_LIBRARIES "${Python3_LIBRARIES}")
if (WIN32)
if (Python3_DIR)
message("Python3_DIR set already: " ${Python3_DIR})
else()
string(LENGTH ${PYTHON_LIBRARIES} PYTHON_LIBRARIES_LEN)
string(LENGTH "libpythonxx.a" Python3_NAME_LEN)
math(EXPR Python3_DIR_LEN ${PYTHON_LIBRARIES_LEN}-${Python3_NAME_LEN})
string(SUBSTRING ${Python3_LIBRARIES} 0 ${Python3_DIR_LEN} Python3_DIR)
message("Python3_DIR: " ${Python3_DIR})
endif()
link_directories(${Python3_DIR})
endif()
else()
find_python_package(py_inc py_lib)
set(PYTHON_INCLUDE_DIRS "${py_inc}")

36
build.bat Normal file
View File

@ -0,0 +1,36 @@
@echo off
@title mindspore_build
SET BASEPATH=%CD%
IF NOT EXIST %BASEPATH%/build (
md "build"
)
cd %BASEPATH%/build
SET BUILD_PATH=%CD%
IF NOT EXIST %BUILD_PATH%/mindspore (
md "mindspore"
)
cd %CD%/mindspore
cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_CPU=ON -DENABLE_MINDDATA=ON -DUSE_GLOG=ON -G "CodeBlocks - MinGW Makefiles" ../..
IF NOT %errorlevel% == 0 (
goto run_fail
)
cmake --build . --target all -- -j6
IF NOT %errorlevel% == 0 (
goto run_fail
)
cd %BASEPATH%
goto run_eof
:run_fail
cd %BASEPATH%
echo "build fail."
:run_eof

3
cmake/external_libs/flatbuffers.cmake
View File

@ -1,5 +1,8 @@
set(flatbuffers_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2")
set(flatbuffers_CFLAGS "-D_FORTIFY_SOURCE=2 -O2")
if (WIN32)
set(flatbuffers_USE_STATIC_LIBS ON)
endif()
mindspore_add_pkg(flatbuffers
VER 1.11.0
LIBS flatbuffers

13
cmake/external_libs/mkl_dnn.cmake
View File

@ -1,11 +1,22 @@
set(onednn_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2")
set(onednn_CFLAGS "-D_FORTIFY_SOURCE=2 -O2")
mindspore_add_pkg(onednn
if (CMAKE_SYSTEM_NAME MATCHES "Windows")
mindspore_add_pkg(onednn
VER 1.1.1
LIBS dnnl mkldnn
HEAD_ONLY ./
RELEASE on
URL https://github.com/oneapi-src/oneDNN/releases/download/v1.1.1/dnnl_win_1.1.1_cpu_vcomp.zip
MD5 ecaab9ed549643067699c80e5cea1c23)
else()
mindspore_add_pkg(onednn
VER 1.1.2
LIBS dnnl mkldnn
URL https://github.com/oneapi-src/oneDNN/archive/v1.1.2.tar.gz
MD5 ab40d52230f3ad1d7a6f06ce0f6bc17a
CMAKE_OPTION -DDNNL_ARCH_OPT_FLAGS='' -DDNNL_CPU_RUNTIME='SEQ' -DDNNL_BUILD_EXAMPLES=OFF -DDNNL_BUILD_TESTS=OFF)
endif()
include_directories(${onednn_INC})
add_library(mindspore::dnnl ALIAS onednn::dnnl)
add_library(mindspore::mkldnn ALIAS onednn::mkldnn)

89
cmake/external_libs/opencv.cmake
View File

@ -2,35 +2,72 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(opencv_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_LDFLAGS "-Wl")
elseif (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
set(opencv_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
else()
set(opencv_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
endif()
mindspore_add_pkg(opencv
VER 4.2.0
LIBS opencv_core opencv_imgcodecs opencv_imgproc
URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz
MD5 e8cb208ce2723481408b604b480183b6
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF
-DBUILD_ZLIB=ON
-DBUILD_JPEG=ON
-DBUILD_PNG=ON
-DBUILD_OPENEXR=ON
-DBUILD_TESTS=OFF
-DBUILD_PERF_TESTS=OFF
-DBUILD_opencv_apps=OFF
-DCMAKE_SKIP_RPATH=TRUE
-DBUILD_opencv_python3=OFF
-DWITH_FFMPEG=OFF
-DWITH_TIFF=ON
-DBUILD_TIFF=OFF
-DWITH_JASPER=OFF
-DBUILD_JASPER=OFF
-DTIFF_INCLUDE_DIR=${tiff_INC}
-DTIFF_LIBRARY=${tiff_LIB})
include_directories(${opencv_INC}/opencv4)
add_library(mindspore::opencv_core ALIAS opencv::opencv_core)
add_library(mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs)
add_library(mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc)
if (WIN32)
mindspore_add_pkg(opencv
VER 4.2.0
LIBS libopencv_core420.dll.a libopencv_imgcodecs420.dll.a libopencv_imgproc420.dll.a
LIB_PATH x64/mingw/lib
URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz
MD5 e8cb208ce2723481408b604b480183b6
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF
-DBUILD_ZLIB=ON
-DBUILD_JPEG=ON
-DBUILD_PNG=ON
-DBUILD_OPENEXR=ON
-DBUILD_TESTS=OFF
-DBUILD_PERF_TESTS=OFF
-DBUILD_opencv_apps=OFF
-DCMAKE_SKIP_RPATH=TRUE
-DBUILD_opencv_python3=OFF
-DWITH_FFMPEG=OFF
-DWITH_TIFF=ON
-DBUILD_TIFF=OFF
-DWITH_JASPER=OFF
-DBUILD_JASPER=OFF
-DTIFF_INCLUDE_DIR=${tiff_INC}
-DTIFF_LIBRARY=${tiff_LIB})
else()
mindspore_add_pkg(opencv
VER 4.2.0
LIBS opencv_core opencv_imgcodecs opencv_imgproc
URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz
MD5 e8cb208ce2723481408b604b480183b6
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF
-DBUILD_ZLIB=ON
-DBUILD_JPEG=ON
-DBUILD_PNG=ON
-DBUILD_OPENEXR=ON
-DBUILD_TESTS=OFF
-DBUILD_PERF_TESTS=OFF
-DBUILD_opencv_apps=OFF
-DCMAKE_SKIP_RPATH=TRUE
-DBUILD_opencv_python3=OFF
-DWITH_FFMPEG=OFF
-DWITH_TIFF=ON
-DBUILD_TIFF=OFF
-DWITH_JASPER=OFF
-DBUILD_JASPER=OFF
-DTIFF_INCLUDE_DIR=${tiff_INC}
-DTIFF_LIBRARY=${tiff_LIB})
endif()
if (WIN32)
include_directories(${opencv_INC})
add_library(mindspore::opencv_core ALIAS opencv::libopencv_core420.dll.a)
add_library(mindspore::opencv_imgcodecs ALIAS opencv::libopencv_imgcodecs420.dll.a)
add_library(mindspore::opencv_imgproc ALIAS opencv::libopencv_imgproc420.dll.a)
else()
include_directories(${opencv_INC}/opencv4)
add_library(mindspore::opencv_core ALIAS opencv::opencv_core)
add_library(mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs)
add_library(mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc)
endif()

44
cmake/external_libs/protobuf.cmake
View File

@ -77,22 +77,36 @@ function(ms_protobuf_generate_py c_var h_var py_var)
list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc")
list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h")
list(APPEND ${py_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py")
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}"
COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
COMMAND cp "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/"
DEPENDS protobuf::protoc ${abs_file}
COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
if (WIN32)
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}"
COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND perl -pi.bak -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/"
DEPENDS protobuf::protoc ${abs_file}
COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
else()
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}"
COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
COMMAND cp "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/"
DEPENDS protobuf::protoc ${abs_file}
COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
endif()
endforeach()
set_source_files_properties(${${c_var}} ${${h_var}} ${${py_var}} PROPERTIES GENERATED TRUE)
set(${c_var} ${${c_var}} PARENT_SCOPE)
set(${h_var} ${${h_var}} PARENT_SCOPE)

39
cmake/external_libs/sqlite.cmake
View File

@ -1,19 +1,36 @@
if (WIN32)
mindspore_add_pkg(sqlite-head
VER 3.31.1
HEAD_ONLY ./
URL https://sqlite.org/2020/sqlite-amalgamation-3310100.zip
MD5 2b7bfcdd97dc281903a9aee966213fe4)
include_directories(${sqlite-head_INC})
mindspore_add_pkg(sqlite
VER 3.31.1
LIBS sqlite3
LIB_PATH ./
HEAD_ONLY ./
RELEASE ON
URL https://sqlite.org/2020/sqlite-dll-win64-x64-3310100.zip
MD5 662c9d2b05467d590ba5c0443e7fd6bd)
set(sqlite_USE_STATIC_LIBS ON)
set(sqlite_CXXFLAGS)
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(sqlite_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
else()
set(sqlite_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
endif()
set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
mindspore_add_pkg(sqlite
else ()
set(sqlite_USE_STATIC_LIBS ON)
set(sqlite_CXXFLAGS)
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(sqlite_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
else()
set(sqlite_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
endif()
set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
mindspore_add_pkg(sqlite
VER 3.31.1
LIBS sqlite3
URL https://github.com/sqlite/sqlite/archive/version-3.31.1.tar.gz
MD5 5f4e7b4016c15f4fb5855615279819da
PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/sqlite/sqlite.patch001
CONFIGURE_COMMAND ./configure --enable-shared=no --disable-tcl --disable-editline --enable-json1)
include_directories(${sqlite_INC})
include_directories(${sqlite_INC})
endif ()
add_library(mindspore::sqlite ALIAS sqlite::sqlite3)

6
cmake/mind_expression.cmake
View File

@ -1,6 +1,10 @@
set(SECURE_CXX_FLAGS "")
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
set(SECURE_CXX_FLAGS "-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
if (WIN32)
set(SECURE_CXX_FLAGS "-fstack-protector-all")
else()
set(SECURE_CXX_FLAGS "-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
endif()
endif()
set(_ms_tmp_CMAKE_CXX_FLAGS_F ${CMAKE_CXX_FLAGS})
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")

6
cmake/options.cmake
View File

@ -19,7 +19,11 @@ option(ENABLE_MPI "enable mpi" OFF)
option(ENABLE_AKG "enable akg" OFF)
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
if (WIN32)
set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all")
else()
set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
endif()
endif()
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")

50
cmake/utils.cmake
View File

@ -103,7 +103,7 @@ function(__download_pkg_with_git pkg_name pkg_url pkg_git_commit pkg_md5)
endfunction()
function(__find_pkg_then_add_target pkg_name pkg_exe)
function(__find_pkg_then_add_target pkg_name pkg_exe lib_path)
unset(${pkg_name}_LIBS)
@ -129,15 +129,24 @@ function(__find_pkg_then_add_target pkg_name pkg_exe)
set(_LIB_TYPE STATIC)
endif ()
set(${_LIB_NAME}_LIB ${_LIB_NAME}_LIB-NOTFOUND)
find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/lib NO_DEFAULT_PATH)
find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/${lib_path} NO_DEFAULT_PATH)
if(NOT ${_LIB_NAME}_LIB)
return()
endif()
add_library(${pkg_name}::${_LIB_NAME} ${_LIB_TYPE} IMPORTED GLOBAL)
set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${${pkg_name}_BASE_DIR}/include"
IMPORTED_LOCATION ${${_LIB_NAME}_LIB}
)
if (WIN32 AND ${_LIB_TYPE} STREQUAL "SHARED")
set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES IMPORTED_IMPLIB_RELEASE ${${_LIB_NAME}_LIB})
else()
set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES IMPORTED_LOCATION ${${_LIB_NAME}_LIB})
endif()
if (EXISTS ${${pkg_name}_BASE_DIR}/include)
set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${${pkg_name}_BASE_DIR}/include")
endif ()
list(APPEND ${pkg_name}_LIBS ${pkg_name}::${_LIB_NAME})
message("found ${${_LIB_NAME}_LIB}")
STRING( REGEX REPLACE "(.+)/(.+)" "\\1" LIBPATH ${${_LIB_NAME}_LIB})
@ -192,12 +201,18 @@ set(MS_FIND_NO_DEFAULT_PATH ${MS_FIND_NO_DEFAULT_PATH} PARENT_SCOPE)
function(mindspore_add_pkg pkg_name )
set(options )
set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH)
set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH RELEASE LIB_PATH)
set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES)
cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} )
if (NOT PKG_CMAKE_PATH)
set(PKG_CMAKE_PATH .)
if (NOT PKG_LIB_PATH)
set(PKG_LIB_PATH lib)
endif ()
if(NOT PKG_EXE)
set(PKG_EXE 0)
endif()
set(__FIND_PKG_NAME ${pkg_name})
string(TOLOWER ${pkg_name} pkg_name)
message("pkg name:${__FIND_PKG_NAME},${pkg_name}")
@ -225,18 +240,17 @@ function(mindspore_add_pkg pkg_name )
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE)
add_library(${pkg_name} INTERFACE)
target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})
if (${PKG_RELEASE})
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS})
endif ()
return()
endif ()
if(NOT PKG_EXE)
set(PKG_EXE 0)
endif()
set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR})
set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR} PARENT_SCOPE)
if (PKG_LIBS)
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS})
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS})
if(${pkg_name}_LIBS)
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
message("Found libs: ${${pkg_name}_LIBS}")
@ -283,8 +297,10 @@ function(mindspore_add_pkg pkg_name )
file(GLOB ${pkg_name}_SOURCE_SUBDIRS ${${pkg_name}_SOURCE_DIR}/*)
file(COPY ${${pkg_name}_SOURCE_SUBDIRS} DESTINATION ${${pkg_name}_BASE_DIR})
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE)
add_library(${pkg_name} INTERFACE)
target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})
if (NOT PKG_RELEASE)
add_library(${pkg_name} INTERFACE)
target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})
endif ()
elseif (PKG_CMAKE_OPTION)
# in cmake
@ -355,7 +371,7 @@ function(mindspore_add_pkg pkg_name )
endif()
if (PKG_LIBS)
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS})
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS})
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
if(NOT ${pkg_name}_LIBS)
message(FATAL_ERROR "Can not find pkg: ${pkg_name}")

62
mindspore/ccsrc/CMakeLists.txt
View File

@ -5,6 +5,10 @@ if(ENABLE_CPU)
file(GLOB_RECURSE CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"device/cpu/*.cc"
)
if (CMAKE_SYSTEM_NAME MATCHES "Windows")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF")
add_compile_definitions(BUILDING_DLL)
endif()
endif()
if(ENABLE_GPU)
@ -150,7 +154,15 @@ file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"kernel/kash/*.cc"
"device/gpu/distribution/collective_init.cc"
)
if (ENABLE_CPU)
list(REMOVE_ITEM MINDSPORE_SRC_LIST "device/gpu/distribution/collective_init.cc")
if (WIN32)
list(REMOVE_ITEM MINDSPORE_SRC_LIST "kernel/kernel_query.cc")
endif()
endif()
if (NOT ENABLE_GPU)
list(APPEND MINDSPORE_SRC_LIST "device/gpu/distribution/collective_fake_init.cc")
endif()
file(GLOB_RECURSE MEM_REUSE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"pre_activate/mem_reuse/*.cc"
)
@ -248,6 +260,7 @@ file(GLOB_RECURSE MS_GVAR_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
add_library(mindspore_gvar SHARED ${MS_GVAR_SRC_LIST})
add_library(mindspore STATIC ${MINDSPORE_SRC_LIST})
add_dependencies(mindspore GENERATED_OUTPUT_DIR)
if(ENABLE_D)
list(APPEND MINDSPORE_PROTO_LIST ${MINDSPORE_PROTO_AICPU_LIST})
@ -315,19 +328,33 @@ if(ENABLE_D)
endif()
target_link_libraries(mindspore securec)
target_link_libraries(mindspore dl)
if (NOT WIN32)
target_link_libraries(mindspore dl)
endif()
target_link_libraries(mindspore mindspore::flatbuffers)
# link protobuf
if (ENABLE_D)
target_link_libraries(mindspore mindspore::protobuf)
endif()
if (WIN32)
target_link_libraries(mindspore ${PYTHON_LIBRARIES} mindspore_gvar)
endif()
# set c_expression building
if (WIN32)
set(PYTHON_MODULE_SOURCE ${MS_GVAR_SRC_LIST}
pipeline/init.cc
kernel/oplib/oplib.cc
${MINDSPORE_SRC_LIST} ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST}
${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST})
else()
set(PYTHON_MODULE_SOURCE
pipeline/init.cc
kernel/oplib/oplib.cc
${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST}
${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST})
${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST}
${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST})
endif()
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
pybind11_add_module(_c_expression ${PYTHON_MODULE_SOURCE})
@ -339,6 +366,8 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux")
elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set_target_properties(_c_expression PROPERTIES MACOSX_RPATH ON)
set(ORIGIN_PATH @loader_path)
elseif (CMAKE_SYSTEM_NAME MATCHES "Windows")
set(ORIGIN_PATH $ORIGIN)
else ()
MESSAGE(FATAL_ERROR "other platform: ${CMAKE_SYSTEM_NAME}")
endif ()
@ -346,11 +375,22 @@ endif ()
set(ORIGIN_PATH ${ORIGIN_PATH}/lib)
set_target_properties(_c_expression PROPERTIES INSTALL_RPATH ${ORIGIN_PATH})
target_link_libraries(_c_expression PRIVATE
mindspore::pybind11_module
mindspore
mindspore_gvar
)
if (WIN32)
target_link_libraries(_c_expression PRIVATE
mindspore::pybind11_module
securec
proto_input
mindspore::flatbuffers
mindspore::glog
)
else()
target_link_libraries(_c_expression PRIVATE
mindspore::pybind11_module
mindspore
mindspore_gvar
mindspore::glog
)
endif()
if(ENABLE_GPU)
execute_process(COMMAND bash ${CMAKE_SOURCE_DIR}/third_party/apply_patches.sh
@ -493,7 +533,3 @@ if(ENABLE_MINDDATA)
add_subdirectory(mindrecord)
add_subdirectory(dataset)
endif()
if (USE_GLOG)
target_link_libraries(_c_expression PRIVATE mindspore::glog)
endif()

18
mindspore/ccsrc/dataset/CMakeLists.txt
View File

@ -80,14 +80,19 @@ set_target_properties(_c_dataengine PROPERTIES
######################################################################
################# Link with external libraries ########################
target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module -ldl mindspore::protobuf ${SECUREC_LIBRARY})
if (WIN32)
target_link_libraries(_c_dataengine PRIVATE mindspore)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module ${PYTHON_LIBRARIES} mindspore::protobuf ${SECUREC_LIBRARY})
else()
target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module -ldl mindspore::protobuf ${SECUREC_LIBRARY})
endif()
target_link_libraries(_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::opencv_core mindspore::opencv_imgcodecs
mindspore::opencv_imgproc)
if (ENABLE_GPUQUE)
target_link_libraries(_c_dataengine PRIVATE gpu_queue
${CUDNN_PATH}/lib64/libcudnn.so
${CUDA_PATH}/lib64/libcudart.so
${CUDA_PATH}/lib64/libcudart.so
${CUDA_PATH}/lib64/stubs/libcuda.so)
endif ()
@ -96,7 +101,12 @@ if (ENABLE_TDTQUE)
endif ()
add_dependencies(_c_dataengine _c_mindrecord)
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord)
if (WIN32)
set(MINDRECORD_LINK_OBJECT ${CMAKE_BINARY_DIR}/mindspore/ccsrc/mindrecord/CMakeFiles/_c_mindrecord.dir/objects.a)
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord ${MINDRECORD_LINK_OBJECT} mindspore::sqlite)
else()
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord)
endif()
if (USE_GLOG)
target_link_libraries(_c_dataengine PRIVATE mindspore::glog)

4
mindspore/ccsrc/dataset/api/python_bindings.cc
View File

@ -19,7 +19,9 @@
#include "dataset/kernels/no_op.h"
#include "dataset/kernels/data/one_hot_op.h"
#include "dataset/kernels/image/center_crop_op.h"
#if !defined(_WIN32) && !defined(_WIN64)
#include "dataset/kernels/image/change_mode_op.h"
#endif
#include "dataset/kernels/image/cut_out_op.h"
#include "dataset/kernels/image/decode_op.h"
#include "dataset/kernels/image/distort_bounding_box_crop_op.h"
@ -279,9 +281,11 @@ void bindTensorOps2(py::module *m) {
py::arg("fillG") = RandomCropOp::kDefFillG, py::arg("fillB") = RandomCropOp::kDefFillB);
(void)py::class_<HwcToChwOp, TensorOp, std::shared_ptr<HwcToChwOp>>(*m, "ChannelSwapOp").def(py::init<>());
#if !defined(_WIN32) && !defined(_WIN64)
(void)py::class_<ChangeModeOp, TensorOp, std::shared_ptr<ChangeModeOp>>(
*m, "ChangeModeOp", "Tensor operation to change colors from BGR to RGB")
.def(py::init<>());
#endif
(void)py::class_<OneHotOp, TensorOp, std::shared_ptr<OneHotOp>>(
*m, "OneHotOp", "Tensor operation to apply one hot encoding. Takes number of classes.")

4
mindspore/ccsrc/dataset/core/tensor.h
View File

@ -22,6 +22,10 @@
#include <vector>
#include "./securec.h"
#include "utils/log_adapter.h"
#if defined(_WIN32) || defined(_WIN64)
#undef HAVE_STDDEF_H
#undef HAVE_STDLIB_H
#endif
#include "pybind11/numpy.h"
#include "pybind11/pybind11.h"
#include "pybind11/stl.h"

4
mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc
View File

@ -85,7 +85,11 @@ Status ShuffleOp::SelfReset() {
if (!reshuffle_each_epoch_) {
rng_ = std::mt19937_64(shuffle_seed_);
} else {
#if defined(_WIN32) || defined(_WIN64)
std::random_device random_device;
#else
std::random_device random_device("/dev/urandom");
#endif
std::uniform_int_distribution<int32_t> distribution(0, std::numeric_limits<int32_t>::max());
shuffle_seed_ = distribution(random_device);
rng_ = std::mt19937_64(shuffle_seed_);

68
mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt
View File

@ -1,22 +1,46 @@
add_library(kernels-image OBJECT
center_crop_op.cc
change_mode_op.cc
cut_out_op.cc
decode_op.cc
distort_bounding_box_crop_op.cc
hwc_to_chw_op.cc
image_utils.cc
normalize_op.cc
pad_op.cc
random_color_adjust_op.cc
random_crop_decode_resize_op.cc
random_crop_and_resize_op.cc
random_crop_op.cc
random_horizontal_flip_op.cc
random_resize_op.cc
random_rotation_op.cc
random_vertical_flip_op.cc
rescale_op.cc
resize_bilinear_op.cc
resize_op.cc
)
if (WIN32)
add_library(kernels-image OBJECT
center_crop_op.cc
cut_out_op.cc
decode_op.cc
distort_bounding_box_crop_op.cc
hwc_to_chw_op.cc
image_utils.cc
normalize_op.cc
pad_op.cc
random_color_adjust_op.cc
random_crop_decode_resize_op.cc
random_crop_and_resize_op.cc
random_crop_op.cc
random_horizontal_flip_op.cc
random_resize_op.cc
random_rotation_op.cc
random_vertical_flip_op.cc
rescale_op.cc
resize_bilinear_op.cc
resize_op.cc
)
else()
add_library(kernels-image OBJECT
center_crop_op.cc
change_mode_op.cc
cut_out_op.cc
decode_op.cc
distort_bounding_box_crop_op.cc
hwc_to_chw_op.cc
image_utils.cc
normalize_op.cc
pad_op.cc
random_color_adjust_op.cc
random_crop_decode_resize_op.cc
random_crop_and_resize_op.cc
random_crop_op.cc
random_horizontal_flip_op.cc
random_resize_op.cc
random_rotation_op.cc
random_vertical_flip_op.cc
rescale_op.cc
resize_bilinear_op.cc
resize_op.cc
)
endif()

4
mindspore/ccsrc/dataset/kernels/image/image_utils.cc
View File

@ -186,7 +186,11 @@ void JpegSetSource(j_decompress_ptr cinfo, const void *data, int64_t datasize) {
(*cinfo->mem->alloc_small)(reinterpret_cast<j_common_ptr>(cinfo), JPOOL_PERMANENT, sizeof(struct jpeg_source_mgr)));
cinfo->src->init_source = JpegInitSource;
cinfo->src->fill_input_buffer = JpegFillInputBuffer;
#if defined(_WIN32) || defined(_WIN64)
cinfo->src->skip_input_data = reinterpret_cast<void (*)(j_decompress_ptr, long)>(JpegSkipInputData);
#else
cinfo->src->skip_input_data = JpegSkipInputData;
#endif
cinfo->src->resync_to_restart = jpeg_resync_to_restart;
cinfo->src->term_source = JpegTermSource;
cinfo->src->bytes_in_buffer = datasize;

4
mindspore/ccsrc/dataset/kernels/image/image_utils.h
View File

@ -22,6 +22,10 @@
#include <random>
#include <string>
#include <vector>
#if defined(_WIN32) || defined(_WIN64)
#undef HAVE_STDDEF_H
#undef HAVE_STDLIB_H
#endif
#include "./jpeglib.h"
#include "./jerror.h"
#include <opencv2/imgproc/imgproc.hpp>

6
mindspore/ccsrc/dataset/util/path.cc
View File

@ -27,7 +27,7 @@
namespace mindspore {
namespace dataset {
#ifdef _WIN32
char Path::_separator = '\\';
char Path::separator_ = '\\';
#else
char Path::separator_ = '/';
#endif
@ -129,7 +129,11 @@ bool Path::IsDirectory() {
Status Path::CreateDirectory() {
if (!Exists()) {
#if defined(_WIN32) || defined(_WIN64)
int rc = mkdir(common::SafeCStr(path_));
#else
int rc = mkdir(common::SafeCStr(path_), 0700);
#endif
if (rc) {
std::ostringstream oss;
oss << "Unable to create directory " << path_ << ". Errno = " << errno;

4
mindspore/ccsrc/dataset/util/random.cc
View File

@ -32,7 +32,11 @@ namespace dataset {
uint32_t GetSeed() {
uint32_t seed = GlobalContext::config_manager()->seed();
if (seed == std::mt19937::default_seed) {
#if defined(_WIN32) || defined(_WIN64)
std::random_device random_device;
#else
std::random_device random_device("/dev/urandom");
#endif
std::uniform_int_distribution<uint32_t> distribution(0, std::numeric_limits<uint32_t>::max());
seed = distribution(random_device);
}

8
mindspore/ccsrc/dataset/util/services.cc
View File

@ -16,7 +16,9 @@
#include "dataset/util/services.h"
#include <limits.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/syscall.h>
#endif
#include <unistd.h>
#include <random>
#include "dataset/util/circular_pool.h"
@ -28,6 +30,7 @@ namespace dataset {
std::unique_ptr<Services> Services::instance_ = nullptr;
std::once_flag Services::init_instance_flag_;
#if !defined(_WIN32) && !defined(_WIN64)
std::string Services::GetUserName() {
char user[LOGIN_NAME_MAX];
(void)getlogin_r(user, sizeof(user));
@ -41,10 +44,15 @@ std::string Services::GetHostName() {
}
int Services::GetLWP() { return syscall(SYS_gettid); }
#endif
std::string Services::GetUniqueID() {
const std::string kStr = "abcdefghijklmnopqrstuvwxyz0123456789";
#if defined(_WIN32) || defined(_WIN64)
std::mt19937 gen{std::random_device{}()};
#else
std::mt19937 gen{std::random_device{"/dev/urandom"}()};
#endif
std::uniform_int_distribution<> dist(0, kStr.size() - 1);
char buffer[UNIQUEID_LEN];
for (int i = 0; i < UNIQUEID_LEN; i++) {

2
mindspore/ccsrc/dataset/util/services.h
View File

@ -62,11 +62,13 @@ class Services {
std::shared_ptr<MemoryPool> GetServiceMemPool() { return pool_; }
#if !defined(_WIN32) && !defined(_WIN64)
static std::string GetUserName();
static std::string GetHostName();
static int GetLWP();
#endif
static std::string GetUniqueID();

4
mindspore/ccsrc/dataset/util/sig_handler.cc
View File

@ -16,13 +16,16 @@
#include "dataset/util/sig_handler.h"
#include <signal.h>
#include <sys/types.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <ucontext.h>
#endif
#include <unistd.h>
#include "dataset/util/task_manager.h"
namespace mindspore {
namespace dataset {
// Register the custom signal handlers
#if !defined(_WIN32) && !defined(_WIN64)
void RegisterHandlers() {
struct sigaction new_int_action;
@ -40,5 +43,6 @@ extern void IntHandler(int sig_num, // The signal that was raised
// Wake up the watchdog which is designed as async-signal-safe.
TaskManager::WakeUpWatchDog();
}
#endif
} // namespace dataset
} // namespace mindspore

2
mindspore/ccsrc/dataset/util/sig_handler.h
View File

@ -22,12 +22,14 @@
namespace mindspore {
namespace dataset {
// Register the custom signal handlers
#if !defined(_WIN32) && !defined(_WIN64)
extern void RegisterHandlers();
// A signal handler for SIGINT. Drives interrupt to watchdog
extern void IntHandler(int sig_num, // The signal that was raised
siginfo_t *sig_info, // The siginfo structure.
void *context); // context info
#endif
} // namespace dataset
} // namespace mindspore

9
mindspore/ccsrc/debug/anf_ir_dump.cc
View File

@ -14,6 +14,9 @@
* limitations under the License.
*/
#include "debug/anf_ir_dump.h"
#if defined(_WIN32) || defined(_WIN64)
#include <stdlib.h>
#endif
#include <fstream>
#include <map>
#include <memory>
@ -434,9 +437,15 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu
return;
}
char real_path[PATH_MAX] = {0};
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(real_path, filename.c_str(), PATH_MAX) == nullptr) {
MS_LOG(DEBUG) << "dir " << filename << " does not exit.";
}
#else
if (nullptr == realpath(filename.c_str(), real_path)) {
MS_LOG(DEBUG) << "Dir " << filename << " does not exit.";
}
#endif
OrderedMap<AnfNodePtr, int32_t> para_map;
std::string path_string = real_path;

16
mindspore/ccsrc/debug/anf_ir_utils.cc
View File

@ -48,9 +48,15 @@ std::string GetMsIrPath(void) {
if (path_ptr != nullptr) {
path = path_ptr;
char real_path[PATH_MAX] = {0};
#if defined(_WIN32) || defined(_WIN64)
if (path.size() > PATH_MAX || _fullpath(real_path, path.c_str(), PATH_MAX) == nullptr) {
MS_LOG(EXCEPTION) << "MS IR Path error, " << path_ptr;
}
#else
if (path.size() > PATH_MAX || nullptr == realpath(path.c_str(), real_path)) {
MS_LOG(EXCEPTION) << "MS IR path error, " << path_ptr;
}
#endif
path = real_path;
}
return path;
@ -2247,8 +2253,14 @@ void DumpIRProto(const FuncGraphPtr& func_graph, const std::string& suffix) {
return;
}
char real_path[PATH_MAX] = {0};
if (nullptr == realpath(file_path.c_str(), real_path)) {
MS_LOG(DEBUG) << "Dir " << file_path << " does not exit.";
char* real_path_ret = nullptr;
#if defined(_WIN32) || defined(_WIN64)
real_path_ret = _fullpath(real_path, file_path.c_str(), PATH_MAX);
#else
real_path_ret = realpath(file_path.c_str(), real_path);
#endif
if (nullptr == real_path_ret) {
MS_LOG(DEBUG) << "dir " << file_path << " does not exit.";
} else {
std::string path_string = real_path;
if (chmod(common::SafeCStr(path_string), S_IRUSR | S_IWUSR) == -1) {

7
mindspore/ccsrc/debug/info.cc
View File

@ -53,10 +53,15 @@ std::string Location::ToString(SourceLineTip tip) {
}
char path[PATH_MAX + 1] = {0x00};
#if defined(_WIN32) || defined(_WIN64)
if (file_name_.size() > PATH_MAX || _fullpath(path, file_name_.c_str(), PATH_MAX) == nullptr) {
return debug_info_ss.str();
}
#else
if (file_name_.size() > PATH_MAX || realpath(file_name_.c_str(), path) == nullptr) {
return debug_info_ss.str();
}
#endif
auto src_path = std::string(path);
std::ifstream file(src_path);
if (!file.is_open()) {

1
mindspore/ccsrc/device/cpu/cpu_resource_manager.cc
View File

@ -60,6 +60,7 @@ void CPUResourceManager::MemMalloc(const session::KernelGraph *graph) {
void *CPUResourceManager::MemMalloc(size_t mem_size) {
void *ptr = malloc(mem_size);
if (ptr != nullptr) {
memset_s(ptr, mem_size, 0, mem_size);
dynamic_mem_[ptr] = mem_size;
return ptr;
} else {

28
mindspore/ccsrc/device/gpu/distribution/collective_fake_init.cc Normal file
View File

@ -0,0 +1,28 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "device/gpu/distribution/collective_fake_init.h"
#include "utils/log_adapter.h"
namespace mindspore {
namespace device {
namespace gpu {
void CollectiveFakeInitializer::InitCollective() { MS_LOG(EXCEPTION) << "build without enable gpu!"; }
void CollectiveFakeInitializer::FinalizeCollective() { MS_LOG(EXCEPTION) << "build without enable gpu!"; }
} // namespace gpu
} // namespace device
} // namespace mindspore

37
mindspore/ccsrc/device/gpu/distribution/collective_fake_init.h Normal file
View File

@ -0,0 +1,37 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_
#define MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_
namespace mindspore {
namespace device {
namespace gpu {
class CollectiveFakeInitializer {
public:
CollectiveFakeInitializer() = default;
~CollectiveFakeInitializer() = default;
CollectiveFakeInitializer(CollectiveFakeInitializer const &) = delete;
CollectiveFakeInitializer &operator=(const CollectiveFakeInitializer &) = delete;
static void InitCollective();
static void FinalizeCollective();
};
} // namespace gpu
} // namespace device
} // namespace mindspore
#endif // MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_

20
mindspore/ccsrc/device/kernel_runtime.cc
View File

@ -44,19 +44,29 @@ bool KernelRuntime::Run(session::KernelGraph *graph) {
bool ret = false;
auto context_ptr = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context_ptr);
#if defined(_WIN32) || defined(_WIN64)
auto start_time = std::chrono::steady_clock::now();
#else
struct timeval start_time, end_time;
(void)gettimeofday(&start_time, nullptr);
#endif
bool is_task_sink = context_ptr->enable_task_sink();
if (is_task_sink) {
ret = RunTask(graph);
} else {
ret = LaunchKernel(graph);
}
#if defined(_WIN32) || defined(_WIN64)
auto end_time = std::chrono::steady_clock::now();
std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time;
MS_LOG(INFO) << "Call MS Run Success in " << cost.count() << " us";
#else
(void)gettimeofday(&end_time, nullptr);
const uint64_t kUSecondInSecond = 1000000;
uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec);
cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec);
MS_LOG(INFO) << "Call MS Run Success in " << cost << " us";
#endif
return ret;
}
@ -561,8 +571,12 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) {
AddressPtrList kernel_workspaces;
AddressPtrList kernel_outputs;
GenLaunchArgs(*kernel_mod, kernel, &kernel_inputs, &kernel_workspaces, &kernel_outputs);
#if defined(_WIN32) || defined(_WIN64)
auto start_time = std::chrono::steady_clock::now();
#else
struct timeval start_time, end_time;
(void)gettimeofday(&start_time, nullptr);
#endif
auto ret =
kernel_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, reinterpret_cast<uintptr_t>(stream_));
if (!ret) {
@ -572,11 +586,17 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) {
if (AnfAlgo::GetKernelType(kernel) == TBE_KERNEL && !SyncStream()) {
MS_LOG(EXCEPTION) << "SyncStream failed.";
}
#if defined(_WIN32) || defined(_WIN64)
auto end_time = std::chrono::steady_clock::now();
std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time;
MS_LOG(DEBUG) << "d " << kernel->fullname_with_scope() << " in " << cost.count() << " us";
#else
(void)gettimeofday(&end_time, nullptr);
const uint64_t kUSecondInSecond = 1000000;
uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec);
cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec);
MS_LOG(DEBUG) << "d " << kernel->fullname_with_scope() << " in " << cost << " us";
#endif
}
}
return true;

11
mindspore/ccsrc/kernel/common_utils.cc
View File

@ -117,7 +117,11 @@ bool IsAtomicNode(const CNodePtr &kernel_node) {
bool KernelMeta::ReadIndex(const std::string &bin_dir) {
DIR *dir = opendir(bin_dir.c_str());
if (dir == nullptr) {
#if defined(_WIN32) || defined(_WIN64)
auto ret = mkdir(bin_dir.c_str());
#else
auto ret = mkdir(bin_dir.c_str(), S_IRWXG | S_IRWXU);
#endif
if (ret != 0) {
MS_LOG(INFO) << "kernel dir not exist[" << bin_dir << "].";
return false;
@ -500,10 +504,17 @@ void SaveJsonInfo(const std::string &json_name, const std::string &info) {
}
filewrite << info << std::endl;
filewrite.close();
#if defined(_WIN32) || defined(_WIN64)
if (nullptr == _fullpath(real_path, path.c_str(), PATH_MAX)) {
MS_LOG(DEBUG) << "dir " << path << " does not exit.";
return;
}
#else
if (nullptr == realpath(path.c_str(), real_path)) {
MS_LOG(DEBUG) << "dir " << path << " does not exit.";
return;
}
#endif
MS_LOG(INFO) << "real path is :" << real_path;
if (chmod(real_path, S_IRUSR) == -1) {
MS_LOG(DEBUG) << "modify file:" << real_path << " to read only fail.";

6
mindspore/ccsrc/mindrecord/CMakeLists.txt
View File

@ -26,7 +26,11 @@ set_target_properties(_c_mindrecord PROPERTIES
)
# add link library
target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite ${PYTHON_LIB} ${SECUREC_LIBRARY} mindspore mindspore_gvar mindspore::protobuf)
if (WIN32)
target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite mindspore mindspore::protobuf)
else()
target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite ${PYTHON_LIB} ${SECUREC_LIBRARY} mindspore mindspore_gvar mindspore::protobuf)
endif()
if (USE_GLOG)
target_link_libraries(_c_mindrecord PRIVATE mindspore::glog)

24
mindspore/ccsrc/mindrecord/common/shard_utils.cc
View File

@ -65,6 +65,15 @@ std::pair<MSRStatus, std::string> GetFileName(const std::string &path) {
return {FAILED, ""};
}
char tmp[PATH_MAX] = {0};
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(tmp, dirname(&(buf[0])), PATH_MAX) == nullptr) {
MS_LOG(ERROR) << "Invalid file path, path: " << buf;
return {FAILED, ""};
}
if (_fullpath(real_path, common::SafeCStr(path), PATH_MAX) == nullptr) {
MS_LOG(DEBUG) << "Path: " << common::SafeCStr(path) << "check successfully";
}
#else
if (realpath(dirname(&(buf[0])), tmp) == nullptr) {
MS_LOG(ERROR) << "Invalid file path, path: " << buf;
return {FAILED, ""};
@ -72,6 +81,7 @@ std::pair<MSRStatus, std::string> GetFileName(const std::string &path) {
if (realpath(common::SafeCStr(path), real_path) == nullptr) {
MS_LOG(DEBUG) << "Path: " << path << "check successfully";
}
#endif
std::string s = real_path;
char sep = '/';
size_t i = s.rfind(sep, s.length());
@ -91,6 +101,15 @@ std::pair<MSRStatus, std::string> GetParentDir(const std::string &path) {
return {FAILED, ""};
}
char tmp[PATH_MAX] = {0};
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(tmp, dirname(&(buf[0])), PATH_MAX) == nullptr) {
MS_LOG(ERROR) << "Invalid file path, path: " << buf;
return {FAILED, ""};
}
if (_fullpath(real_path, common::SafeCStr(path), PATH_MAX) == nullptr) {
MS_LOG(DEBUG) << "Path: " << common::SafeCStr(path) << "check successfully";
}
#else
if (realpath(dirname(&(buf[0])), tmp) == nullptr) {
MS_LOG(ERROR) << "Invalid file path, path: " << buf;
return {FAILED, ""};
@ -98,6 +117,7 @@ std::pair<MSRStatus, std::string> GetParentDir(const std::string &path) {
if (realpath(common::SafeCStr(path), real_path) == nullptr) {
MS_LOG(DEBUG) << "Path: " << path << "check successfully";
}
#endif
std::string s = real_path;
if (s.rfind('/') + 1 <= s.size()) {
return {SUCCESS, s.substr(0, s.rfind('/') + 1)};
@ -144,6 +164,9 @@ bool IsLegalFile(const std::string &path) {
}
std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type) {
#if defined(_WIN32) || defined(_WIN64)
return {SUCCESS, 100};
#else
uint64_t ll_count = 0;
struct statfs disk_info;
if (statfs(common::SafeCStr(str_dir), &disk_info) == -1) {
@ -166,6 +189,7 @@ std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const Dis
}
return {SUCCESS, ll_count};
#endif
}
uint32_t GetMaxThreadNum() {

2
mindspore/ccsrc/mindrecord/include/common/shard_utils.h
View File

@ -21,8 +21,10 @@
#include <limits.h>
#include <stdlib.h>
#include <sys/stat.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/statfs.h>
#include <sys/wait.h>
#endif
#include <unistd.h>
#include <cassert>
#include <cmath>

2
mindspore/ccsrc/mindrecord/include/shard_reader.h
View File

@ -19,7 +19,9 @@
#include <dirent.h>
#include <signal.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/prctl.h>
#endif
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

4
mindspore/ccsrc/mindrecord/io/shard_reader.cc
View File

@ -987,8 +987,10 @@ TASK_RETURN_CONTENT ShardReader::ConsumerOneTask(int task_id, uint32_t consumer_
MSRStatus ShardReader::ConsumerByRow(int consumer_id) {
// Set thread name
#if !defined(_WIN32) && !defined(_WIN64)
auto thread_id = kThreadName + std::to_string(consumer_id);
prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0);
#endif
// Loop forever
for (;;) {
@ -1040,8 +1042,10 @@ MSRStatus ShardReader::ReadBlob(const int &shard_id, const uint64_t &page_offset
MSRStatus ShardReader::ConsumerByBlock(int consumer_id) {
// Set thread name
#if !defined(_WIN32) && !defined(_WIN64)
auto thread_id = kThreadName + std::to_string(consumer_id);
prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0);
#endif
// Loop forever
for (;;) {

10
mindspore/ccsrc/mindrecord/io/shard_writer.cc
View File

@ -63,6 +63,15 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append)
MS_LOG(ERROR) << "Securec func failed";
return FAILED;
}
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(resolved_path, dirname(&(buf[0])), PATH_MAX) == nullptr) {
MS_LOG(ERROR) << "Invalid file path";
return FAILED;
}
if (_fullpath(resolved_path, common::SafeCStr(path), PATH_MAX) == nullptr) {
MS_LOG(DEBUG) << "Path " << resolved_path;
}
#else
if (realpath(dirname(&(buf[0])), resolved_path) == nullptr) {
MS_LOG(ERROR) << "Invalid file path";
return FAILED;
@ -70,6 +79,7 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append)
if (realpath(common::SafeCStr(path), resolved_path) == nullptr) {
MS_LOG(DEBUG) << "Path " << resolved_path;
}
#endif
file_paths_.emplace_back(string(resolved_path));
}

13
mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.cc
View File

@ -38,10 +38,12 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti
(root->has_flag(ALLREDUCE_FUSION_RUN_ONCE_ONLY))) {
return changes;
}
#if defined(_WIN32) || defined(_WIN64)
auto start_time = std::chrono::steady_clock::now();
#else
struct timeval start_time, end_time;
(void)gettimeofday(&start_time, nullptr);
#endif
MS_LOG(INFO) << "Now entering allreduce fusion";
DumpGraph(root, std::string(ALLREDUCE_FUSION_BEGIN));
@ -63,11 +65,16 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti
// allreduce fusion only run once
root->flags()[ALLREDUCE_FUSION_RUN_ONCE_ONLY] = true;
res->results()[pipeline::kStepParallelGraph] = root;
#if defined(_WIN32) || defined(_WIN64)
auto end_time = std::chrono::steady_clock::now();
std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time;
MS_LOG(INFO) << "Now leaving allreduce fusion, used time: " << cost.count() << " us";
#else
(void)gettimeofday(&end_time, nullptr);
uint64_t time = 1000000 * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec);
time += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec);
MS_LOG(INFO) << "Now leaving allreduce fusion, used time: " << time << " us";
#endif
return changes;
}
} // namespace parallel

2
mindspore/ccsrc/parallel/step_auto_parallel.cc
View File

@ -229,7 +229,7 @@ size_t GetLengthOfDataType(const TypePtr &type) {
case kNumberTypeInt:
return sizeof(int);
case kNumberTypeUInt:
return sizeof(uint);
return sizeof(unsigned int);
case kNumberTypeFloat:
return sizeof(float);
default:

2
mindspore/ccsrc/pipeline/action.cc
View File

@ -264,7 +264,7 @@ bool TaskEmitAction(const ResourcePtr& res) {
auto bc_ptr = res->results()[kBackend].cast<compile::BackendPtr>();
std::vector<PrimitivePtr> cut_list = compile::nonlinear_ops;
if (bc_ptr->name() == kMsConvert) {
cut_list = compile::ms_nonlinear_ops;
cut_list = compile::GetMsNonlinearOps();
}
std::shared_ptr<CompileGraphs> compile = std::make_shared<CompileGraphs>(bc_ptr, cut_list);
res->results()[kOutput] = compile->CompileAndLink(func_graph);

14
mindspore/ccsrc/pipeline/init.cc
View File

@ -29,8 +29,11 @@
#include "parallel/context.h"
#include "parallel/device_manager.h"
#include "parallel/costmodel_context.h"
#ifdef ENABLE_GPUQUE
#include "device/gpu/distribution/collective_init.h"
#else
#include "device/gpu/distribution/collective_fake_init.h"
#endif
namespace py = pybind11;
using FuncGraph = mindspore::FuncGraph;
@ -297,9 +300,16 @@ PYBIND11_MODULE(_c_expression, m) {
(void)py::class_<OpLib, std::shared_ptr<OpLib>>(m, "Oplib")
.def(py::init())
.def("reg_op", &OpLib::RegOp, "Register op info.");
#ifdef ENABLE_GPUQUE
(void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::InitCollective,
"Init gpu collective communication mode.");
(void)m.def("finalize_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::FinalizeCollective,
"Finalize gpu collective communication mode.");
#else
(void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveFakeInitializer::InitCollective,
"Init gpu collective communication mode.");
(void)m.def("finalize_gpu_collective", &mindspore::device::gpu::CollectiveFakeInitializer::FinalizeCollective,
"Finalize gpu collective communication mode.");
#endif
}

10
mindspore/ccsrc/pre_activate/common/pass_manager.cc
View File

@ -52,17 +52,27 @@ bool PassManager::Run(const FuncGraphPtr &func_graph, const std::vector<PassPtr>
size_t num = 0;
for (const auto &pass : passes) {
if (pass != nullptr) {
#if defined(_WIN32) || defined(_WIN64)
auto start_time = std::chrono::steady_clock::now();
#else
struct timeval start_time {};
struct timeval end_time {};
(void)gettimeofday(&start_time, nullptr);
#endif
if (pass->Run(func_graph)) {
changed = true;
}
#if defined(_WIN32) || defined(_WIN64)
auto end_time = std::chrono::steady_clock::now();
std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time;
MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost.count() << " us";
#else
(void)gettimeofday(&end_time, nullptr);
const uint64_t kUSecondInSecond = 1000000;
uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec);
cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec);
MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost << " us";
#endif
if (save_graphs) {
auto dump_file_path =
save_graphs_path + "/" + "hwopt_" + name() + "_" + std::to_string(num) + "_" + pass->name() + ".ir";

10
mindspore/ccsrc/utils/log_adapter.cc
View File

@ -26,12 +26,19 @@ namespace mindspore {
#ifdef USE_GLOG
static std::string GetTime() {
#define BUFLEN 80
static char buf[BUFLEN];
#if defined(_WIN32) || defined(_WIN64)
time_t time_seconds = time(0);
struct tm now_time;
localtime_s(&now_time, &time_seconds);
sprintf_s(buf, BUFLEN, "%d-%d-%d %d:%d:%d", now_time.tm_year + 1900, now_time.tm_mon + 1, now_time.tm_mday,
now_time.tm_hour, now_time.tm_min, now_time.tm_sec);
#else
struct timeval cur_time;
(void)gettimeofday(&cur_time, NULL);
struct tm now;
(void)localtime_r(&cur_time.tv_sec, &now);
static char buf[BUFLEN];
(void)strftime(buf, BUFLEN, "%Y-%m-%d-%H:%M:%S", &now); // format date and time
// set micro-second
buf[27] = '\0';
@ -44,6 +51,7 @@ static std::string GetTime() {
buf[idx--] = '.';
}
}
#endif
return std::string(buf);
}

6
mindspore/ccsrc/vm/transform.cc
View File

@ -41,8 +41,10 @@ using TypedPrimitiveAbstractClosurePtr = std::shared_ptr<abstract::TypedPrimitiv
std::vector<PrimitivePtr> nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch,
prim::kPrimMakeTuple};
std::vector<PrimitivePtr> ms_nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch};
const std::vector<PrimitivePtr>& GetMsNonlinearOps() {
static const std::vector<PrimitivePtr> ms_nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch};
return ms_nonlinear_ops;
}
CompileGraph::CompileGraph(const BackendPtr& backend, const std::vector<PrimitivePtr>& cut_list)
: backend_(backend), cut_list_(cut_list) {

2
mindspore/ccsrc/vm/transform.h
View File

@ -42,7 +42,7 @@ extern const char kGeVm[];
// A sub namespace in ME to support compile related definition.
namespace compile {
extern std::vector<PrimitivePtr> nonlinear_ops;
extern std::vector<PrimitivePtr> ms_nonlinear_ops;
const std::vector<PrimitivePtr>& GetMsNonlinearOps();
using VmEvalFunc = std::function<BaseRef(const VectorRef&)>;
using VmEvalFuncPtr = std::shared_ptr<std::function<BaseRef(const VectorRef&)>>;

7
mindspore/log.py
View File

@ -19,11 +19,13 @@ import sys
import os
import stat
import time
import fcntl
import logging
from logging.handlers import RotatingFileHandler
import traceback
import threading
import platform
if platform.system() != "Windows":
import fcntl
__all__ = ['get_level', 'get_log_config']
@ -90,7 +92,8 @@ class _MultiCompatibleRotatingFileHandler(RotatingFileHandler):
# Attain an exclusive lock with bloking mode by `fcntl` module.
with open(self.baseFilename, 'a') as file_pointer:
fcntl.lockf(file_pointer.fileno(), fcntl.LOCK_EX)
if platform.system() != "Windows":
fcntl.lockf(file_pointer.fileno(), fcntl.LOCK_EX)
if self.backupCount > 0:
self.rolling_rename()

12
third_party/securec/CMakeLists.txt vendored
View File

@ -1,10 +1,18 @@
SET(CMAKE_BUILD_TYPE "Debug")
SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='")
if (WIN32)
SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer")
else()
SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='")
endif()
SET(CMAKE_C_FLAGS_RELEASE "$ENV{CFLAGS} -fPIC -O3 -Wall -Wno-deprecated-declarations")
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
#add flags
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include -Werror")
if (WIN32)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include")
else()
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include -Werror")
endif()
include_directories(./include)