!329 [MS]support building on windows 10

Merge pull request !329 from chenjianping/building-on-windows-2
This commit is contained in:
mindspore-ci-bot 2020-04-15 19:10:50 +08:00 committed by Gitee
commit b2b3e24a8e
48 changed files with 619 additions and 133 deletions

View File

@ -12,6 +12,7 @@ else()
endif() endif()
set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -Wl,--allow-shlib-undefined -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)=' -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp") set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -Wl,--allow-shlib-undefined -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)=' -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/include -std=c++17 -Werror -Wall -Wno-deprecated-declarations -fPIC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/include -std=c++17 -Werror -Wall -Wno-deprecated-declarations -fPIC")
set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
@ -29,6 +30,18 @@ find_package(Python3 3.7 COMPONENTS Interpreter Development)
if(Python3_FOUND) if(Python3_FOUND)
set(PYTHON_INCLUDE_DIRS "${Python3_INCLUDE_DIRS}") set(PYTHON_INCLUDE_DIRS "${Python3_INCLUDE_DIRS}")
set(PYTHON_LIBRARIES "${Python3_LIBRARIES}") set(PYTHON_LIBRARIES "${Python3_LIBRARIES}")
if (WIN32)
if (Python3_DIR)
message("Python3_DIR set already: " ${Python3_DIR})
else()
string(LENGTH ${PYTHON_LIBRARIES} PYTHON_LIBRARIES_LEN)
string(LENGTH "libpythonxx.a" Python3_NAME_LEN)
math(EXPR Python3_DIR_LEN ${PYTHON_LIBRARIES_LEN}-${Python3_NAME_LEN})
string(SUBSTRING ${Python3_LIBRARIES} 0 ${Python3_DIR_LEN} Python3_DIR)
message("Python3_DIR: " ${Python3_DIR})
endif()
link_directories(${Python3_DIR})
endif()
else() else()
find_python_package(py_inc py_lib) find_python_package(py_inc py_lib)
set(PYTHON_INCLUDE_DIRS "${py_inc}") set(PYTHON_INCLUDE_DIRS "${py_inc}")

36
build.bat Normal file
View File

@ -0,0 +1,36 @@
@echo off
@title mindspore_build
SET BASEPATH=%CD%
IF NOT EXIST %BASEPATH%/build (
md "build"
)
cd %BASEPATH%/build
SET BUILD_PATH=%CD%
IF NOT EXIST %BUILD_PATH%/mindspore (
md "mindspore"
)
cd %CD%/mindspore
cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_CPU=ON -DENABLE_MINDDATA=ON -DUSE_GLOG=ON -G "CodeBlocks - MinGW Makefiles" ../..
IF NOT %errorlevel% == 0 (
goto run_fail
)
cmake --build . --target all -- -j6
IF NOT %errorlevel% == 0 (
goto run_fail
)
cd %BASEPATH%
goto run_eof
:run_fail
cd %BASEPATH%
echo "build fail."
:run_eof

View File

@ -1,5 +1,8 @@
set(flatbuffers_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") set(flatbuffers_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2")
set(flatbuffers_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") set(flatbuffers_CFLAGS "-D_FORTIFY_SOURCE=2 -O2")
if (WIN32)
set(flatbuffers_USE_STATIC_LIBS ON)
endif()
mindspore_add_pkg(flatbuffers mindspore_add_pkg(flatbuffers
VER 1.11.0 VER 1.11.0
LIBS flatbuffers LIBS flatbuffers

View File

@ -1,11 +1,22 @@
set(onednn_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") set(onednn_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2")
set(onednn_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") set(onednn_CFLAGS "-D_FORTIFY_SOURCE=2 -O2")
mindspore_add_pkg(onednn if (CMAKE_SYSTEM_NAME MATCHES "Windows")
mindspore_add_pkg(onednn
VER 1.1.1
LIBS dnnl mkldnn
HEAD_ONLY ./
RELEASE on
URL https://github.com/oneapi-src/oneDNN/releases/download/v1.1.1/dnnl_win_1.1.1_cpu_vcomp.zip
MD5 ecaab9ed549643067699c80e5cea1c23)
else()
mindspore_add_pkg(onednn
VER 1.1.2 VER 1.1.2
LIBS dnnl mkldnn LIBS dnnl mkldnn
URL https://github.com/oneapi-src/oneDNN/archive/v1.1.2.tar.gz URL https://github.com/oneapi-src/oneDNN/archive/v1.1.2.tar.gz
MD5 ab40d52230f3ad1d7a6f06ce0f6bc17a MD5 ab40d52230f3ad1d7a6f06ce0f6bc17a
CMAKE_OPTION -DDNNL_ARCH_OPT_FLAGS='' -DDNNL_CPU_RUNTIME='SEQ' -DDNNL_BUILD_EXAMPLES=OFF -DDNNL_BUILD_TESTS=OFF) CMAKE_OPTION -DDNNL_ARCH_OPT_FLAGS='' -DDNNL_CPU_RUNTIME='SEQ' -DDNNL_BUILD_EXAMPLES=OFF -DDNNL_BUILD_TESTS=OFF)
endif()
include_directories(${onednn_INC}) include_directories(${onednn_INC})
add_library(mindspore::dnnl ALIAS onednn::dnnl) add_library(mindspore::dnnl ALIAS onednn::dnnl)
add_library(mindspore::mkldnn ALIAS onednn::mkldnn) add_library(mindspore::mkldnn ALIAS onednn::mkldnn)

View File

@ -2,35 +2,72 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(opencv_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") set(opencv_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") set(opencv_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_LDFLAGS "-Wl") set(opencv_LDFLAGS "-Wl")
elseif (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
set(opencv_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
else() else()
set(opencv_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") set(opencv_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") set(opencv_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") set(opencv_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
endif() endif()
mindspore_add_pkg(opencv if (WIN32)
VER 4.2.0 mindspore_add_pkg(opencv
LIBS opencv_core opencv_imgcodecs opencv_imgproc VER 4.2.0
URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz LIBS libopencv_core420.dll.a libopencv_imgcodecs420.dll.a libopencv_imgproc420.dll.a
MD5 e8cb208ce2723481408b604b480183b6 LIB_PATH x64/mingw/lib
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz
-DBUILD_ZLIB=ON MD5 e8cb208ce2723481408b604b480183b6
-DBUILD_JPEG=ON CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF
-DBUILD_PNG=ON -DBUILD_ZLIB=ON
-DBUILD_OPENEXR=ON -DBUILD_JPEG=ON
-DBUILD_TESTS=OFF -DBUILD_PNG=ON
-DBUILD_PERF_TESTS=OFF -DBUILD_OPENEXR=ON
-DBUILD_opencv_apps=OFF -DBUILD_TESTS=OFF
-DCMAKE_SKIP_RPATH=TRUE -DBUILD_PERF_TESTS=OFF
-DBUILD_opencv_python3=OFF -DBUILD_opencv_apps=OFF
-DWITH_FFMPEG=OFF -DCMAKE_SKIP_RPATH=TRUE
-DWITH_TIFF=ON -DBUILD_opencv_python3=OFF
-DBUILD_TIFF=OFF -DWITH_FFMPEG=OFF
-DWITH_JASPER=OFF -DWITH_TIFF=ON
-DBUILD_JASPER=OFF -DBUILD_TIFF=OFF
-DTIFF_INCLUDE_DIR=${tiff_INC} -DWITH_JASPER=OFF
-DTIFF_LIBRARY=${tiff_LIB}) -DBUILD_JASPER=OFF
include_directories(${opencv_INC}/opencv4) -DTIFF_INCLUDE_DIR=${tiff_INC}
add_library(mindspore::opencv_core ALIAS opencv::opencv_core) -DTIFF_LIBRARY=${tiff_LIB})
add_library(mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs) else()
add_library(mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc) mindspore_add_pkg(opencv
VER 4.2.0
LIBS opencv_core opencv_imgcodecs opencv_imgproc
URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz
MD5 e8cb208ce2723481408b604b480183b6
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF
-DBUILD_ZLIB=ON
-DBUILD_JPEG=ON
-DBUILD_PNG=ON
-DBUILD_OPENEXR=ON
-DBUILD_TESTS=OFF
-DBUILD_PERF_TESTS=OFF
-DBUILD_opencv_apps=OFF
-DCMAKE_SKIP_RPATH=TRUE
-DBUILD_opencv_python3=OFF
-DWITH_FFMPEG=OFF
-DWITH_TIFF=ON
-DBUILD_TIFF=OFF
-DWITH_JASPER=OFF
-DBUILD_JASPER=OFF
-DTIFF_INCLUDE_DIR=${tiff_INC}
-DTIFF_LIBRARY=${tiff_LIB})
endif()
if (WIN32)
include_directories(${opencv_INC})
add_library(mindspore::opencv_core ALIAS opencv::libopencv_core420.dll.a)
add_library(mindspore::opencv_imgcodecs ALIAS opencv::libopencv_imgcodecs420.dll.a)
add_library(mindspore::opencv_imgproc ALIAS opencv::libopencv_imgproc420.dll.a)
else()
include_directories(${opencv_INC}/opencv4)
add_library(mindspore::opencv_core ALIAS opencv::opencv_core)
add_library(mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs)
add_library(mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc)
endif()

View File

@ -77,22 +77,36 @@ function(ms_protobuf_generate_py c_var h_var py_var)
list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc") list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc")
list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h") list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h")
list(APPEND ${py_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py") list(APPEND ${py_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py")
if (WIN32)
add_custom_command( add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc" OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}" COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}"
COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" COMMAND perl -pi.bak -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
COMMAND cp "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/"
DEPENDS protobuf::protoc ${abs_file} DEPENDS protobuf::protoc ${abs_file}
COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM ) COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
else()
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}"
COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
COMMAND cp "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/"
DEPENDS protobuf::protoc ${abs_file}
COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
endif()
endforeach() endforeach()
set_source_files_properties(${${c_var}} ${${h_var}} ${${py_var}} PROPERTIES GENERATED TRUE) set_source_files_properties(${${c_var}} ${${h_var}} ${${py_var}} PROPERTIES GENERATED TRUE)
set(${c_var} ${${c_var}} PARENT_SCOPE) set(${c_var} ${${c_var}} PARENT_SCOPE)
set(${h_var} ${${h_var}} PARENT_SCOPE) set(${h_var} ${${h_var}} PARENT_SCOPE)

View File

@ -1,19 +1,36 @@
if (WIN32)
mindspore_add_pkg(sqlite-head
VER 3.31.1
HEAD_ONLY ./
URL https://sqlite.org/2020/sqlite-amalgamation-3310100.zip
MD5 2b7bfcdd97dc281903a9aee966213fe4)
include_directories(${sqlite-head_INC})
mindspore_add_pkg(sqlite
VER 3.31.1
LIBS sqlite3
LIB_PATH ./
HEAD_ONLY ./
RELEASE ON
URL https://sqlite.org/2020/sqlite-dll-win64-x64-3310100.zip
MD5 662c9d2b05467d590ba5c0443e7fd6bd)
set(sqlite_USE_STATIC_LIBS ON) else ()
set(sqlite_CXXFLAGS) set(sqlite_USE_STATIC_LIBS ON)
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(sqlite_CXXFLAGS)
set(sqlite_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
else() set(sqlite_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
set(sqlite_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") else()
endif() set(sqlite_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") endif()
set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
mindspore_add_pkg(sqlite mindspore_add_pkg(sqlite
VER 3.31.1 VER 3.31.1
LIBS sqlite3 LIBS sqlite3
URL https://github.com/sqlite/sqlite/archive/version-3.31.1.tar.gz URL https://github.com/sqlite/sqlite/archive/version-3.31.1.tar.gz
MD5 5f4e7b4016c15f4fb5855615279819da MD5 5f4e7b4016c15f4fb5855615279819da
PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/sqlite/sqlite.patch001 PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/sqlite/sqlite.patch001
CONFIGURE_COMMAND ./configure --enable-shared=no --disable-tcl --disable-editline --enable-json1) CONFIGURE_COMMAND ./configure --enable-shared=no --disable-tcl --disable-editline --enable-json1)
include_directories(${sqlite_INC}) include_directories(${sqlite_INC})
add_library(mindspore::sqlite ALIAS sqlite::sqlite3) endif ()
add_library(mindspore::sqlite ALIAS sqlite::sqlite3)

View File

@ -1,6 +1,10 @@
set(SECURE_CXX_FLAGS "") set(SECURE_CXX_FLAGS "")
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
set(SECURE_CXX_FLAGS "-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") if (WIN32)
set(SECURE_CXX_FLAGS "-fstack-protector-all")
else()
set(SECURE_CXX_FLAGS "-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
endif()
endif() endif()
set(_ms_tmp_CMAKE_CXX_FLAGS_F ${CMAKE_CXX_FLAGS}) set(_ms_tmp_CMAKE_CXX_FLAGS_F ${CMAKE_CXX_FLAGS})
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")

View File

@ -19,7 +19,11 @@ option(ENABLE_MPI "enable mpi" OFF)
option(ENABLE_AKG "enable akg" OFF) option(ENABLE_AKG "enable akg" OFF)
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") if (WIN32)
set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all")
else()
set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
endif()
endif() endif()
if (CMAKE_SYSTEM_NAME MATCHES "Darwin") if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
@ -106,4 +110,4 @@ endif()
if(ENABLE_DUMP_E2E) if(ENABLE_DUMP_E2E)
add_compile_definitions(ENABLE_DUMP_E2E) add_compile_definitions(ENABLE_DUMP_E2E)
endif() endif()

View File

@ -103,7 +103,7 @@ function(__download_pkg_with_git pkg_name pkg_url pkg_git_commit pkg_md5)
endfunction() endfunction()
function(__find_pkg_then_add_target pkg_name pkg_exe) function(__find_pkg_then_add_target pkg_name pkg_exe lib_path)
unset(${pkg_name}_LIBS) unset(${pkg_name}_LIBS)
@ -129,15 +129,24 @@ function(__find_pkg_then_add_target pkg_name pkg_exe)
set(_LIB_TYPE STATIC) set(_LIB_TYPE STATIC)
endif () endif ()
set(${_LIB_NAME}_LIB ${_LIB_NAME}_LIB-NOTFOUND) set(${_LIB_NAME}_LIB ${_LIB_NAME}_LIB-NOTFOUND)
find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/lib NO_DEFAULT_PATH) find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/${lib_path} NO_DEFAULT_PATH)
if(NOT ${_LIB_NAME}_LIB) if(NOT ${_LIB_NAME}_LIB)
return() return()
endif() endif()
add_library(${pkg_name}::${_LIB_NAME} ${_LIB_TYPE} IMPORTED GLOBAL) add_library(${pkg_name}::${_LIB_NAME} ${_LIB_TYPE} IMPORTED GLOBAL)
set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES if (WIN32 AND ${_LIB_TYPE} STREQUAL "SHARED")
INTERFACE_INCLUDE_DIRECTORIES "${${pkg_name}_BASE_DIR}/include" set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES IMPORTED_IMPLIB_RELEASE ${${_LIB_NAME}_LIB})
IMPORTED_LOCATION ${${_LIB_NAME}_LIB} else()
) set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES IMPORTED_LOCATION ${${_LIB_NAME}_LIB})
endif()
if (EXISTS ${${pkg_name}_BASE_DIR}/include)
set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${${pkg_name}_BASE_DIR}/include")
endif ()
list(APPEND ${pkg_name}_LIBS ${pkg_name}::${_LIB_NAME}) list(APPEND ${pkg_name}_LIBS ${pkg_name}::${_LIB_NAME})
message("found ${${_LIB_NAME}_LIB}") message("found ${${_LIB_NAME}_LIB}")
STRING( REGEX REPLACE "(.+)/(.+)" "\\1" LIBPATH ${${_LIB_NAME}_LIB}) STRING( REGEX REPLACE "(.+)/(.+)" "\\1" LIBPATH ${${_LIB_NAME}_LIB})
@ -192,12 +201,18 @@ set(MS_FIND_NO_DEFAULT_PATH ${MS_FIND_NO_DEFAULT_PATH} PARENT_SCOPE)
function(mindspore_add_pkg pkg_name ) function(mindspore_add_pkg pkg_name )
set(options ) set(options )
set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH) set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH RELEASE LIB_PATH)
set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES) set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES)
cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} )
if (NOT PKG_CMAKE_PATH)
set(PKG_CMAKE_PATH .) if (NOT PKG_LIB_PATH)
set(PKG_LIB_PATH lib)
endif () endif ()
if(NOT PKG_EXE)
set(PKG_EXE 0)
endif()
set(__FIND_PKG_NAME ${pkg_name}) set(__FIND_PKG_NAME ${pkg_name})
string(TOLOWER ${pkg_name} pkg_name) string(TOLOWER ${pkg_name} pkg_name)
message("pkg name:${__FIND_PKG_NAME},${pkg_name}") message("pkg name:${__FIND_PKG_NAME},${pkg_name}")
@ -225,18 +240,17 @@ function(mindspore_add_pkg pkg_name )
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE) set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE)
add_library(${pkg_name} INTERFACE) add_library(${pkg_name} INTERFACE)
target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC}) target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})
if (${PKG_RELEASE})
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS})
endif ()
return() return()
endif () endif ()
if(NOT PKG_EXE)
set(PKG_EXE 0)
endif()
set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR}) set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR})
set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR} PARENT_SCOPE) set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR} PARENT_SCOPE)
if (PKG_LIBS) if (PKG_LIBS)
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS}) __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS})
if(${pkg_name}_LIBS) if(${pkg_name}_LIBS)
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE) set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
message("Found libs: ${${pkg_name}_LIBS}") message("Found libs: ${${pkg_name}_LIBS}")
@ -283,8 +297,10 @@ function(mindspore_add_pkg pkg_name )
file(GLOB ${pkg_name}_SOURCE_SUBDIRS ${${pkg_name}_SOURCE_DIR}/*) file(GLOB ${pkg_name}_SOURCE_SUBDIRS ${${pkg_name}_SOURCE_DIR}/*)
file(COPY ${${pkg_name}_SOURCE_SUBDIRS} DESTINATION ${${pkg_name}_BASE_DIR}) file(COPY ${${pkg_name}_SOURCE_SUBDIRS} DESTINATION ${${pkg_name}_BASE_DIR})
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE) set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE)
add_library(${pkg_name} INTERFACE) if (NOT PKG_RELEASE)
target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC}) add_library(${pkg_name} INTERFACE)
target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})
endif ()
elseif (PKG_CMAKE_OPTION) elseif (PKG_CMAKE_OPTION)
# in cmake # in cmake
@ -355,7 +371,7 @@ function(mindspore_add_pkg pkg_name )
endif() endif()
if (PKG_LIBS) if (PKG_LIBS)
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS}) __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS})
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE) set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
if(NOT ${pkg_name}_LIBS) if(NOT ${pkg_name}_LIBS)
message(FATAL_ERROR "Can not find pkg: ${pkg_name}") message(FATAL_ERROR "Can not find pkg: ${pkg_name}")

View File

@ -5,6 +5,10 @@ if(ENABLE_CPU)
file(GLOB_RECURSE CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} file(GLOB_RECURSE CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"device/cpu/*.cc" "device/cpu/*.cc"
) )
if (CMAKE_SYSTEM_NAME MATCHES "Windows")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF")
add_compile_definitions(BUILDING_DLL)
endif()
endif() endif()
if(ENABLE_GPU) if(ENABLE_GPU)
@ -150,7 +154,15 @@ file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"kernel/kash/*.cc" "kernel/kash/*.cc"
"device/gpu/distribution/collective_init.cc" "device/gpu/distribution/collective_init.cc"
) )
if (ENABLE_CPU)
list(REMOVE_ITEM MINDSPORE_SRC_LIST "device/gpu/distribution/collective_init.cc")
if (WIN32)
list(REMOVE_ITEM MINDSPORE_SRC_LIST "kernel/kernel_query.cc")
endif()
endif()
if (NOT ENABLE_GPU)
list(APPEND MINDSPORE_SRC_LIST "device/gpu/distribution/collective_fake_init.cc")
endif()
file(GLOB_RECURSE MEM_REUSE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} file(GLOB_RECURSE MEM_REUSE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"pre_activate/mem_reuse/*.cc" "pre_activate/mem_reuse/*.cc"
) )
@ -248,6 +260,7 @@ file(GLOB_RECURSE MS_GVAR_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
add_library(mindspore_gvar SHARED ${MS_GVAR_SRC_LIST}) add_library(mindspore_gvar SHARED ${MS_GVAR_SRC_LIST})
add_library(mindspore STATIC ${MINDSPORE_SRC_LIST}) add_library(mindspore STATIC ${MINDSPORE_SRC_LIST})
add_dependencies(mindspore GENERATED_OUTPUT_DIR)
if(ENABLE_D) if(ENABLE_D)
list(APPEND MINDSPORE_PROTO_LIST ${MINDSPORE_PROTO_AICPU_LIST}) list(APPEND MINDSPORE_PROTO_LIST ${MINDSPORE_PROTO_AICPU_LIST})
@ -315,19 +328,33 @@ if(ENABLE_D)
endif() endif()
target_link_libraries(mindspore securec) target_link_libraries(mindspore securec)
target_link_libraries(mindspore dl) if (NOT WIN32)
target_link_libraries(mindspore dl)
endif()
target_link_libraries(mindspore mindspore::flatbuffers) target_link_libraries(mindspore mindspore::flatbuffers)
# link protobuf # link protobuf
if (ENABLE_D) if (ENABLE_D)
target_link_libraries(mindspore mindspore::protobuf) target_link_libraries(mindspore mindspore::protobuf)
endif() endif()
if (WIN32)
target_link_libraries(mindspore ${PYTHON_LIBRARIES} mindspore_gvar)
endif()
# set c_expression building # set c_expression building
if (WIN32)
set(PYTHON_MODULE_SOURCE ${MS_GVAR_SRC_LIST}
pipeline/init.cc
kernel/oplib/oplib.cc
${MINDSPORE_SRC_LIST} ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST}
${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST})
else()
set(PYTHON_MODULE_SOURCE set(PYTHON_MODULE_SOURCE
pipeline/init.cc pipeline/init.cc
kernel/oplib/oplib.cc kernel/oplib/oplib.cc
${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST} ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST}
${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST}) ${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST})
endif()
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
pybind11_add_module(_c_expression ${PYTHON_MODULE_SOURCE}) pybind11_add_module(_c_expression ${PYTHON_MODULE_SOURCE})
@ -339,6 +366,8 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux")
elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set_target_properties(_c_expression PROPERTIES MACOSX_RPATH ON) set_target_properties(_c_expression PROPERTIES MACOSX_RPATH ON)
set(ORIGIN_PATH @loader_path) set(ORIGIN_PATH @loader_path)
elseif (CMAKE_SYSTEM_NAME MATCHES "Windows")
set(ORIGIN_PATH $ORIGIN)
else () else ()
MESSAGE(FATAL_ERROR "other platform: ${CMAKE_SYSTEM_NAME}") MESSAGE(FATAL_ERROR "other platform: ${CMAKE_SYSTEM_NAME}")
endif () endif ()
@ -346,11 +375,22 @@ endif ()
set(ORIGIN_PATH ${ORIGIN_PATH}/lib) set(ORIGIN_PATH ${ORIGIN_PATH}/lib)
set_target_properties(_c_expression PROPERTIES INSTALL_RPATH ${ORIGIN_PATH}) set_target_properties(_c_expression PROPERTIES INSTALL_RPATH ${ORIGIN_PATH})
target_link_libraries(_c_expression PRIVATE if (WIN32)
mindspore::pybind11_module target_link_libraries(_c_expression PRIVATE
mindspore mindspore::pybind11_module
mindspore_gvar securec
) proto_input
mindspore::flatbuffers
mindspore::glog
)
else()
target_link_libraries(_c_expression PRIVATE
mindspore::pybind11_module
mindspore
mindspore_gvar
mindspore::glog
)
endif()
if(ENABLE_GPU) if(ENABLE_GPU)
execute_process(COMMAND bash ${CMAKE_SOURCE_DIR}/third_party/apply_patches.sh execute_process(COMMAND bash ${CMAKE_SOURCE_DIR}/third_party/apply_patches.sh
@ -493,7 +533,3 @@ if(ENABLE_MINDDATA)
add_subdirectory(mindrecord) add_subdirectory(mindrecord)
add_subdirectory(dataset) add_subdirectory(dataset)
endif() endif()
if (USE_GLOG)
target_link_libraries(_c_expression PRIVATE mindspore::glog)
endif()

View File

@ -80,14 +80,19 @@ set_target_properties(_c_dataengine PROPERTIES
###################################################################### ######################################################################
################# Link with external libraries ######################## ################# Link with external libraries ########################
target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar) if (WIN32)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module -ldl mindspore::protobuf ${SECUREC_LIBRARY}) target_link_libraries(_c_dataengine PRIVATE mindspore)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module ${PYTHON_LIBRARIES} mindspore::protobuf ${SECUREC_LIBRARY})
else()
target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module -ldl mindspore::protobuf ${SECUREC_LIBRARY})
endif()
target_link_libraries(_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::opencv_core mindspore::opencv_imgcodecs target_link_libraries(_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::opencv_core mindspore::opencv_imgcodecs
mindspore::opencv_imgproc) mindspore::opencv_imgproc)
if (ENABLE_GPUQUE) if (ENABLE_GPUQUE)
target_link_libraries(_c_dataengine PRIVATE gpu_queue target_link_libraries(_c_dataengine PRIVATE gpu_queue
${CUDNN_PATH}/lib64/libcudnn.so ${CUDNN_PATH}/lib64/libcudnn.so
${CUDA_PATH}/lib64/libcudart.so ${CUDA_PATH}/lib64/libcudart.so
${CUDA_PATH}/lib64/stubs/libcuda.so) ${CUDA_PATH}/lib64/stubs/libcuda.so)
endif () endif ()
@ -96,7 +101,12 @@ if (ENABLE_TDTQUE)
endif () endif ()
add_dependencies(_c_dataengine _c_mindrecord) add_dependencies(_c_dataengine _c_mindrecord)
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord) if (WIN32)
set(MINDRECORD_LINK_OBJECT ${CMAKE_BINARY_DIR}/mindspore/ccsrc/mindrecord/CMakeFiles/_c_mindrecord.dir/objects.a)
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord ${MINDRECORD_LINK_OBJECT} mindspore::sqlite)
else()
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord)
endif()
if (USE_GLOG) if (USE_GLOG)
target_link_libraries(_c_dataengine PRIVATE mindspore::glog) target_link_libraries(_c_dataengine PRIVATE mindspore::glog)

View File

@ -19,7 +19,9 @@
#include "dataset/kernels/no_op.h" #include "dataset/kernels/no_op.h"
#include "dataset/kernels/data/one_hot_op.h" #include "dataset/kernels/data/one_hot_op.h"
#include "dataset/kernels/image/center_crop_op.h" #include "dataset/kernels/image/center_crop_op.h"
#if !defined(_WIN32) && !defined(_WIN64)
#include "dataset/kernels/image/change_mode_op.h" #include "dataset/kernels/image/change_mode_op.h"
#endif
#include "dataset/kernels/image/cut_out_op.h" #include "dataset/kernels/image/cut_out_op.h"
#include "dataset/kernels/image/decode_op.h" #include "dataset/kernels/image/decode_op.h"
#include "dataset/kernels/image/distort_bounding_box_crop_op.h" #include "dataset/kernels/image/distort_bounding_box_crop_op.h"
@ -279,9 +281,11 @@ void bindTensorOps2(py::module *m) {
py::arg("fillG") = RandomCropOp::kDefFillG, py::arg("fillB") = RandomCropOp::kDefFillB); py::arg("fillG") = RandomCropOp::kDefFillG, py::arg("fillB") = RandomCropOp::kDefFillB);
(void)py::class_<HwcToChwOp, TensorOp, std::shared_ptr<HwcToChwOp>>(*m, "ChannelSwapOp").def(py::init<>()); (void)py::class_<HwcToChwOp, TensorOp, std::shared_ptr<HwcToChwOp>>(*m, "ChannelSwapOp").def(py::init<>());
#if !defined(_WIN32) && !defined(_WIN64)
(void)py::class_<ChangeModeOp, TensorOp, std::shared_ptr<ChangeModeOp>>( (void)py::class_<ChangeModeOp, TensorOp, std::shared_ptr<ChangeModeOp>>(
*m, "ChangeModeOp", "Tensor operation to change colors from BGR to RGB") *m, "ChangeModeOp", "Tensor operation to change colors from BGR to RGB")
.def(py::init<>()); .def(py::init<>());
#endif
(void)py::class_<OneHotOp, TensorOp, std::shared_ptr<OneHotOp>>( (void)py::class_<OneHotOp, TensorOp, std::shared_ptr<OneHotOp>>(
*m, "OneHotOp", "Tensor operation to apply one hot encoding. Takes number of classes.") *m, "OneHotOp", "Tensor operation to apply one hot encoding. Takes number of classes.")

View File

@ -22,6 +22,10 @@
#include <vector> #include <vector>
#include "./securec.h" #include "./securec.h"
#include "utils/log_adapter.h" #include "utils/log_adapter.h"
#if defined(_WIN32) || defined(_WIN64)
#undef HAVE_STDDEF_H
#undef HAVE_STDLIB_H
#endif
#include "pybind11/numpy.h" #include "pybind11/numpy.h"
#include "pybind11/pybind11.h" #include "pybind11/pybind11.h"
#include "pybind11/stl.h" #include "pybind11/stl.h"

View File

@ -85,7 +85,11 @@ Status ShuffleOp::SelfReset() {
if (!reshuffle_each_epoch_) { if (!reshuffle_each_epoch_) {
rng_ = std::mt19937_64(shuffle_seed_); rng_ = std::mt19937_64(shuffle_seed_);
} else { } else {
#if defined(_WIN32) || defined(_WIN64)
std::random_device random_device;
#else
std::random_device random_device("/dev/urandom"); std::random_device random_device("/dev/urandom");
#endif
std::uniform_int_distribution<int32_t> distribution(0, std::numeric_limits<int32_t>::max()); std::uniform_int_distribution<int32_t> distribution(0, std::numeric_limits<int32_t>::max());
shuffle_seed_ = distribution(random_device); shuffle_seed_ = distribution(random_device);
rng_ = std::mt19937_64(shuffle_seed_); rng_ = std::mt19937_64(shuffle_seed_);

View File

@ -1,22 +1,46 @@
add_library(kernels-image OBJECT if (WIN32)
center_crop_op.cc add_library(kernels-image OBJECT
change_mode_op.cc center_crop_op.cc
cut_out_op.cc cut_out_op.cc
decode_op.cc decode_op.cc
distort_bounding_box_crop_op.cc distort_bounding_box_crop_op.cc
hwc_to_chw_op.cc hwc_to_chw_op.cc
image_utils.cc image_utils.cc
normalize_op.cc normalize_op.cc
pad_op.cc pad_op.cc
random_color_adjust_op.cc random_color_adjust_op.cc
random_crop_decode_resize_op.cc random_crop_decode_resize_op.cc
random_crop_and_resize_op.cc random_crop_and_resize_op.cc
random_crop_op.cc random_crop_op.cc
random_horizontal_flip_op.cc random_horizontal_flip_op.cc
random_resize_op.cc random_resize_op.cc
random_rotation_op.cc random_rotation_op.cc
random_vertical_flip_op.cc random_vertical_flip_op.cc
rescale_op.cc rescale_op.cc
resize_bilinear_op.cc resize_bilinear_op.cc
resize_op.cc resize_op.cc
) )
else()
add_library(kernels-image OBJECT
center_crop_op.cc
change_mode_op.cc
cut_out_op.cc
decode_op.cc
distort_bounding_box_crop_op.cc
hwc_to_chw_op.cc
image_utils.cc
normalize_op.cc
pad_op.cc
random_color_adjust_op.cc
random_crop_decode_resize_op.cc
random_crop_and_resize_op.cc
random_crop_op.cc
random_horizontal_flip_op.cc
random_resize_op.cc
random_rotation_op.cc
random_vertical_flip_op.cc
rescale_op.cc
resize_bilinear_op.cc
resize_op.cc
)
endif()

View File

@ -186,7 +186,11 @@ void JpegSetSource(j_decompress_ptr cinfo, const void *data, int64_t datasize) {
(*cinfo->mem->alloc_small)(reinterpret_cast<j_common_ptr>(cinfo), JPOOL_PERMANENT, sizeof(struct jpeg_source_mgr))); (*cinfo->mem->alloc_small)(reinterpret_cast<j_common_ptr>(cinfo), JPOOL_PERMANENT, sizeof(struct jpeg_source_mgr)));
cinfo->src->init_source = JpegInitSource; cinfo->src->init_source = JpegInitSource;
cinfo->src->fill_input_buffer = JpegFillInputBuffer; cinfo->src->fill_input_buffer = JpegFillInputBuffer;
#if defined(_WIN32) || defined(_WIN64)
cinfo->src->skip_input_data = reinterpret_cast<void (*)(j_decompress_ptr, long)>(JpegSkipInputData);
#else
cinfo->src->skip_input_data = JpegSkipInputData; cinfo->src->skip_input_data = JpegSkipInputData;
#endif
cinfo->src->resync_to_restart = jpeg_resync_to_restart; cinfo->src->resync_to_restart = jpeg_resync_to_restart;
cinfo->src->term_source = JpegTermSource; cinfo->src->term_source = JpegTermSource;
cinfo->src->bytes_in_buffer = datasize; cinfo->src->bytes_in_buffer = datasize;

View File

@ -22,6 +22,10 @@
#include <random> #include <random>
#include <string> #include <string>
#include <vector> #include <vector>
#if defined(_WIN32) || defined(_WIN64)
#undef HAVE_STDDEF_H
#undef HAVE_STDLIB_H
#endif
#include "./jpeglib.h" #include "./jpeglib.h"
#include "./jerror.h" #include "./jerror.h"
#include <opencv2/imgproc/imgproc.hpp> #include <opencv2/imgproc/imgproc.hpp>

View File

@ -27,7 +27,7 @@
namespace mindspore { namespace mindspore {
namespace dataset { namespace dataset {
#ifdef _WIN32 #ifdef _WIN32
char Path::_separator = '\\'; char Path::separator_ = '\\';
#else #else
char Path::separator_ = '/'; char Path::separator_ = '/';
#endif #endif
@ -129,7 +129,11 @@ bool Path::IsDirectory() {
Status Path::CreateDirectory() { Status Path::CreateDirectory() {
if (!Exists()) { if (!Exists()) {
#if defined(_WIN32) || defined(_WIN64)
int rc = mkdir(common::SafeCStr(path_));
#else
int rc = mkdir(common::SafeCStr(path_), 0700); int rc = mkdir(common::SafeCStr(path_), 0700);
#endif
if (rc) { if (rc) {
std::ostringstream oss; std::ostringstream oss;
oss << "Unable to create directory " << path_ << ". Errno = " << errno; oss << "Unable to create directory " << path_ << ". Errno = " << errno;

View File

@ -32,7 +32,11 @@ namespace dataset {
uint32_t GetSeed() { uint32_t GetSeed() {
uint32_t seed = GlobalContext::config_manager()->seed(); uint32_t seed = GlobalContext::config_manager()->seed();
if (seed == std::mt19937::default_seed) { if (seed == std::mt19937::default_seed) {
#if defined(_WIN32) || defined(_WIN64)
std::random_device random_device;
#else
std::random_device random_device("/dev/urandom"); std::random_device random_device("/dev/urandom");
#endif
std::uniform_int_distribution<uint32_t> distribution(0, std::numeric_limits<uint32_t>::max()); std::uniform_int_distribution<uint32_t> distribution(0, std::numeric_limits<uint32_t>::max());
seed = distribution(random_device); seed = distribution(random_device);
} }

View File

@ -16,7 +16,9 @@
#include "dataset/util/services.h" #include "dataset/util/services.h"
#include <limits.h> #include <limits.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/syscall.h> #include <sys/syscall.h>
#endif
#include <unistd.h> #include <unistd.h>
#include <random> #include <random>
#include "dataset/util/circular_pool.h" #include "dataset/util/circular_pool.h"
@ -28,6 +30,7 @@ namespace dataset {
std::unique_ptr<Services> Services::instance_ = nullptr; std::unique_ptr<Services> Services::instance_ = nullptr;
std::once_flag Services::init_instance_flag_; std::once_flag Services::init_instance_flag_;
#if !defined(_WIN32) && !defined(_WIN64)
std::string Services::GetUserName() { std::string Services::GetUserName() {
char user[LOGIN_NAME_MAX]; char user[LOGIN_NAME_MAX];
(void)getlogin_r(user, sizeof(user)); (void)getlogin_r(user, sizeof(user));
@ -41,10 +44,15 @@ std::string Services::GetHostName() {
} }
int Services::GetLWP() { return syscall(SYS_gettid); } int Services::GetLWP() { return syscall(SYS_gettid); }
#endif
std::string Services::GetUniqueID() { std::string Services::GetUniqueID() {
const std::string kStr = "abcdefghijklmnopqrstuvwxyz0123456789"; const std::string kStr = "abcdefghijklmnopqrstuvwxyz0123456789";
#if defined(_WIN32) || defined(_WIN64)
std::mt19937 gen{std::random_device{}()};
#else
std::mt19937 gen{std::random_device{"/dev/urandom"}()}; std::mt19937 gen{std::random_device{"/dev/urandom"}()};
#endif
std::uniform_int_distribution<> dist(0, kStr.size() - 1); std::uniform_int_distribution<> dist(0, kStr.size() - 1);
char buffer[UNIQUEID_LEN]; char buffer[UNIQUEID_LEN];
for (int i = 0; i < UNIQUEID_LEN; i++) { for (int i = 0; i < UNIQUEID_LEN; i++) {

View File

@ -62,11 +62,13 @@ class Services {
std::shared_ptr<MemoryPool> GetServiceMemPool() { return pool_; } std::shared_ptr<MemoryPool> GetServiceMemPool() { return pool_; }
#if !defined(_WIN32) && !defined(_WIN64)
static std::string GetUserName(); static std::string GetUserName();
static std::string GetHostName(); static std::string GetHostName();
static int GetLWP(); static int GetLWP();
#endif
static std::string GetUniqueID(); static std::string GetUniqueID();

View File

@ -16,13 +16,16 @@
#include "dataset/util/sig_handler.h" #include "dataset/util/sig_handler.h"
#include <signal.h> #include <signal.h>
#include <sys/types.h> #include <sys/types.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <ucontext.h> #include <ucontext.h>
#endif
#include <unistd.h> #include <unistd.h>
#include "dataset/util/task_manager.h" #include "dataset/util/task_manager.h"
namespace mindspore { namespace mindspore {
namespace dataset { namespace dataset {
// Register the custom signal handlers // Register the custom signal handlers
#if !defined(_WIN32) && !defined(_WIN64)
void RegisterHandlers() { void RegisterHandlers() {
struct sigaction new_int_action; struct sigaction new_int_action;
@ -40,5 +43,6 @@ extern void IntHandler(int sig_num, // The signal that was raised
// Wake up the watchdog which is designed as async-signal-safe. // Wake up the watchdog which is designed as async-signal-safe.
TaskManager::WakeUpWatchDog(); TaskManager::WakeUpWatchDog();
} }
#endif
} // namespace dataset } // namespace dataset
} // namespace mindspore } // namespace mindspore

View File

@ -22,12 +22,14 @@
namespace mindspore { namespace mindspore {
namespace dataset { namespace dataset {
// Register the custom signal handlers // Register the custom signal handlers
#if !defined(_WIN32) && !defined(_WIN64)
extern void RegisterHandlers(); extern void RegisterHandlers();
// A signal handler for SIGINT. Drives interrupt to watchdog // A signal handler for SIGINT. Drives interrupt to watchdog
extern void IntHandler(int sig_num, // The signal that was raised extern void IntHandler(int sig_num, // The signal that was raised
siginfo_t *sig_info, // The siginfo structure. siginfo_t *sig_info, // The siginfo structure.
void *context); // context info void *context); // context info
#endif
} // namespace dataset } // namespace dataset
} // namespace mindspore } // namespace mindspore

View File

@ -14,6 +14,9 @@
* limitations under the License. * limitations under the License.
*/ */
#include "debug/anf_ir_dump.h" #include "debug/anf_ir_dump.h"
#if defined(_WIN32) || defined(_WIN64)
#include <stdlib.h>
#endif
#include <fstream> #include <fstream>
#include <map> #include <map>
#include <memory> #include <memory>
@ -434,9 +437,15 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu
return; return;
} }
char real_path[PATH_MAX] = {0}; char real_path[PATH_MAX] = {0};
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(real_path, filename.c_str(), PATH_MAX) == nullptr) {
MS_LOG(DEBUG) << "dir " << filename << " does not exit.";
}
#else
if (nullptr == realpath(filename.c_str(), real_path)) { if (nullptr == realpath(filename.c_str(), real_path)) {
MS_LOG(DEBUG) << "Dir " << filename << " does not exit."; MS_LOG(DEBUG) << "Dir " << filename << " does not exit.";
} }
#endif
OrderedMap<AnfNodePtr, int32_t> para_map; OrderedMap<AnfNodePtr, int32_t> para_map;
std::string path_string = real_path; std::string path_string = real_path;

View File

@ -48,9 +48,15 @@ std::string GetMsIrPath(void) {
if (path_ptr != nullptr) { if (path_ptr != nullptr) {
path = path_ptr; path = path_ptr;
char real_path[PATH_MAX] = {0}; char real_path[PATH_MAX] = {0};
#if defined(_WIN32) || defined(_WIN64)
if (path.size() > PATH_MAX || _fullpath(real_path, path.c_str(), PATH_MAX) == nullptr) {
MS_LOG(EXCEPTION) << "MS IR Path error, " << path_ptr;
}
#else
if (path.size() > PATH_MAX || nullptr == realpath(path.c_str(), real_path)) { if (path.size() > PATH_MAX || nullptr == realpath(path.c_str(), real_path)) {
MS_LOG(EXCEPTION) << "MS IR path error, " << path_ptr; MS_LOG(EXCEPTION) << "MS IR path error, " << path_ptr;
} }
#endif
path = real_path; path = real_path;
} }
return path; return path;
@ -2247,8 +2253,14 @@ void DumpIRProto(const FuncGraphPtr& func_graph, const std::string& suffix) {
return; return;
} }
char real_path[PATH_MAX] = {0}; char real_path[PATH_MAX] = {0};
if (nullptr == realpath(file_path.c_str(), real_path)) { char* real_path_ret = nullptr;
MS_LOG(DEBUG) << "Dir " << file_path << " does not exit."; #if defined(_WIN32) || defined(_WIN64)
real_path_ret = _fullpath(real_path, file_path.c_str(), PATH_MAX);
#else
real_path_ret = realpath(file_path.c_str(), real_path);
#endif
if (nullptr == real_path_ret) {
MS_LOG(DEBUG) << "dir " << file_path << " does not exit.";
} else { } else {
std::string path_string = real_path; std::string path_string = real_path;
if (chmod(common::SafeCStr(path_string), S_IRUSR | S_IWUSR) == -1) { if (chmod(common::SafeCStr(path_string), S_IRUSR | S_IWUSR) == -1) {

View File

@ -53,10 +53,15 @@ std::string Location::ToString(SourceLineTip tip) {
} }
char path[PATH_MAX + 1] = {0x00}; char path[PATH_MAX + 1] = {0x00};
#if defined(_WIN32) || defined(_WIN64)
if (file_name_.size() > PATH_MAX || _fullpath(path, file_name_.c_str(), PATH_MAX) == nullptr) {
return debug_info_ss.str();
}
#else
if (file_name_.size() > PATH_MAX || realpath(file_name_.c_str(), path) == nullptr) { if (file_name_.size() > PATH_MAX || realpath(file_name_.c_str(), path) == nullptr) {
return debug_info_ss.str(); return debug_info_ss.str();
} }
#endif
auto src_path = std::string(path); auto src_path = std::string(path);
std::ifstream file(src_path); std::ifstream file(src_path);
if (!file.is_open()) { if (!file.is_open()) {

View File

@ -60,6 +60,7 @@ void CPUResourceManager::MemMalloc(const session::KernelGraph *graph) {
void *CPUResourceManager::MemMalloc(size_t mem_size) { void *CPUResourceManager::MemMalloc(size_t mem_size) {
void *ptr = malloc(mem_size); void *ptr = malloc(mem_size);
if (ptr != nullptr) { if (ptr != nullptr) {
memset_s(ptr, mem_size, 0, mem_size);
dynamic_mem_[ptr] = mem_size; dynamic_mem_[ptr] = mem_size;
return ptr; return ptr;
} else { } else {

View File

@ -0,0 +1,28 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "device/gpu/distribution/collective_fake_init.h"
#include "utils/log_adapter.h"
namespace mindspore {
namespace device {
namespace gpu {
void CollectiveFakeInitializer::InitCollective() { MS_LOG(EXCEPTION) << "build without enable gpu!"; }
void CollectiveFakeInitializer::FinalizeCollective() { MS_LOG(EXCEPTION) << "build without enable gpu!"; }
} // namespace gpu
} // namespace device
} // namespace mindspore

View File

@ -0,0 +1,37 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_
#define MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_
namespace mindspore {
namespace device {
namespace gpu {
class CollectiveFakeInitializer {
public:
CollectiveFakeInitializer() = default;
~CollectiveFakeInitializer() = default;
CollectiveFakeInitializer(CollectiveFakeInitializer const &) = delete;
CollectiveFakeInitializer &operator=(const CollectiveFakeInitializer &) = delete;
static void InitCollective();
static void FinalizeCollective();
};
} // namespace gpu
} // namespace device
} // namespace mindspore
#endif // MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_

View File

@ -44,19 +44,29 @@ bool KernelRuntime::Run(session::KernelGraph *graph) {
bool ret = false; bool ret = false;
auto context_ptr = MsContext::GetInstance(); auto context_ptr = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context_ptr); MS_EXCEPTION_IF_NULL(context_ptr);
#if defined(_WIN32) || defined(_WIN64)
auto start_time = std::chrono::steady_clock::now();
#else
struct timeval start_time, end_time; struct timeval start_time, end_time;
(void)gettimeofday(&start_time, nullptr); (void)gettimeofday(&start_time, nullptr);
#endif
bool is_task_sink = context_ptr->enable_task_sink(); bool is_task_sink = context_ptr->enable_task_sink();
if (is_task_sink) { if (is_task_sink) {
ret = RunTask(graph); ret = RunTask(graph);
} else { } else {
ret = LaunchKernel(graph); ret = LaunchKernel(graph);
} }
#if defined(_WIN32) || defined(_WIN64)
auto end_time = std::chrono::steady_clock::now();
std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time;
MS_LOG(INFO) << "Call MS Run Success in " << cost.count() << " us";
#else
(void)gettimeofday(&end_time, nullptr); (void)gettimeofday(&end_time, nullptr);
const uint64_t kUSecondInSecond = 1000000; const uint64_t kUSecondInSecond = 1000000;
uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec); uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec);
cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec); cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec);
MS_LOG(INFO) << "Call MS Run Success in " << cost << " us"; MS_LOG(INFO) << "Call MS Run Success in " << cost << " us";
#endif
return ret; return ret;
} }
@ -561,8 +571,12 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) {
AddressPtrList kernel_workspaces; AddressPtrList kernel_workspaces;
AddressPtrList kernel_outputs; AddressPtrList kernel_outputs;
GenLaunchArgs(*kernel_mod, kernel, &kernel_inputs, &kernel_workspaces, &kernel_outputs); GenLaunchArgs(*kernel_mod, kernel, &kernel_inputs, &kernel_workspaces, &kernel_outputs);
#if defined(_WIN32) || defined(_WIN64)
auto start_time = std::chrono::steady_clock::now();
#else
struct timeval start_time, end_time; struct timeval start_time, end_time;
(void)gettimeofday(&start_time, nullptr); (void)gettimeofday(&start_time, nullptr);
#endif
auto ret = auto ret =
kernel_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, reinterpret_cast<uintptr_t>(stream_)); kernel_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, reinterpret_cast<uintptr_t>(stream_));
if (!ret) { if (!ret) {
@ -572,11 +586,17 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) {
if (AnfAlgo::GetKernelType(kernel) == TBE_KERNEL && !SyncStream()) { if (AnfAlgo::GetKernelType(kernel) == TBE_KERNEL && !SyncStream()) {
MS_LOG(EXCEPTION) << "SyncStream failed."; MS_LOG(EXCEPTION) << "SyncStream failed.";
} }
#if defined(_WIN32) || defined(_WIN64)
auto end_time = std::chrono::steady_clock::now();
std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time;
MS_LOG(DEBUG) << "d " << kernel->fullname_with_scope() << " in " << cost.count() << " us";
#else
(void)gettimeofday(&end_time, nullptr); (void)gettimeofday(&end_time, nullptr);
const uint64_t kUSecondInSecond = 1000000; const uint64_t kUSecondInSecond = 1000000;
uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec); uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec);
cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec); cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec);
MS_LOG(DEBUG) << "d " << kernel->fullname_with_scope() << " in " << cost << " us"; MS_LOG(DEBUG) << "d " << kernel->fullname_with_scope() << " in " << cost << " us";
#endif
} }
} }
return true; return true;

View File

@ -117,7 +117,11 @@ bool IsAtomicNode(const CNodePtr &kernel_node) {
bool KernelMeta::ReadIndex(const std::string &bin_dir) { bool KernelMeta::ReadIndex(const std::string &bin_dir) {
DIR *dir = opendir(bin_dir.c_str()); DIR *dir = opendir(bin_dir.c_str());
if (dir == nullptr) { if (dir == nullptr) {
#if defined(_WIN32) || defined(_WIN64)
auto ret = mkdir(bin_dir.c_str());
#else
auto ret = mkdir(bin_dir.c_str(), S_IRWXG | S_IRWXU); auto ret = mkdir(bin_dir.c_str(), S_IRWXG | S_IRWXU);
#endif
if (ret != 0) { if (ret != 0) {
MS_LOG(INFO) << "kernel dir not exist[" << bin_dir << "]."; MS_LOG(INFO) << "kernel dir not exist[" << bin_dir << "].";
return false; return false;
@ -500,10 +504,17 @@ void SaveJsonInfo(const std::string &json_name, const std::string &info) {
} }
filewrite << info << std::endl; filewrite << info << std::endl;
filewrite.close(); filewrite.close();
#if defined(_WIN32) || defined(_WIN64)
if (nullptr == _fullpath(real_path, path.c_str(), PATH_MAX)) {
MS_LOG(DEBUG) << "dir " << path << " does not exit.";
return;
}
#else
if (nullptr == realpath(path.c_str(), real_path)) { if (nullptr == realpath(path.c_str(), real_path)) {
MS_LOG(DEBUG) << "dir " << path << " does not exit."; MS_LOG(DEBUG) << "dir " << path << " does not exit.";
return; return;
} }
#endif
MS_LOG(INFO) << "real path is :" << real_path; MS_LOG(INFO) << "real path is :" << real_path;
if (chmod(real_path, S_IRUSR) == -1) { if (chmod(real_path, S_IRUSR) == -1) {
MS_LOG(DEBUG) << "modify file:" << real_path << " to read only fail."; MS_LOG(DEBUG) << "modify file:" << real_path << " to read only fail.";

View File

@ -26,7 +26,11 @@ set_target_properties(_c_mindrecord PROPERTIES
) )
# add link library # add link library
target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite ${PYTHON_LIB} ${SECUREC_LIBRARY} mindspore mindspore_gvar mindspore::protobuf) if (WIN32)
target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite mindspore mindspore::protobuf)
else()
target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite ${PYTHON_LIB} ${SECUREC_LIBRARY} mindspore mindspore_gvar mindspore::protobuf)
endif()
if (USE_GLOG) if (USE_GLOG)
target_link_libraries(_c_mindrecord PRIVATE mindspore::glog) target_link_libraries(_c_mindrecord PRIVATE mindspore::glog)

View File

@ -65,6 +65,15 @@ std::pair<MSRStatus, std::string> GetFileName(const std::string &path) {
return {FAILED, ""}; return {FAILED, ""};
} }
char tmp[PATH_MAX] = {0}; char tmp[PATH_MAX] = {0};
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(tmp, dirname(&(buf[0])), PATH_MAX) == nullptr) {
MS_LOG(ERROR) << "Invalid file path, path: " << buf;
return {FAILED, ""};
}
if (_fullpath(real_path, common::SafeCStr(path), PATH_MAX) == nullptr) {
MS_LOG(DEBUG) << "Path: " << common::SafeCStr(path) << "check successfully";
}
#else
if (realpath(dirname(&(buf[0])), tmp) == nullptr) { if (realpath(dirname(&(buf[0])), tmp) == nullptr) {
MS_LOG(ERROR) << "Invalid file path, path: " << buf; MS_LOG(ERROR) << "Invalid file path, path: " << buf;
return {FAILED, ""}; return {FAILED, ""};
@ -72,6 +81,7 @@ std::pair<MSRStatus, std::string> GetFileName(const std::string &path) {
if (realpath(common::SafeCStr(path), real_path) == nullptr) { if (realpath(common::SafeCStr(path), real_path) == nullptr) {
MS_LOG(DEBUG) << "Path: " << path << "check successfully"; MS_LOG(DEBUG) << "Path: " << path << "check successfully";
} }
#endif
std::string s = real_path; std::string s = real_path;
char sep = '/'; char sep = '/';
size_t i = s.rfind(sep, s.length()); size_t i = s.rfind(sep, s.length());
@ -91,6 +101,15 @@ std::pair<MSRStatus, std::string> GetParentDir(const std::string &path) {
return {FAILED, ""}; return {FAILED, ""};
} }
char tmp[PATH_MAX] = {0}; char tmp[PATH_MAX] = {0};
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(tmp, dirname(&(buf[0])), PATH_MAX) == nullptr) {
MS_LOG(ERROR) << "Invalid file path, path: " << buf;
return {FAILED, ""};
}
if (_fullpath(real_path, common::SafeCStr(path), PATH_MAX) == nullptr) {
MS_LOG(DEBUG) << "Path: " << common::SafeCStr(path) << "check successfully";
}
#else
if (realpath(dirname(&(buf[0])), tmp) == nullptr) { if (realpath(dirname(&(buf[0])), tmp) == nullptr) {
MS_LOG(ERROR) << "Invalid file path, path: " << buf; MS_LOG(ERROR) << "Invalid file path, path: " << buf;
return {FAILED, ""}; return {FAILED, ""};
@ -98,6 +117,7 @@ std::pair<MSRStatus, std::string> GetParentDir(const std::string &path) {
if (realpath(common::SafeCStr(path), real_path) == nullptr) { if (realpath(common::SafeCStr(path), real_path) == nullptr) {
MS_LOG(DEBUG) << "Path: " << path << "check successfully"; MS_LOG(DEBUG) << "Path: " << path << "check successfully";
} }
#endif
std::string s = real_path; std::string s = real_path;
if (s.rfind('/') + 1 <= s.size()) { if (s.rfind('/') + 1 <= s.size()) {
return {SUCCESS, s.substr(0, s.rfind('/') + 1)}; return {SUCCESS, s.substr(0, s.rfind('/') + 1)};
@ -144,6 +164,9 @@ bool IsLegalFile(const std::string &path) {
} }
std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type) { std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type) {
#if defined(_WIN32) || defined(_WIN64)
return {SUCCESS, 100};
#else
uint64_t ll_count = 0; uint64_t ll_count = 0;
struct statfs disk_info; struct statfs disk_info;
if (statfs(common::SafeCStr(str_dir), &disk_info) == -1) { if (statfs(common::SafeCStr(str_dir), &disk_info) == -1) {
@ -166,6 +189,7 @@ std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const Dis
} }
return {SUCCESS, ll_count}; return {SUCCESS, ll_count};
#endif
} }
uint32_t GetMaxThreadNum() { uint32_t GetMaxThreadNum() {

View File

@ -21,8 +21,10 @@
#include <limits.h> #include <limits.h>
#include <stdlib.h> #include <stdlib.h>
#include <sys/stat.h> #include <sys/stat.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/statfs.h> #include <sys/statfs.h>
#include <sys/wait.h> #include <sys/wait.h>
#endif
#include <unistd.h> #include <unistd.h>
#include <cassert> #include <cassert>
#include <cmath> #include <cmath>

View File

@ -19,7 +19,9 @@
#include <dirent.h> #include <dirent.h>
#include <signal.h> #include <signal.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/prctl.h> #include <sys/prctl.h>
#endif
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/types.h> #include <sys/types.h>
#include <unistd.h> #include <unistd.h>

View File

@ -987,8 +987,10 @@ TASK_RETURN_CONTENT ShardReader::ConsumerOneTask(int task_id, uint32_t consumer_
MSRStatus ShardReader::ConsumerByRow(int consumer_id) { MSRStatus ShardReader::ConsumerByRow(int consumer_id) {
// Set thread name // Set thread name
#if !defined(_WIN32) && !defined(_WIN64)
auto thread_id = kThreadName + std::to_string(consumer_id); auto thread_id = kThreadName + std::to_string(consumer_id);
prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0); prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0);
#endif
// Loop forever // Loop forever
for (;;) { for (;;) {
@ -1040,8 +1042,10 @@ MSRStatus ShardReader::ReadBlob(const int &shard_id, const uint64_t &page_offset
MSRStatus ShardReader::ConsumerByBlock(int consumer_id) { MSRStatus ShardReader::ConsumerByBlock(int consumer_id) {
// Set thread name // Set thread name
#if !defined(_WIN32) && !defined(_WIN64)
auto thread_id = kThreadName + std::to_string(consumer_id); auto thread_id = kThreadName + std::to_string(consumer_id);
prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0); prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0);
#endif
// Loop forever // Loop forever
for (;;) { for (;;) {

View File

@ -63,6 +63,15 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append)
MS_LOG(ERROR) << "Securec func failed"; MS_LOG(ERROR) << "Securec func failed";
return FAILED; return FAILED;
} }
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(resolved_path, dirname(&(buf[0])), PATH_MAX) == nullptr) {
MS_LOG(ERROR) << "Invalid file path";
return FAILED;
}
if (_fullpath(resolved_path, common::SafeCStr(path), PATH_MAX) == nullptr) {
MS_LOG(DEBUG) << "Path " << resolved_path;
}
#else
if (realpath(dirname(&(buf[0])), resolved_path) == nullptr) { if (realpath(dirname(&(buf[0])), resolved_path) == nullptr) {
MS_LOG(ERROR) << "Invalid file path"; MS_LOG(ERROR) << "Invalid file path";
return FAILED; return FAILED;
@ -70,6 +79,7 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append)
if (realpath(common::SafeCStr(path), resolved_path) == nullptr) { if (realpath(common::SafeCStr(path), resolved_path) == nullptr) {
MS_LOG(DEBUG) << "Path " << resolved_path; MS_LOG(DEBUG) << "Path " << resolved_path;
} }
#endif
file_paths_.emplace_back(string(resolved_path)); file_paths_.emplace_back(string(resolved_path));
} }

View File

@ -38,10 +38,12 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti
(root->has_flag(ALLREDUCE_FUSION_RUN_ONCE_ONLY))) { (root->has_flag(ALLREDUCE_FUSION_RUN_ONCE_ONLY))) {
return changes; return changes;
} }
#if defined(_WIN32) || defined(_WIN64)
auto start_time = std::chrono::steady_clock::now();
#else
struct timeval start_time, end_time; struct timeval start_time, end_time;
(void)gettimeofday(&start_time, nullptr); (void)gettimeofday(&start_time, nullptr);
#endif
MS_LOG(INFO) << "Now entering allreduce fusion"; MS_LOG(INFO) << "Now entering allreduce fusion";
DumpGraph(root, std::string(ALLREDUCE_FUSION_BEGIN)); DumpGraph(root, std::string(ALLREDUCE_FUSION_BEGIN));
@ -63,11 +65,16 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti
// allreduce fusion only run once // allreduce fusion only run once
root->flags()[ALLREDUCE_FUSION_RUN_ONCE_ONLY] = true; root->flags()[ALLREDUCE_FUSION_RUN_ONCE_ONLY] = true;
res->results()[pipeline::kStepParallelGraph] = root; res->results()[pipeline::kStepParallelGraph] = root;
#if defined(_WIN32) || defined(_WIN64)
auto end_time = std::chrono::steady_clock::now();
std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time;
MS_LOG(INFO) << "Now leaving allreduce fusion, used time: " << cost.count() << " us";
#else
(void)gettimeofday(&end_time, nullptr); (void)gettimeofday(&end_time, nullptr);
uint64_t time = 1000000 * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec); uint64_t time = 1000000 * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec);
time += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec); time += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec);
MS_LOG(INFO) << "Now leaving allreduce fusion, used time: " << time << " us"; MS_LOG(INFO) << "Now leaving allreduce fusion, used time: " << time << " us";
#endif
return changes; return changes;
} }
} // namespace parallel } // namespace parallel

View File

@ -227,7 +227,7 @@ size_t GetLengthOfDataType(const TypePtr &type) {
case kNumberTypeInt: case kNumberTypeInt:
return sizeof(int); return sizeof(int);
case kNumberTypeUInt: case kNumberTypeUInt:
return sizeof(uint); return sizeof(unsigned int);
case kNumberTypeFloat: case kNumberTypeFloat:
return sizeof(float); return sizeof(float);
default: default:

View File

@ -264,7 +264,7 @@ bool TaskEmitAction(const ResourcePtr& res) {
auto bc_ptr = res->results()[kBackend].cast<compile::BackendPtr>(); auto bc_ptr = res->results()[kBackend].cast<compile::BackendPtr>();
std::vector<PrimitivePtr> cut_list = compile::nonlinear_ops; std::vector<PrimitivePtr> cut_list = compile::nonlinear_ops;
if (bc_ptr->name() == kMsConvert) { if (bc_ptr->name() == kMsConvert) {
cut_list = compile::ms_nonlinear_ops; cut_list = compile::GetMsNonlinearOps();
} }
std::shared_ptr<CompileGraphs> compile = std::make_shared<CompileGraphs>(bc_ptr, cut_list); std::shared_ptr<CompileGraphs> compile = std::make_shared<CompileGraphs>(bc_ptr, cut_list);
res->results()[kOutput] = compile->CompileAndLink(func_graph); res->results()[kOutput] = compile->CompileAndLink(func_graph);

View File

@ -29,8 +29,11 @@
#include "parallel/context.h" #include "parallel/context.h"
#include "parallel/device_manager.h" #include "parallel/device_manager.h"
#include "parallel/costmodel_context.h" #include "parallel/costmodel_context.h"
#ifdef ENABLE_GPUQUE
#include "device/gpu/distribution/collective_init.h" #include "device/gpu/distribution/collective_init.h"
#else
#include "device/gpu/distribution/collective_fake_init.h"
#endif
namespace py = pybind11; namespace py = pybind11;
using FuncGraph = mindspore::FuncGraph; using FuncGraph = mindspore::FuncGraph;
@ -297,9 +300,16 @@ PYBIND11_MODULE(_c_expression, m) {
(void)py::class_<OpLib, std::shared_ptr<OpLib>>(m, "Oplib") (void)py::class_<OpLib, std::shared_ptr<OpLib>>(m, "Oplib")
.def(py::init()) .def(py::init())
.def("reg_op", &OpLib::RegOp, "Register op info."); .def("reg_op", &OpLib::RegOp, "Register op info.");
#ifdef ENABLE_GPUQUE
(void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::InitCollective, (void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::InitCollective,
"Init gpu collective communication mode."); "Init gpu collective communication mode.");
(void)m.def("finalize_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::FinalizeCollective, (void)m.def("finalize_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::FinalizeCollective,
"Finalize gpu collective communication mode."); "Finalize gpu collective communication mode.");
#else
(void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveFakeInitializer::InitCollective,
"Init gpu collective communication mode.");
(void)m.def("finalize_gpu_collective", &mindspore::device::gpu::CollectiveFakeInitializer::FinalizeCollective,
"Finalize gpu collective communication mode.");
#endif
} }

View File

@ -52,17 +52,27 @@ bool PassManager::Run(const FuncGraphPtr &func_graph, const std::vector<PassPtr>
size_t num = 0; size_t num = 0;
for (const auto &pass : passes) { for (const auto &pass : passes) {
if (pass != nullptr) { if (pass != nullptr) {
#if defined(_WIN32) || defined(_WIN64)
auto start_time = std::chrono::steady_clock::now();
#else
struct timeval start_time {}; struct timeval start_time {};
struct timeval end_time {}; struct timeval end_time {};
(void)gettimeofday(&start_time, nullptr); (void)gettimeofday(&start_time, nullptr);
#endif
if (pass->Run(func_graph)) { if (pass->Run(func_graph)) {
changed = true; changed = true;
} }
#if defined(_WIN32) || defined(_WIN64)
auto end_time = std::chrono::steady_clock::now();
std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time;
MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost.count() << " us";
#else
(void)gettimeofday(&end_time, nullptr); (void)gettimeofday(&end_time, nullptr);
const uint64_t kUSecondInSecond = 1000000; const uint64_t kUSecondInSecond = 1000000;
uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec); uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec);
cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec); cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec);
MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost << " us"; MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost << " us";
#endif
if (save_graphs) { if (save_graphs) {
auto dump_file_path = auto dump_file_path =
save_graphs_path + "/" + "hwopt_" + name() + "_" + std::to_string(num) + "_" + pass->name() + ".ir"; save_graphs_path + "/" + "hwopt_" + name() + "_" + std::to_string(num) + "_" + pass->name() + ".ir";

View File

@ -26,12 +26,19 @@ namespace mindspore {
#ifdef USE_GLOG #ifdef USE_GLOG
static std::string GetTime() { static std::string GetTime() {
#define BUFLEN 80 #define BUFLEN 80
static char buf[BUFLEN];
#if defined(_WIN32) || defined(_WIN64)
time_t time_seconds = time(0);
struct tm now_time;
localtime_s(&now_time, &time_seconds);
sprintf_s(buf, BUFLEN, "%d-%d-%d %d:%d:%d", now_time.tm_year + 1900, now_time.tm_mon + 1, now_time.tm_mday,
now_time.tm_hour, now_time.tm_min, now_time.tm_sec);
#else
struct timeval cur_time; struct timeval cur_time;
(void)gettimeofday(&cur_time, NULL); (void)gettimeofday(&cur_time, NULL);
struct tm now; struct tm now;
(void)localtime_r(&cur_time.tv_sec, &now); (void)localtime_r(&cur_time.tv_sec, &now);
static char buf[BUFLEN];
(void)strftime(buf, BUFLEN, "%Y-%m-%d-%H:%M:%S", &now); // format date and time (void)strftime(buf, BUFLEN, "%Y-%m-%d-%H:%M:%S", &now); // format date and time
// set micro-second // set micro-second
buf[27] = '\0'; buf[27] = '\0';
@ -44,6 +51,7 @@ static std::string GetTime() {
buf[idx--] = '.'; buf[idx--] = '.';
} }
} }
#endif
return std::string(buf); return std::string(buf);
} }

View File

@ -41,8 +41,10 @@ using TypedPrimitiveAbstractClosurePtr = std::shared_ptr<abstract::TypedPrimitiv
std::vector<PrimitivePtr> nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch, std::vector<PrimitivePtr> nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch,
prim::kPrimMakeTuple}; prim::kPrimMakeTuple};
const std::vector<PrimitivePtr>& GetMsNonlinearOps() {
std::vector<PrimitivePtr> ms_nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch}; static const std::vector<PrimitivePtr> ms_nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch};
return ms_nonlinear_ops;
}
CompileGraph::CompileGraph(const BackendPtr& backend, const std::vector<PrimitivePtr>& cut_list) CompileGraph::CompileGraph(const BackendPtr& backend, const std::vector<PrimitivePtr>& cut_list)
: backend_(backend), cut_list_(cut_list) { : backend_(backend), cut_list_(cut_list) {

View File

@ -42,7 +42,7 @@ extern const char kGeVm[];
// A sub namespace in ME to support compile related definition. // A sub namespace in ME to support compile related definition.
namespace compile { namespace compile {
extern std::vector<PrimitivePtr> nonlinear_ops; extern std::vector<PrimitivePtr> nonlinear_ops;
extern std::vector<PrimitivePtr> ms_nonlinear_ops; const std::vector<PrimitivePtr>& GetMsNonlinearOps();
using VmEvalFunc = std::function<BaseRef(const VectorRef&)>; using VmEvalFunc = std::function<BaseRef(const VectorRef&)>;
using VmEvalFuncPtr = std::shared_ptr<std::function<BaseRef(const VectorRef&)>>; using VmEvalFuncPtr = std::shared_ptr<std::function<BaseRef(const VectorRef&)>>;

View File

@ -19,11 +19,13 @@ import sys
import os import os
import stat import stat
import time import time
import fcntl
import logging import logging
from logging.handlers import RotatingFileHandler from logging.handlers import RotatingFileHandler
import traceback import traceback
import threading import threading
import platform
if platform.system() != "Windows":
import fcntl
__all__ = ['get_level', 'get_log_config'] __all__ = ['get_level', 'get_log_config']
@ -90,7 +92,8 @@ class _MultiCompatibleRotatingFileHandler(RotatingFileHandler):
# Attain an exclusive lock with bloking mode by `fcntl` module. # Attain an exclusive lock with bloking mode by `fcntl` module.
with open(self.baseFilename, 'a') as file_pointer: with open(self.baseFilename, 'a') as file_pointer:
fcntl.lockf(file_pointer.fileno(), fcntl.LOCK_EX) if platform.system() != "Windows":
fcntl.lockf(file_pointer.fileno(), fcntl.LOCK_EX)
if self.backupCount > 0: if self.backupCount > 0:
self.rolling_rename() self.rolling_rename()

View File

@ -1,10 +1,18 @@
SET(CMAKE_BUILD_TYPE "Debug") SET(CMAKE_BUILD_TYPE "Debug")
SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='") if (WIN32)
SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer")
else()
SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='")
endif()
SET(CMAKE_C_FLAGS_RELEASE "$ENV{CFLAGS} -fPIC -O3 -Wall -Wno-deprecated-declarations") SET(CMAKE_C_FLAGS_RELEASE "$ENV{CFLAGS} -fPIC -O3 -Wall -Wno-deprecated-declarations")
set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
#add flags #add flags
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include -Werror") if (WIN32)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include")
else()
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include -Werror")
endif()
include_directories(./include) include_directories(./include)