From 1286767d0e083ad5ac6a9f50a56451b843463805 Mon Sep 17 00:00:00 2001 From: chenjianping Date: Wed, 15 Apr 2020 09:36:48 +0000 Subject: [PATCH] support building on windows --- CMakeLists.txt | 13 +++ build.bat | 36 ++++++++ cmake/external_libs/flatbuffers.cmake | 3 + cmake/external_libs/mkl_dnn.cmake | 13 ++- cmake/external_libs/opencv.cmake | 89 +++++++++++++------ cmake/external_libs/protobuf.cmake | 44 +++++---- cmake/external_libs/sqlite.cmake | 41 ++++++--- cmake/mind_expression.cmake | 6 +- cmake/options.cmake | 8 +- cmake/utils.cmake | 50 +++++++---- mindspore/ccsrc/CMakeLists.txt | 62 ++++++++++--- mindspore/ccsrc/dataset/CMakeLists.txt | 18 +++- .../ccsrc/dataset/api/python_bindings.cc | 4 + mindspore/ccsrc/dataset/core/tensor.h | 4 + .../dataset/engine/datasetops/shuffle_op.cc | 4 + .../dataset/kernels/image/CMakeLists.txt | 68 +++++++++----- .../dataset/kernels/image/image_utils.cc | 4 + .../ccsrc/dataset/kernels/image/image_utils.h | 4 + mindspore/ccsrc/dataset/util/path.cc | 6 +- mindspore/ccsrc/dataset/util/random.cc | 4 + mindspore/ccsrc/dataset/util/services.cc | 8 ++ mindspore/ccsrc/dataset/util/services.h | 2 + mindspore/ccsrc/dataset/util/sig_handler.cc | 4 + mindspore/ccsrc/dataset/util/sig_handler.h | 2 + mindspore/ccsrc/debug/anf_ir_dump.cc | 9 ++ mindspore/ccsrc/debug/anf_ir_utils.cc | 16 +++- mindspore/ccsrc/debug/info.cc | 7 +- .../ccsrc/device/cpu/cpu_resource_manager.cc | 1 + .../gpu/distribution/collective_fake_init.cc | 28 ++++++ .../gpu/distribution/collective_fake_init.h | 37 ++++++++ mindspore/ccsrc/device/kernel_runtime.cc | 20 +++++ mindspore/ccsrc/kernel/common_utils.cc | 11 +++ mindspore/ccsrc/mindrecord/CMakeLists.txt | 6 +- .../ccsrc/mindrecord/common/shard_utils.cc | 24 +++++ .../mindrecord/include/common/shard_utils.h | 2 + .../ccsrc/mindrecord/include/shard_reader.h | 2 + mindspore/ccsrc/mindrecord/io/shard_reader.cc | 4 + mindspore/ccsrc/mindrecord/io/shard_writer.cc | 10 +++ .../allreduce_fusion/step_allreduce_fusion.cc | 13 ++- .../ccsrc/parallel/step_auto_parallel.cc | 2 +- mindspore/ccsrc/pipeline/action.cc | 2 +- mindspore/ccsrc/pipeline/init.cc | 14 ++- .../ccsrc/pre_activate/common/pass_manager.cc | 10 +++ mindspore/ccsrc/utils/log_adapter.cc | 10 ++- mindspore/ccsrc/vm/transform.cc | 6 +- mindspore/ccsrc/vm/transform.h | 2 +- mindspore/log.py | 7 +- third_party/securec/CMakeLists.txt | 12 ++- 48 files changed, 619 insertions(+), 133 deletions(-) create mode 100644 build.bat create mode 100644 mindspore/ccsrc/device/gpu/distribution/collective_fake_init.cc create mode 100644 mindspore/ccsrc/device/gpu/distribution/collective_fake_init.h diff --git a/CMakeLists.txt b/CMakeLists.txt index a02fcd1cafa..9c6ee73687a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,7 @@ else() endif() set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -Wl,--allow-shlib-undefined -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)=' -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/include -std=c++17 -Werror -Wall -Wno-deprecated-declarations -fPIC") set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -29,6 +30,18 @@ find_package(Python3 3.7 COMPONENTS Interpreter Development) if(Python3_FOUND) set(PYTHON_INCLUDE_DIRS "${Python3_INCLUDE_DIRS}") set(PYTHON_LIBRARIES "${Python3_LIBRARIES}") + if (WIN32) + if (Python3_DIR) + message("Python3_DIR set already: " ${Python3_DIR}) + else() + string(LENGTH ${PYTHON_LIBRARIES} PYTHON_LIBRARIES_LEN) + string(LENGTH "libpythonxx.a" Python3_NAME_LEN) + math(EXPR Python3_DIR_LEN ${PYTHON_LIBRARIES_LEN}-${Python3_NAME_LEN}) + string(SUBSTRING ${Python3_LIBRARIES} 0 ${Python3_DIR_LEN} Python3_DIR) + message("Python3_DIR: " ${Python3_DIR}) + endif() + link_directories(${Python3_DIR}) + endif() else() find_python_package(py_inc py_lib) set(PYTHON_INCLUDE_DIRS "${py_inc}") diff --git a/build.bat b/build.bat new file mode 100644 index 00000000000..b4f3420e140 --- /dev/null +++ b/build.bat @@ -0,0 +1,36 @@ +@echo off +@title mindspore_build + +SET BASEPATH=%CD% +IF NOT EXIST %BASEPATH%/build ( + md "build" + ) + +cd %BASEPATH%/build +SET BUILD_PATH=%CD% + +IF NOT EXIST %BUILD_PATH%/mindspore ( + md "mindspore" + ) + +cd %CD%/mindspore + +cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_CPU=ON -DENABLE_MINDDATA=ON -DUSE_GLOG=ON -G "CodeBlocks - MinGW Makefiles" ../.. +IF NOT %errorlevel% == 0 ( + goto run_fail + ) + +cmake --build . --target all -- -j6 +IF NOT %errorlevel% == 0 ( + goto run_fail + ) + +cd %BASEPATH% + +goto run_eof + +:run_fail + cd %BASEPATH% + echo "build fail." + +:run_eof diff --git a/cmake/external_libs/flatbuffers.cmake b/cmake/external_libs/flatbuffers.cmake index 7d7c74b9e16..18549ed1b5d 100644 --- a/cmake/external_libs/flatbuffers.cmake +++ b/cmake/external_libs/flatbuffers.cmake @@ -1,5 +1,8 @@ set(flatbuffers_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") set(flatbuffers_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") +if (WIN32) + set(flatbuffers_USE_STATIC_LIBS ON) +endif() mindspore_add_pkg(flatbuffers VER 1.11.0 LIBS flatbuffers diff --git a/cmake/external_libs/mkl_dnn.cmake b/cmake/external_libs/mkl_dnn.cmake index 6f033fa5655..4b2c46670aa 100644 --- a/cmake/external_libs/mkl_dnn.cmake +++ b/cmake/external_libs/mkl_dnn.cmake @@ -1,11 +1,22 @@ set(onednn_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") set(onednn_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") -mindspore_add_pkg(onednn +if (CMAKE_SYSTEM_NAME MATCHES "Windows") + mindspore_add_pkg(onednn + VER 1.1.1 + LIBS dnnl mkldnn + HEAD_ONLY ./ + RELEASE on + URL https://github.com/oneapi-src/oneDNN/releases/download/v1.1.1/dnnl_win_1.1.1_cpu_vcomp.zip + MD5 ecaab9ed549643067699c80e5cea1c23) +else() + mindspore_add_pkg(onednn VER 1.1.2 LIBS dnnl mkldnn URL https://github.com/oneapi-src/oneDNN/archive/v1.1.2.tar.gz MD5 ab40d52230f3ad1d7a6f06ce0f6bc17a CMAKE_OPTION -DDNNL_ARCH_OPT_FLAGS='' -DDNNL_CPU_RUNTIME='SEQ' -DDNNL_BUILD_EXAMPLES=OFF -DDNNL_BUILD_TESTS=OFF) +endif() + include_directories(${onednn_INC}) add_library(mindspore::dnnl ALIAS onednn::dnnl) add_library(mindspore::mkldnn ALIAS onednn::mkldnn) diff --git a/cmake/external_libs/opencv.cmake b/cmake/external_libs/opencv.cmake index 5c60a2fa617..1c40769f627 100644 --- a/cmake/external_libs/opencv.cmake +++ b/cmake/external_libs/opencv.cmake @@ -2,35 +2,72 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(opencv_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") set(opencv_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") set(opencv_LDFLAGS "-Wl") +elseif (${CMAKE_SYSTEM_NAME} MATCHES "Windows") + set(opencv_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") + set(opencv_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") else() set(opencv_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") set(opencv_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") set(opencv_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") endif() -mindspore_add_pkg(opencv - VER 4.2.0 - LIBS opencv_core opencv_imgcodecs opencv_imgproc - URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz - MD5 e8cb208ce2723481408b604b480183b6 - CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF - -DBUILD_ZLIB=ON - -DBUILD_JPEG=ON - -DBUILD_PNG=ON - -DBUILD_OPENEXR=ON - -DBUILD_TESTS=OFF - -DBUILD_PERF_TESTS=OFF - -DBUILD_opencv_apps=OFF - -DCMAKE_SKIP_RPATH=TRUE - -DBUILD_opencv_python3=OFF - -DWITH_FFMPEG=OFF - -DWITH_TIFF=ON - -DBUILD_TIFF=OFF - -DWITH_JASPER=OFF - -DBUILD_JASPER=OFF - -DTIFF_INCLUDE_DIR=${tiff_INC} - -DTIFF_LIBRARY=${tiff_LIB}) -include_directories(${opencv_INC}/opencv4) -add_library(mindspore::opencv_core ALIAS opencv::opencv_core) -add_library(mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs) -add_library(mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc) +if (WIN32) + mindspore_add_pkg(opencv + VER 4.2.0 + LIBS libopencv_core420.dll.a libopencv_imgcodecs420.dll.a libopencv_imgproc420.dll.a + LIB_PATH x64/mingw/lib + URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz + MD5 e8cb208ce2723481408b604b480183b6 + CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF + -DBUILD_ZLIB=ON + -DBUILD_JPEG=ON + -DBUILD_PNG=ON + -DBUILD_OPENEXR=ON + -DBUILD_TESTS=OFF + -DBUILD_PERF_TESTS=OFF + -DBUILD_opencv_apps=OFF + -DCMAKE_SKIP_RPATH=TRUE + -DBUILD_opencv_python3=OFF + -DWITH_FFMPEG=OFF + -DWITH_TIFF=ON + -DBUILD_TIFF=OFF + -DWITH_JASPER=OFF + -DBUILD_JASPER=OFF + -DTIFF_INCLUDE_DIR=${tiff_INC} + -DTIFF_LIBRARY=${tiff_LIB}) +else() + mindspore_add_pkg(opencv + VER 4.2.0 + LIBS opencv_core opencv_imgcodecs opencv_imgproc + URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz + MD5 e8cb208ce2723481408b604b480183b6 + CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF + -DBUILD_ZLIB=ON + -DBUILD_JPEG=ON + -DBUILD_PNG=ON + -DBUILD_OPENEXR=ON + -DBUILD_TESTS=OFF + -DBUILD_PERF_TESTS=OFF + -DBUILD_opencv_apps=OFF + -DCMAKE_SKIP_RPATH=TRUE + -DBUILD_opencv_python3=OFF + -DWITH_FFMPEG=OFF + -DWITH_TIFF=ON + -DBUILD_TIFF=OFF + -DWITH_JASPER=OFF + -DBUILD_JASPER=OFF + -DTIFF_INCLUDE_DIR=${tiff_INC} + -DTIFF_LIBRARY=${tiff_LIB}) +endif() + +if (WIN32) + include_directories(${opencv_INC}) + add_library(mindspore::opencv_core ALIAS opencv::libopencv_core420.dll.a) + add_library(mindspore::opencv_imgcodecs ALIAS opencv::libopencv_imgcodecs420.dll.a) + add_library(mindspore::opencv_imgproc ALIAS opencv::libopencv_imgproc420.dll.a) +else() + include_directories(${opencv_INC}/opencv4) + add_library(mindspore::opencv_core ALIAS opencv::opencv_core) + add_library(mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs) + add_library(mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc) +endif() diff --git a/cmake/external_libs/protobuf.cmake b/cmake/external_libs/protobuf.cmake index 987d5c6a8b0..a574e789db8 100644 --- a/cmake/external_libs/protobuf.cmake +++ b/cmake/external_libs/protobuf.cmake @@ -77,22 +77,36 @@ function(ms_protobuf_generate_py c_var h_var py_var) list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc") list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h") list(APPEND ${py_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py") - - add_custom_command( - OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc" - "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h" - "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} - COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}" - COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} - COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} - COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} - COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" - COMMAND cp "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/" - DEPENDS protobuf::protoc ${abs_file} - COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM ) + if (WIN32) + add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc" + "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h" + "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}" + COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} + COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} + COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} + COMMAND perl -pi.bak -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/" + DEPENDS protobuf::protoc ${abs_file} + COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM ) + else() + add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc" + "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h" + "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}" + COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} + COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} + COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} + COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" + COMMAND cp "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/" + DEPENDS protobuf::protoc ${abs_file} + COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM ) + endif() endforeach() - set_source_files_properties(${${c_var}} ${${h_var}} ${${py_var}} PROPERTIES GENERATED TRUE) set(${c_var} ${${c_var}} PARENT_SCOPE) set(${h_var} ${${h_var}} PARENT_SCOPE) diff --git a/cmake/external_libs/sqlite.cmake b/cmake/external_libs/sqlite.cmake index e3fe77d96b0..69692f284d9 100644 --- a/cmake/external_libs/sqlite.cmake +++ b/cmake/external_libs/sqlite.cmake @@ -1,19 +1,36 @@ +if (WIN32) + mindspore_add_pkg(sqlite-head + VER 3.31.1 + HEAD_ONLY ./ + URL https://sqlite.org/2020/sqlite-amalgamation-3310100.zip + MD5 2b7bfcdd97dc281903a9aee966213fe4) + include_directories(${sqlite-head_INC}) + mindspore_add_pkg(sqlite + VER 3.31.1 + LIBS sqlite3 + LIB_PATH ./ + HEAD_ONLY ./ + RELEASE ON + URL https://sqlite.org/2020/sqlite-dll-win64-x64-3310100.zip + MD5 662c9d2b05467d590ba5c0443e7fd6bd) -set(sqlite_USE_STATIC_LIBS ON) -set(sqlite_CXXFLAGS) -if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(sqlite_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") -else() - set(sqlite_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") -endif() -set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") - -mindspore_add_pkg(sqlite +else () + set(sqlite_USE_STATIC_LIBS ON) + set(sqlite_CXXFLAGS) + if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(sqlite_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") + else() + set(sqlite_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") + endif() + set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") + mindspore_add_pkg(sqlite VER 3.31.1 LIBS sqlite3 URL https://github.com/sqlite/sqlite/archive/version-3.31.1.tar.gz MD5 5f4e7b4016c15f4fb5855615279819da PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/sqlite/sqlite.patch001 CONFIGURE_COMMAND ./configure --enable-shared=no --disable-tcl --disable-editline --enable-json1) -include_directories(${sqlite_INC}) -add_library(mindspore::sqlite ALIAS sqlite::sqlite3) \ No newline at end of file + include_directories(${sqlite_INC}) +endif () + +add_library(mindspore::sqlite ALIAS sqlite::sqlite3) diff --git a/cmake/mind_expression.cmake b/cmake/mind_expression.cmake index af122d41174..7e5e07bdbb8 100644 --- a/cmake/mind_expression.cmake +++ b/cmake/mind_expression.cmake @@ -1,6 +1,10 @@ set(SECURE_CXX_FLAGS "") if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - set(SECURE_CXX_FLAGS "-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") + if (WIN32) + set(SECURE_CXX_FLAGS "-fstack-protector-all") + else() + set(SECURE_CXX_FLAGS "-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") + endif() endif() set(_ms_tmp_CMAKE_CXX_FLAGS_F ${CMAKE_CXX_FLAGS}) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") diff --git a/cmake/options.cmake b/cmake/options.cmake index 6ec577f3120..2f44e9a17ca 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -19,7 +19,11 @@ option(ENABLE_MPI "enable mpi" OFF) option(ENABLE_AKG "enable akg" OFF) if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") + if (WIN32) + set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all") + else() + set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") + endif() endif() if (CMAKE_SYSTEM_NAME MATCHES "Darwin") @@ -106,4 +110,4 @@ endif() if(ENABLE_DUMP_E2E) add_compile_definitions(ENABLE_DUMP_E2E) -endif() \ No newline at end of file +endif() diff --git a/cmake/utils.cmake b/cmake/utils.cmake index c9b7d944f03..f0181de5ea8 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -103,7 +103,7 @@ function(__download_pkg_with_git pkg_name pkg_url pkg_git_commit pkg_md5) endfunction() -function(__find_pkg_then_add_target pkg_name pkg_exe) +function(__find_pkg_then_add_target pkg_name pkg_exe lib_path) unset(${pkg_name}_LIBS) @@ -129,15 +129,24 @@ function(__find_pkg_then_add_target pkg_name pkg_exe) set(_LIB_TYPE STATIC) endif () set(${_LIB_NAME}_LIB ${_LIB_NAME}_LIB-NOTFOUND) - find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/lib NO_DEFAULT_PATH) + find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/${lib_path} NO_DEFAULT_PATH) + if(NOT ${_LIB_NAME}_LIB) return() endif() + add_library(${pkg_name}::${_LIB_NAME} ${_LIB_TYPE} IMPORTED GLOBAL) - set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${${pkg_name}_BASE_DIR}/include" - IMPORTED_LOCATION ${${_LIB_NAME}_LIB} - ) + if (WIN32 AND ${_LIB_TYPE} STREQUAL "SHARED") + set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES IMPORTED_IMPLIB_RELEASE ${${_LIB_NAME}_LIB}) + else() + set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES IMPORTED_LOCATION ${${_LIB_NAME}_LIB}) + endif() + + if (EXISTS ${${pkg_name}_BASE_DIR}/include) + set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${${pkg_name}_BASE_DIR}/include") + endif () + list(APPEND ${pkg_name}_LIBS ${pkg_name}::${_LIB_NAME}) message("found ${${_LIB_NAME}_LIB}") STRING( REGEX REPLACE "(.+)/(.+)" "\\1" LIBPATH ${${_LIB_NAME}_LIB}) @@ -192,12 +201,18 @@ set(MS_FIND_NO_DEFAULT_PATH ${MS_FIND_NO_DEFAULT_PATH} PARENT_SCOPE) function(mindspore_add_pkg pkg_name ) set(options ) - set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH) + set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH RELEASE LIB_PATH) set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) - if (NOT PKG_CMAKE_PATH) - set(PKG_CMAKE_PATH .) + + if (NOT PKG_LIB_PATH) + set(PKG_LIB_PATH lib) endif () + + if(NOT PKG_EXE) + set(PKG_EXE 0) + endif() + set(__FIND_PKG_NAME ${pkg_name}) string(TOLOWER ${pkg_name} pkg_name) message("pkg name:${__FIND_PKG_NAME},${pkg_name}") @@ -225,18 +240,17 @@ function(mindspore_add_pkg pkg_name ) set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE) add_library(${pkg_name} INTERFACE) target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC}) + if (${PKG_RELEASE}) + __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS}) + endif () return() endif () - if(NOT PKG_EXE) - set(PKG_EXE 0) - endif() - set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR}) set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR} PARENT_SCOPE) if (PKG_LIBS) - __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS}) + __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS}) if(${pkg_name}_LIBS) set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE) message("Found libs: ${${pkg_name}_LIBS}") @@ -283,8 +297,10 @@ function(mindspore_add_pkg pkg_name ) file(GLOB ${pkg_name}_SOURCE_SUBDIRS ${${pkg_name}_SOURCE_DIR}/*) file(COPY ${${pkg_name}_SOURCE_SUBDIRS} DESTINATION ${${pkg_name}_BASE_DIR}) set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE) - add_library(${pkg_name} INTERFACE) - target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC}) + if (NOT PKG_RELEASE) + add_library(${pkg_name} INTERFACE) + target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC}) + endif () elseif (PKG_CMAKE_OPTION) # in cmake @@ -355,7 +371,7 @@ function(mindspore_add_pkg pkg_name ) endif() if (PKG_LIBS) - __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS}) + __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS}) set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE) if(NOT ${pkg_name}_LIBS) message(FATAL_ERROR "Can not find pkg: ${pkg_name}") diff --git a/mindspore/ccsrc/CMakeLists.txt b/mindspore/ccsrc/CMakeLists.txt index d4103f30c0c..500972b0d66 100644 --- a/mindspore/ccsrc/CMakeLists.txt +++ b/mindspore/ccsrc/CMakeLists.txt @@ -5,6 +5,10 @@ if(ENABLE_CPU) file(GLOB_RECURSE CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "device/cpu/*.cc" ) + if (CMAKE_SYSTEM_NAME MATCHES "Windows") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF") + add_compile_definitions(BUILDING_DLL) + endif() endif() if(ENABLE_GPU) @@ -150,7 +154,15 @@ file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "kernel/kash/*.cc" "device/gpu/distribution/collective_init.cc" ) - +if (ENABLE_CPU) + list(REMOVE_ITEM MINDSPORE_SRC_LIST "device/gpu/distribution/collective_init.cc") + if (WIN32) + list(REMOVE_ITEM MINDSPORE_SRC_LIST "kernel/kernel_query.cc") + endif() +endif() +if (NOT ENABLE_GPU) + list(APPEND MINDSPORE_SRC_LIST "device/gpu/distribution/collective_fake_init.cc") +endif() file(GLOB_RECURSE MEM_REUSE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "pre_activate/mem_reuse/*.cc" ) @@ -248,6 +260,7 @@ file(GLOB_RECURSE MS_GVAR_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} add_library(mindspore_gvar SHARED ${MS_GVAR_SRC_LIST}) add_library(mindspore STATIC ${MINDSPORE_SRC_LIST}) +add_dependencies(mindspore GENERATED_OUTPUT_DIR) if(ENABLE_D) list(APPEND MINDSPORE_PROTO_LIST ${MINDSPORE_PROTO_AICPU_LIST}) @@ -315,19 +328,33 @@ if(ENABLE_D) endif() target_link_libraries(mindspore securec) -target_link_libraries(mindspore dl) +if (NOT WIN32) + target_link_libraries(mindspore dl) +endif() target_link_libraries(mindspore mindspore::flatbuffers) # link protobuf if (ENABLE_D) target_link_libraries(mindspore mindspore::protobuf) endif() +if (WIN32) + target_link_libraries(mindspore ${PYTHON_LIBRARIES} mindspore_gvar) +endif() + # set c_expression building +if (WIN32) +set(PYTHON_MODULE_SOURCE ${MS_GVAR_SRC_LIST} + pipeline/init.cc + kernel/oplib/oplib.cc + ${MINDSPORE_SRC_LIST} ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST} + ${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST}) +else() set(PYTHON_MODULE_SOURCE pipeline/init.cc kernel/oplib/oplib.cc - ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST} - ${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST}) + ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST} + ${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST}) +endif() set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) pybind11_add_module(_c_expression ${PYTHON_MODULE_SOURCE}) @@ -339,6 +366,8 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux") elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") set_target_properties(_c_expression PROPERTIES MACOSX_RPATH ON) set(ORIGIN_PATH @loader_path) +elseif (CMAKE_SYSTEM_NAME MATCHES "Windows") + set(ORIGIN_PATH $ORIGIN) else () MESSAGE(FATAL_ERROR "other platform: ${CMAKE_SYSTEM_NAME}") endif () @@ -346,11 +375,22 @@ endif () set(ORIGIN_PATH ${ORIGIN_PATH}/lib) set_target_properties(_c_expression PROPERTIES INSTALL_RPATH ${ORIGIN_PATH}) -target_link_libraries(_c_expression PRIVATE - mindspore::pybind11_module - mindspore - mindspore_gvar - ) +if (WIN32) + target_link_libraries(_c_expression PRIVATE + mindspore::pybind11_module + securec + proto_input + mindspore::flatbuffers + mindspore::glog + ) +else() + target_link_libraries(_c_expression PRIVATE + mindspore::pybind11_module + mindspore + mindspore_gvar + mindspore::glog + ) +endif() if(ENABLE_GPU) execute_process(COMMAND bash ${CMAKE_SOURCE_DIR}/third_party/apply_patches.sh @@ -493,7 +533,3 @@ if(ENABLE_MINDDATA) add_subdirectory(mindrecord) add_subdirectory(dataset) endif() - -if (USE_GLOG) - target_link_libraries(_c_expression PRIVATE mindspore::glog) -endif() \ No newline at end of file diff --git a/mindspore/ccsrc/dataset/CMakeLists.txt b/mindspore/ccsrc/dataset/CMakeLists.txt index 52ba3288281..b3ac34de70f 100644 --- a/mindspore/ccsrc/dataset/CMakeLists.txt +++ b/mindspore/ccsrc/dataset/CMakeLists.txt @@ -80,14 +80,19 @@ set_target_properties(_c_dataengine PROPERTIES ###################################################################### ################# Link with external libraries ######################## -target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar) -target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module -ldl mindspore::protobuf ${SECUREC_LIBRARY}) +if (WIN32) + target_link_libraries(_c_dataengine PRIVATE mindspore) + target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module ${PYTHON_LIBRARIES} mindspore::protobuf ${SECUREC_LIBRARY}) +else() + target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar) + target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module -ldl mindspore::protobuf ${SECUREC_LIBRARY}) +endif() target_link_libraries(_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::opencv_core mindspore::opencv_imgcodecs mindspore::opencv_imgproc) if (ENABLE_GPUQUE) target_link_libraries(_c_dataengine PRIVATE gpu_queue ${CUDNN_PATH}/lib64/libcudnn.so - ${CUDA_PATH}/lib64/libcudart.so + ${CUDA_PATH}/lib64/libcudart.so ${CUDA_PATH}/lib64/stubs/libcuda.so) endif () @@ -96,7 +101,12 @@ if (ENABLE_TDTQUE) endif () add_dependencies(_c_dataengine _c_mindrecord) -target_link_libraries(_c_dataengine PRIVATE _c_mindrecord) +if (WIN32) + set(MINDRECORD_LINK_OBJECT ${CMAKE_BINARY_DIR}/mindspore/ccsrc/mindrecord/CMakeFiles/_c_mindrecord.dir/objects.a) + target_link_libraries(_c_dataengine PRIVATE _c_mindrecord ${MINDRECORD_LINK_OBJECT} mindspore::sqlite) +else() + target_link_libraries(_c_dataengine PRIVATE _c_mindrecord) +endif() if (USE_GLOG) target_link_libraries(_c_dataengine PRIVATE mindspore::glog) diff --git a/mindspore/ccsrc/dataset/api/python_bindings.cc b/mindspore/ccsrc/dataset/api/python_bindings.cc index 3d543f946b3..e2675ee217c 100644 --- a/mindspore/ccsrc/dataset/api/python_bindings.cc +++ b/mindspore/ccsrc/dataset/api/python_bindings.cc @@ -19,7 +19,9 @@ #include "dataset/kernels/no_op.h" #include "dataset/kernels/data/one_hot_op.h" #include "dataset/kernels/image/center_crop_op.h" +#if !defined(_WIN32) && !defined(_WIN64) #include "dataset/kernels/image/change_mode_op.h" +#endif #include "dataset/kernels/image/cut_out_op.h" #include "dataset/kernels/image/decode_op.h" #include "dataset/kernels/image/distort_bounding_box_crop_op.h" @@ -279,9 +281,11 @@ void bindTensorOps2(py::module *m) { py::arg("fillG") = RandomCropOp::kDefFillG, py::arg("fillB") = RandomCropOp::kDefFillB); (void)py::class_>(*m, "ChannelSwapOp").def(py::init<>()); +#if !defined(_WIN32) && !defined(_WIN64) (void)py::class_>( *m, "ChangeModeOp", "Tensor operation to change colors from BGR to RGB") .def(py::init<>()); +#endif (void)py::class_>( *m, "OneHotOp", "Tensor operation to apply one hot encoding. Takes number of classes.") diff --git a/mindspore/ccsrc/dataset/core/tensor.h b/mindspore/ccsrc/dataset/core/tensor.h index 2017c2dfab6..3409354d196 100644 --- a/mindspore/ccsrc/dataset/core/tensor.h +++ b/mindspore/ccsrc/dataset/core/tensor.h @@ -22,6 +22,10 @@ #include #include "./securec.h" #include "utils/log_adapter.h" +#if defined(_WIN32) || defined(_WIN64) +#undef HAVE_STDDEF_H +#undef HAVE_STDLIB_H +#endif #include "pybind11/numpy.h" #include "pybind11/pybind11.h" #include "pybind11/stl.h" diff --git a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc index 2afafe21289..bdf39b6a392 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc @@ -85,7 +85,11 @@ Status ShuffleOp::SelfReset() { if (!reshuffle_each_epoch_) { rng_ = std::mt19937_64(shuffle_seed_); } else { +#if defined(_WIN32) || defined(_WIN64) + std::random_device random_device; +#else std::random_device random_device("/dev/urandom"); +#endif std::uniform_int_distribution distribution(0, std::numeric_limits::max()); shuffle_seed_ = distribution(random_device); rng_ = std::mt19937_64(shuffle_seed_); diff --git a/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt b/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt index 7a6240d1732..23a26d5214a 100644 --- a/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt +++ b/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt @@ -1,22 +1,46 @@ -add_library(kernels-image OBJECT - center_crop_op.cc - change_mode_op.cc - cut_out_op.cc - decode_op.cc - distort_bounding_box_crop_op.cc - hwc_to_chw_op.cc - image_utils.cc - normalize_op.cc - pad_op.cc - random_color_adjust_op.cc - random_crop_decode_resize_op.cc - random_crop_and_resize_op.cc - random_crop_op.cc - random_horizontal_flip_op.cc - random_resize_op.cc - random_rotation_op.cc - random_vertical_flip_op.cc - rescale_op.cc - resize_bilinear_op.cc - resize_op.cc - ) +if (WIN32) + add_library(kernels-image OBJECT + center_crop_op.cc + cut_out_op.cc + decode_op.cc + distort_bounding_box_crop_op.cc + hwc_to_chw_op.cc + image_utils.cc + normalize_op.cc + pad_op.cc + random_color_adjust_op.cc + random_crop_decode_resize_op.cc + random_crop_and_resize_op.cc + random_crop_op.cc + random_horizontal_flip_op.cc + random_resize_op.cc + random_rotation_op.cc + random_vertical_flip_op.cc + rescale_op.cc + resize_bilinear_op.cc + resize_op.cc + ) +else() + add_library(kernels-image OBJECT + center_crop_op.cc + change_mode_op.cc + cut_out_op.cc + decode_op.cc + distort_bounding_box_crop_op.cc + hwc_to_chw_op.cc + image_utils.cc + normalize_op.cc + pad_op.cc + random_color_adjust_op.cc + random_crop_decode_resize_op.cc + random_crop_and_resize_op.cc + random_crop_op.cc + random_horizontal_flip_op.cc + random_resize_op.cc + random_rotation_op.cc + random_vertical_flip_op.cc + rescale_op.cc + resize_bilinear_op.cc + resize_op.cc + ) +endif() diff --git a/mindspore/ccsrc/dataset/kernels/image/image_utils.cc b/mindspore/ccsrc/dataset/kernels/image/image_utils.cc index 8735cf7a059..63c9bb2641b 100644 --- a/mindspore/ccsrc/dataset/kernels/image/image_utils.cc +++ b/mindspore/ccsrc/dataset/kernels/image/image_utils.cc @@ -186,7 +186,11 @@ void JpegSetSource(j_decompress_ptr cinfo, const void *data, int64_t datasize) { (*cinfo->mem->alloc_small)(reinterpret_cast(cinfo), JPOOL_PERMANENT, sizeof(struct jpeg_source_mgr))); cinfo->src->init_source = JpegInitSource; cinfo->src->fill_input_buffer = JpegFillInputBuffer; +#if defined(_WIN32) || defined(_WIN64) + cinfo->src->skip_input_data = reinterpret_cast(JpegSkipInputData); +#else cinfo->src->skip_input_data = JpegSkipInputData; +#endif cinfo->src->resync_to_restart = jpeg_resync_to_restart; cinfo->src->term_source = JpegTermSource; cinfo->src->bytes_in_buffer = datasize; diff --git a/mindspore/ccsrc/dataset/kernels/image/image_utils.h b/mindspore/ccsrc/dataset/kernels/image/image_utils.h index a4ddef40d6b..51090fb9ea8 100644 --- a/mindspore/ccsrc/dataset/kernels/image/image_utils.h +++ b/mindspore/ccsrc/dataset/kernels/image/image_utils.h @@ -22,6 +22,10 @@ #include #include #include +#if defined(_WIN32) || defined(_WIN64) +#undef HAVE_STDDEF_H +#undef HAVE_STDLIB_H +#endif #include "./jpeglib.h" #include "./jerror.h" #include diff --git a/mindspore/ccsrc/dataset/util/path.cc b/mindspore/ccsrc/dataset/util/path.cc index dd72f807667..24c8db5b9ce 100644 --- a/mindspore/ccsrc/dataset/util/path.cc +++ b/mindspore/ccsrc/dataset/util/path.cc @@ -27,7 +27,7 @@ namespace mindspore { namespace dataset { #ifdef _WIN32 -char Path::_separator = '\\'; +char Path::separator_ = '\\'; #else char Path::separator_ = '/'; #endif @@ -129,7 +129,11 @@ bool Path::IsDirectory() { Status Path::CreateDirectory() { if (!Exists()) { +#if defined(_WIN32) || defined(_WIN64) + int rc = mkdir(common::SafeCStr(path_)); +#else int rc = mkdir(common::SafeCStr(path_), 0700); +#endif if (rc) { std::ostringstream oss; oss << "Unable to create directory " << path_ << ". Errno = " << errno; diff --git a/mindspore/ccsrc/dataset/util/random.cc b/mindspore/ccsrc/dataset/util/random.cc index e4bab6094c9..2a0762c9208 100644 --- a/mindspore/ccsrc/dataset/util/random.cc +++ b/mindspore/ccsrc/dataset/util/random.cc @@ -32,7 +32,11 @@ namespace dataset { uint32_t GetSeed() { uint32_t seed = GlobalContext::config_manager()->seed(); if (seed == std::mt19937::default_seed) { +#if defined(_WIN32) || defined(_WIN64) + std::random_device random_device; +#else std::random_device random_device("/dev/urandom"); +#endif std::uniform_int_distribution distribution(0, std::numeric_limits::max()); seed = distribution(random_device); } diff --git a/mindspore/ccsrc/dataset/util/services.cc b/mindspore/ccsrc/dataset/util/services.cc index 7dcb5b14c90..ea7b11014c4 100644 --- a/mindspore/ccsrc/dataset/util/services.cc +++ b/mindspore/ccsrc/dataset/util/services.cc @@ -16,7 +16,9 @@ #include "dataset/util/services.h" #include +#if !defined(_WIN32) && !defined(_WIN64) #include +#endif #include #include #include "dataset/util/circular_pool.h" @@ -28,6 +30,7 @@ namespace dataset { std::unique_ptr Services::instance_ = nullptr; std::once_flag Services::init_instance_flag_; +#if !defined(_WIN32) && !defined(_WIN64) std::string Services::GetUserName() { char user[LOGIN_NAME_MAX]; (void)getlogin_r(user, sizeof(user)); @@ -41,10 +44,15 @@ std::string Services::GetHostName() { } int Services::GetLWP() { return syscall(SYS_gettid); } +#endif std::string Services::GetUniqueID() { const std::string kStr = "abcdefghijklmnopqrstuvwxyz0123456789"; +#if defined(_WIN32) || defined(_WIN64) + std::mt19937 gen{std::random_device{}()}; +#else std::mt19937 gen{std::random_device{"/dev/urandom"}()}; +#endif std::uniform_int_distribution<> dist(0, kStr.size() - 1); char buffer[UNIQUEID_LEN]; for (int i = 0; i < UNIQUEID_LEN; i++) { diff --git a/mindspore/ccsrc/dataset/util/services.h b/mindspore/ccsrc/dataset/util/services.h index 65a302af918..5e81c4816ed 100644 --- a/mindspore/ccsrc/dataset/util/services.h +++ b/mindspore/ccsrc/dataset/util/services.h @@ -62,11 +62,13 @@ class Services { std::shared_ptr GetServiceMemPool() { return pool_; } +#if !defined(_WIN32) && !defined(_WIN64) static std::string GetUserName(); static std::string GetHostName(); static int GetLWP(); +#endif static std::string GetUniqueID(); diff --git a/mindspore/ccsrc/dataset/util/sig_handler.cc b/mindspore/ccsrc/dataset/util/sig_handler.cc index 1b6a3701c3c..644a6330665 100644 --- a/mindspore/ccsrc/dataset/util/sig_handler.cc +++ b/mindspore/ccsrc/dataset/util/sig_handler.cc @@ -16,13 +16,16 @@ #include "dataset/util/sig_handler.h" #include #include +#if !defined(_WIN32) && !defined(_WIN64) #include +#endif #include #include "dataset/util/task_manager.h" namespace mindspore { namespace dataset { // Register the custom signal handlers +#if !defined(_WIN32) && !defined(_WIN64) void RegisterHandlers() { struct sigaction new_int_action; @@ -40,5 +43,6 @@ extern void IntHandler(int sig_num, // The signal that was raised // Wake up the watchdog which is designed as async-signal-safe. TaskManager::WakeUpWatchDog(); } +#endif } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/sig_handler.h b/mindspore/ccsrc/dataset/util/sig_handler.h index 6c5e1f015c7..af40738feb4 100644 --- a/mindspore/ccsrc/dataset/util/sig_handler.h +++ b/mindspore/ccsrc/dataset/util/sig_handler.h @@ -22,12 +22,14 @@ namespace mindspore { namespace dataset { // Register the custom signal handlers +#if !defined(_WIN32) && !defined(_WIN64) extern void RegisterHandlers(); // A signal handler for SIGINT. Drives interrupt to watchdog extern void IntHandler(int sig_num, // The signal that was raised siginfo_t *sig_info, // The siginfo structure. void *context); // context info +#endif } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/debug/anf_ir_dump.cc b/mindspore/ccsrc/debug/anf_ir_dump.cc index fcf0777d169..e977084ab80 100644 --- a/mindspore/ccsrc/debug/anf_ir_dump.cc +++ b/mindspore/ccsrc/debug/anf_ir_dump.cc @@ -14,6 +14,9 @@ * limitations under the License. */ #include "debug/anf_ir_dump.h" +#if defined(_WIN32) || defined(_WIN64) +#include +#endif #include #include #include @@ -434,9 +437,15 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu return; } char real_path[PATH_MAX] = {0}; +#if defined(_WIN32) || defined(_WIN64) + if (_fullpath(real_path, filename.c_str(), PATH_MAX) == nullptr) { + MS_LOG(DEBUG) << "dir " << filename << " does not exit."; + } +#else if (nullptr == realpath(filename.c_str(), real_path)) { MS_LOG(DEBUG) << "Dir " << filename << " does not exit."; } +#endif OrderedMap para_map; std::string path_string = real_path; diff --git a/mindspore/ccsrc/debug/anf_ir_utils.cc b/mindspore/ccsrc/debug/anf_ir_utils.cc index c25ad862df5..8e626d6f9a7 100644 --- a/mindspore/ccsrc/debug/anf_ir_utils.cc +++ b/mindspore/ccsrc/debug/anf_ir_utils.cc @@ -48,9 +48,15 @@ std::string GetMsIrPath(void) { if (path_ptr != nullptr) { path = path_ptr; char real_path[PATH_MAX] = {0}; +#if defined(_WIN32) || defined(_WIN64) + if (path.size() > PATH_MAX || _fullpath(real_path, path.c_str(), PATH_MAX) == nullptr) { + MS_LOG(EXCEPTION) << "MS IR Path error, " << path_ptr; + } +#else if (path.size() > PATH_MAX || nullptr == realpath(path.c_str(), real_path)) { MS_LOG(EXCEPTION) << "MS IR path error, " << path_ptr; } +#endif path = real_path; } return path; @@ -2247,8 +2253,14 @@ void DumpIRProto(const FuncGraphPtr& func_graph, const std::string& suffix) { return; } char real_path[PATH_MAX] = {0}; - if (nullptr == realpath(file_path.c_str(), real_path)) { - MS_LOG(DEBUG) << "Dir " << file_path << " does not exit."; + char* real_path_ret = nullptr; +#if defined(_WIN32) || defined(_WIN64) + real_path_ret = _fullpath(real_path, file_path.c_str(), PATH_MAX); +#else + real_path_ret = realpath(file_path.c_str(), real_path); +#endif + if (nullptr == real_path_ret) { + MS_LOG(DEBUG) << "dir " << file_path << " does not exit."; } else { std::string path_string = real_path; if (chmod(common::SafeCStr(path_string), S_IRUSR | S_IWUSR) == -1) { diff --git a/mindspore/ccsrc/debug/info.cc b/mindspore/ccsrc/debug/info.cc index 5c1fc372c5f..3c43bfa9b13 100644 --- a/mindspore/ccsrc/debug/info.cc +++ b/mindspore/ccsrc/debug/info.cc @@ -53,10 +53,15 @@ std::string Location::ToString(SourceLineTip tip) { } char path[PATH_MAX + 1] = {0x00}; +#if defined(_WIN32) || defined(_WIN64) + if (file_name_.size() > PATH_MAX || _fullpath(path, file_name_.c_str(), PATH_MAX) == nullptr) { + return debug_info_ss.str(); + } +#else if (file_name_.size() > PATH_MAX || realpath(file_name_.c_str(), path) == nullptr) { return debug_info_ss.str(); } - +#endif auto src_path = std::string(path); std::ifstream file(src_path); if (!file.is_open()) { diff --git a/mindspore/ccsrc/device/cpu/cpu_resource_manager.cc b/mindspore/ccsrc/device/cpu/cpu_resource_manager.cc index 97df7d44874..45b9ea5bedd 100644 --- a/mindspore/ccsrc/device/cpu/cpu_resource_manager.cc +++ b/mindspore/ccsrc/device/cpu/cpu_resource_manager.cc @@ -60,6 +60,7 @@ void CPUResourceManager::MemMalloc(const session::KernelGraph *graph) { void *CPUResourceManager::MemMalloc(size_t mem_size) { void *ptr = malloc(mem_size); if (ptr != nullptr) { + memset_s(ptr, mem_size, 0, mem_size); dynamic_mem_[ptr] = mem_size; return ptr; } else { diff --git a/mindspore/ccsrc/device/gpu/distribution/collective_fake_init.cc b/mindspore/ccsrc/device/gpu/distribution/collective_fake_init.cc new file mode 100644 index 00000000000..06497a2e82c --- /dev/null +++ b/mindspore/ccsrc/device/gpu/distribution/collective_fake_init.cc @@ -0,0 +1,28 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/gpu/distribution/collective_fake_init.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace device { +namespace gpu { +void CollectiveFakeInitializer::InitCollective() { MS_LOG(EXCEPTION) << "build without enable gpu!"; } + +void CollectiveFakeInitializer::FinalizeCollective() { MS_LOG(EXCEPTION) << "build without enable gpu!"; } +} // namespace gpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/distribution/collective_fake_init.h b/mindspore/ccsrc/device/gpu/distribution/collective_fake_init.h new file mode 100644 index 00000000000..65467139c07 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/distribution/collective_fake_init.h @@ -0,0 +1,37 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_ + +namespace mindspore { +namespace device { +namespace gpu { + +class CollectiveFakeInitializer { + public: + CollectiveFakeInitializer() = default; + ~CollectiveFakeInitializer() = default; + CollectiveFakeInitializer(CollectiveFakeInitializer const &) = delete; + CollectiveFakeInitializer &operator=(const CollectiveFakeInitializer &) = delete; + static void InitCollective(); + static void FinalizeCollective(); +}; +} // namespace gpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_ diff --git a/mindspore/ccsrc/device/kernel_runtime.cc b/mindspore/ccsrc/device/kernel_runtime.cc index 3b0d84eada2..7f3d31d8d08 100644 --- a/mindspore/ccsrc/device/kernel_runtime.cc +++ b/mindspore/ccsrc/device/kernel_runtime.cc @@ -44,19 +44,29 @@ bool KernelRuntime::Run(session::KernelGraph *graph) { bool ret = false; auto context_ptr = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context_ptr); +#if defined(_WIN32) || defined(_WIN64) + auto start_time = std::chrono::steady_clock::now(); +#else struct timeval start_time, end_time; (void)gettimeofday(&start_time, nullptr); +#endif bool is_task_sink = context_ptr->enable_task_sink(); if (is_task_sink) { ret = RunTask(graph); } else { ret = LaunchKernel(graph); } +#if defined(_WIN32) || defined(_WIN64) + auto end_time = std::chrono::steady_clock::now(); + std::chrono::duration> cost = end_time - start_time; + MS_LOG(INFO) << "Call MS Run Success in " << cost.count() << " us"; +#else (void)gettimeofday(&end_time, nullptr); const uint64_t kUSecondInSecond = 1000000; uint64_t cost = kUSecondInSecond * static_cast(end_time.tv_sec - start_time.tv_sec); cost += static_cast(end_time.tv_usec - start_time.tv_usec); MS_LOG(INFO) << "Call MS Run Success in " << cost << " us"; +#endif return ret; } @@ -561,8 +571,12 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) { AddressPtrList kernel_workspaces; AddressPtrList kernel_outputs; GenLaunchArgs(*kernel_mod, kernel, &kernel_inputs, &kernel_workspaces, &kernel_outputs); +#if defined(_WIN32) || defined(_WIN64) + auto start_time = std::chrono::steady_clock::now(); +#else struct timeval start_time, end_time; (void)gettimeofday(&start_time, nullptr); +#endif auto ret = kernel_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, reinterpret_cast(stream_)); if (!ret) { @@ -572,11 +586,17 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) { if (AnfAlgo::GetKernelType(kernel) == TBE_KERNEL && !SyncStream()) { MS_LOG(EXCEPTION) << "SyncStream failed."; } +#if defined(_WIN32) || defined(_WIN64) + auto end_time = std::chrono::steady_clock::now(); + std::chrono::duration> cost = end_time - start_time; + MS_LOG(DEBUG) << "d " << kernel->fullname_with_scope() << " in " << cost.count() << " us"; +#else (void)gettimeofday(&end_time, nullptr); const uint64_t kUSecondInSecond = 1000000; uint64_t cost = kUSecondInSecond * static_cast(end_time.tv_sec - start_time.tv_sec); cost += static_cast(end_time.tv_usec - start_time.tv_usec); MS_LOG(DEBUG) << "d " << kernel->fullname_with_scope() << " in " << cost << " us"; +#endif } } return true; diff --git a/mindspore/ccsrc/kernel/common_utils.cc b/mindspore/ccsrc/kernel/common_utils.cc index 065cfaddc8b..5abaff412ec 100644 --- a/mindspore/ccsrc/kernel/common_utils.cc +++ b/mindspore/ccsrc/kernel/common_utils.cc @@ -117,7 +117,11 @@ bool IsAtomicNode(const CNodePtr &kernel_node) { bool KernelMeta::ReadIndex(const std::string &bin_dir) { DIR *dir = opendir(bin_dir.c_str()); if (dir == nullptr) { +#if defined(_WIN32) || defined(_WIN64) + auto ret = mkdir(bin_dir.c_str()); +#else auto ret = mkdir(bin_dir.c_str(), S_IRWXG | S_IRWXU); +#endif if (ret != 0) { MS_LOG(INFO) << "kernel dir not exist[" << bin_dir << "]."; return false; @@ -500,10 +504,17 @@ void SaveJsonInfo(const std::string &json_name, const std::string &info) { } filewrite << info << std::endl; filewrite.close(); +#if defined(_WIN32) || defined(_WIN64) + if (nullptr == _fullpath(real_path, path.c_str(), PATH_MAX)) { + MS_LOG(DEBUG) << "dir " << path << " does not exit."; + return; + } +#else if (nullptr == realpath(path.c_str(), real_path)) { MS_LOG(DEBUG) << "dir " << path << " does not exit."; return; } +#endif MS_LOG(INFO) << "real path is :" << real_path; if (chmod(real_path, S_IRUSR) == -1) { MS_LOG(DEBUG) << "modify file:" << real_path << " to read only fail."; diff --git a/mindspore/ccsrc/mindrecord/CMakeLists.txt b/mindspore/ccsrc/mindrecord/CMakeLists.txt index 4b8448287d7..a2b8897b16e 100644 --- a/mindspore/ccsrc/mindrecord/CMakeLists.txt +++ b/mindspore/ccsrc/mindrecord/CMakeLists.txt @@ -26,7 +26,11 @@ set_target_properties(_c_mindrecord PROPERTIES ) # add link library -target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite ${PYTHON_LIB} ${SECUREC_LIBRARY} mindspore mindspore_gvar mindspore::protobuf) +if (WIN32) + target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite mindspore mindspore::protobuf) +else() + target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite ${PYTHON_LIB} ${SECUREC_LIBRARY} mindspore mindspore_gvar mindspore::protobuf) +endif() if (USE_GLOG) target_link_libraries(_c_mindrecord PRIVATE mindspore::glog) diff --git a/mindspore/ccsrc/mindrecord/common/shard_utils.cc b/mindspore/ccsrc/mindrecord/common/shard_utils.cc index ca4bb8a261b..51de0c5f64d 100644 --- a/mindspore/ccsrc/mindrecord/common/shard_utils.cc +++ b/mindspore/ccsrc/mindrecord/common/shard_utils.cc @@ -65,6 +65,15 @@ std::pair GetFileName(const std::string &path) { return {FAILED, ""}; } char tmp[PATH_MAX] = {0}; +#if defined(_WIN32) || defined(_WIN64) + if (_fullpath(tmp, dirname(&(buf[0])), PATH_MAX) == nullptr) { + MS_LOG(ERROR) << "Invalid file path, path: " << buf; + return {FAILED, ""}; + } + if (_fullpath(real_path, common::SafeCStr(path), PATH_MAX) == nullptr) { + MS_LOG(DEBUG) << "Path: " << common::SafeCStr(path) << "check successfully"; + } +#else if (realpath(dirname(&(buf[0])), tmp) == nullptr) { MS_LOG(ERROR) << "Invalid file path, path: " << buf; return {FAILED, ""}; @@ -72,6 +81,7 @@ std::pair GetFileName(const std::string &path) { if (realpath(common::SafeCStr(path), real_path) == nullptr) { MS_LOG(DEBUG) << "Path: " << path << "check successfully"; } +#endif std::string s = real_path; char sep = '/'; size_t i = s.rfind(sep, s.length()); @@ -91,6 +101,15 @@ std::pair GetParentDir(const std::string &path) { return {FAILED, ""}; } char tmp[PATH_MAX] = {0}; +#if defined(_WIN32) || defined(_WIN64) + if (_fullpath(tmp, dirname(&(buf[0])), PATH_MAX) == nullptr) { + MS_LOG(ERROR) << "Invalid file path, path: " << buf; + return {FAILED, ""}; + } + if (_fullpath(real_path, common::SafeCStr(path), PATH_MAX) == nullptr) { + MS_LOG(DEBUG) << "Path: " << common::SafeCStr(path) << "check successfully"; + } +#else if (realpath(dirname(&(buf[0])), tmp) == nullptr) { MS_LOG(ERROR) << "Invalid file path, path: " << buf; return {FAILED, ""}; @@ -98,6 +117,7 @@ std::pair GetParentDir(const std::string &path) { if (realpath(common::SafeCStr(path), real_path) == nullptr) { MS_LOG(DEBUG) << "Path: " << path << "check successfully"; } +#endif std::string s = real_path; if (s.rfind('/') + 1 <= s.size()) { return {SUCCESS, s.substr(0, s.rfind('/') + 1)}; @@ -144,6 +164,9 @@ bool IsLegalFile(const std::string &path) { } std::pair GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type) { +#if defined(_WIN32) || defined(_WIN64) + return {SUCCESS, 100}; +#else uint64_t ll_count = 0; struct statfs disk_info; if (statfs(common::SafeCStr(str_dir), &disk_info) == -1) { @@ -166,6 +189,7 @@ std::pair GetDiskSize(const std::string &str_dir, const Dis } return {SUCCESS, ll_count}; +#endif } uint32_t GetMaxThreadNum() { diff --git a/mindspore/ccsrc/mindrecord/include/common/shard_utils.h b/mindspore/ccsrc/mindrecord/include/common/shard_utils.h index e18cbb75b98..d31037c8ad3 100644 --- a/mindspore/ccsrc/mindrecord/include/common/shard_utils.h +++ b/mindspore/ccsrc/mindrecord/include/common/shard_utils.h @@ -21,8 +21,10 @@ #include #include #include +#if !defined(_WIN32) && !defined(_WIN64) #include #include +#endif #include #include #include diff --git a/mindspore/ccsrc/mindrecord/include/shard_reader.h b/mindspore/ccsrc/mindrecord/include/shard_reader.h index c114b17951d..5548473cd78 100644 --- a/mindspore/ccsrc/mindrecord/include/shard_reader.h +++ b/mindspore/ccsrc/mindrecord/include/shard_reader.h @@ -19,7 +19,9 @@ #include #include +#if !defined(_WIN32) && !defined(_WIN64) #include +#endif #include #include #include diff --git a/mindspore/ccsrc/mindrecord/io/shard_reader.cc b/mindspore/ccsrc/mindrecord/io/shard_reader.cc index 085f148a881..fd3fede5a2d 100644 --- a/mindspore/ccsrc/mindrecord/io/shard_reader.cc +++ b/mindspore/ccsrc/mindrecord/io/shard_reader.cc @@ -987,8 +987,10 @@ TASK_RETURN_CONTENT ShardReader::ConsumerOneTask(int task_id, uint32_t consumer_ MSRStatus ShardReader::ConsumerByRow(int consumer_id) { // Set thread name +#if !defined(_WIN32) && !defined(_WIN64) auto thread_id = kThreadName + std::to_string(consumer_id); prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0); +#endif // Loop forever for (;;) { @@ -1040,8 +1042,10 @@ MSRStatus ShardReader::ReadBlob(const int &shard_id, const uint64_t &page_offset MSRStatus ShardReader::ConsumerByBlock(int consumer_id) { // Set thread name +#if !defined(_WIN32) && !defined(_WIN64) auto thread_id = kThreadName + std::to_string(consumer_id); prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0); +#endif // Loop forever for (;;) { diff --git a/mindspore/ccsrc/mindrecord/io/shard_writer.cc b/mindspore/ccsrc/mindrecord/io/shard_writer.cc index 3d4259ebbdd..864e6697d03 100644 --- a/mindspore/ccsrc/mindrecord/io/shard_writer.cc +++ b/mindspore/ccsrc/mindrecord/io/shard_writer.cc @@ -63,6 +63,15 @@ MSRStatus ShardWriter::Open(const std::vector &paths, bool append) MS_LOG(ERROR) << "Securec func failed"; return FAILED; } +#if defined(_WIN32) || defined(_WIN64) + if (_fullpath(resolved_path, dirname(&(buf[0])), PATH_MAX) == nullptr) { + MS_LOG(ERROR) << "Invalid file path"; + return FAILED; + } + if (_fullpath(resolved_path, common::SafeCStr(path), PATH_MAX) == nullptr) { + MS_LOG(DEBUG) << "Path " << resolved_path; + } +#else if (realpath(dirname(&(buf[0])), resolved_path) == nullptr) { MS_LOG(ERROR) << "Invalid file path"; return FAILED; @@ -70,6 +79,7 @@ MSRStatus ShardWriter::Open(const std::vector &paths, bool append) if (realpath(common::SafeCStr(path), resolved_path) == nullptr) { MS_LOG(DEBUG) << "Path " << resolved_path; } +#endif file_paths_.emplace_back(string(resolved_path)); } diff --git a/mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.cc b/mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.cc index 9dbd3a02465..8ab08952160 100644 --- a/mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.cc +++ b/mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.cc @@ -38,10 +38,12 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti (root->has_flag(ALLREDUCE_FUSION_RUN_ONCE_ONLY))) { return changes; } - +#if defined(_WIN32) || defined(_WIN64) + auto start_time = std::chrono::steady_clock::now(); +#else struct timeval start_time, end_time; (void)gettimeofday(&start_time, nullptr); - +#endif MS_LOG(INFO) << "Now entering allreduce fusion"; DumpGraph(root, std::string(ALLREDUCE_FUSION_BEGIN)); @@ -63,11 +65,16 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti // allreduce fusion only run once root->flags()[ALLREDUCE_FUSION_RUN_ONCE_ONLY] = true; res->results()[pipeline::kStepParallelGraph] = root; - +#if defined(_WIN32) || defined(_WIN64) + auto end_time = std::chrono::steady_clock::now(); + std::chrono::duration> cost = end_time - start_time; + MS_LOG(INFO) << "Now leaving allreduce fusion, used time: " << cost.count() << " us"; +#else (void)gettimeofday(&end_time, nullptr); uint64_t time = 1000000 * static_cast(end_time.tv_sec - start_time.tv_sec); time += static_cast(end_time.tv_usec - start_time.tv_usec); MS_LOG(INFO) << "Now leaving allreduce fusion, used time: " << time << " us"; +#endif return changes; } } // namespace parallel diff --git a/mindspore/ccsrc/parallel/step_auto_parallel.cc b/mindspore/ccsrc/parallel/step_auto_parallel.cc index a42ce612fb5..1b881e5d239 100644 --- a/mindspore/ccsrc/parallel/step_auto_parallel.cc +++ b/mindspore/ccsrc/parallel/step_auto_parallel.cc @@ -229,7 +229,7 @@ size_t GetLengthOfDataType(const TypePtr &type) { case kNumberTypeInt: return sizeof(int); case kNumberTypeUInt: - return sizeof(uint); + return sizeof(unsigned int); case kNumberTypeFloat: return sizeof(float); default: diff --git a/mindspore/ccsrc/pipeline/action.cc b/mindspore/ccsrc/pipeline/action.cc index d77fee84aa7..3e0f8804e71 100644 --- a/mindspore/ccsrc/pipeline/action.cc +++ b/mindspore/ccsrc/pipeline/action.cc @@ -264,7 +264,7 @@ bool TaskEmitAction(const ResourcePtr& res) { auto bc_ptr = res->results()[kBackend].cast(); std::vector cut_list = compile::nonlinear_ops; if (bc_ptr->name() == kMsConvert) { - cut_list = compile::ms_nonlinear_ops; + cut_list = compile::GetMsNonlinearOps(); } std::shared_ptr compile = std::make_shared(bc_ptr, cut_list); res->results()[kOutput] = compile->CompileAndLink(func_graph); diff --git a/mindspore/ccsrc/pipeline/init.cc b/mindspore/ccsrc/pipeline/init.cc index 24ead047d3f..aa370254bbe 100644 --- a/mindspore/ccsrc/pipeline/init.cc +++ b/mindspore/ccsrc/pipeline/init.cc @@ -29,8 +29,11 @@ #include "parallel/context.h" #include "parallel/device_manager.h" #include "parallel/costmodel_context.h" +#ifdef ENABLE_GPUQUE #include "device/gpu/distribution/collective_init.h" - +#else +#include "device/gpu/distribution/collective_fake_init.h" +#endif namespace py = pybind11; using FuncGraph = mindspore::FuncGraph; @@ -297,9 +300,16 @@ PYBIND11_MODULE(_c_expression, m) { (void)py::class_>(m, "Oplib") .def(py::init()) .def("reg_op", &OpLib::RegOp, "Register op info."); - +#ifdef ENABLE_GPUQUE (void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::InitCollective, "Init gpu collective communication mode."); (void)m.def("finalize_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::FinalizeCollective, "Finalize gpu collective communication mode."); +#else + (void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveFakeInitializer::InitCollective, + "Init gpu collective communication mode."); + (void)m.def("finalize_gpu_collective", &mindspore::device::gpu::CollectiveFakeInitializer::FinalizeCollective, + "Finalize gpu collective communication mode."); + +#endif } diff --git a/mindspore/ccsrc/pre_activate/common/pass_manager.cc b/mindspore/ccsrc/pre_activate/common/pass_manager.cc index 0dfe4c763fd..f93a1ad51d5 100644 --- a/mindspore/ccsrc/pre_activate/common/pass_manager.cc +++ b/mindspore/ccsrc/pre_activate/common/pass_manager.cc @@ -52,17 +52,27 @@ bool PassManager::Run(const FuncGraphPtr &func_graph, const std::vector size_t num = 0; for (const auto &pass : passes) { if (pass != nullptr) { +#if defined(_WIN32) || defined(_WIN64) + auto start_time = std::chrono::steady_clock::now(); +#else struct timeval start_time {}; struct timeval end_time {}; (void)gettimeofday(&start_time, nullptr); +#endif if (pass->Run(func_graph)) { changed = true; } +#if defined(_WIN32) || defined(_WIN64) + auto end_time = std::chrono::steady_clock::now(); + std::chrono::duration> cost = end_time - start_time; + MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost.count() << " us"; +#else (void)gettimeofday(&end_time, nullptr); const uint64_t kUSecondInSecond = 1000000; uint64_t cost = kUSecondInSecond * static_cast(end_time.tv_sec - start_time.tv_sec); cost += static_cast(end_time.tv_usec - start_time.tv_usec); MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost << " us"; +#endif if (save_graphs) { auto dump_file_path = save_graphs_path + "/" + "hwopt_" + name() + "_" + std::to_string(num) + "_" + pass->name() + ".ir"; diff --git a/mindspore/ccsrc/utils/log_adapter.cc b/mindspore/ccsrc/utils/log_adapter.cc index 19482ec1939..5d6e734fea0 100644 --- a/mindspore/ccsrc/utils/log_adapter.cc +++ b/mindspore/ccsrc/utils/log_adapter.cc @@ -26,12 +26,19 @@ namespace mindspore { #ifdef USE_GLOG static std::string GetTime() { #define BUFLEN 80 + static char buf[BUFLEN]; +#if defined(_WIN32) || defined(_WIN64) + time_t time_seconds = time(0); + struct tm now_time; + localtime_s(&now_time, &time_seconds); + sprintf_s(buf, BUFLEN, "%d-%d-%d %d:%d:%d", now_time.tm_year + 1900, now_time.tm_mon + 1, now_time.tm_mday, + now_time.tm_hour, now_time.tm_min, now_time.tm_sec); +#else struct timeval cur_time; (void)gettimeofday(&cur_time, NULL); struct tm now; (void)localtime_r(&cur_time.tv_sec, &now); - static char buf[BUFLEN]; (void)strftime(buf, BUFLEN, "%Y-%m-%d-%H:%M:%S", &now); // format date and time // set micro-second buf[27] = '\0'; @@ -44,6 +51,7 @@ static std::string GetTime() { buf[idx--] = '.'; } } +#endif return std::string(buf); } diff --git a/mindspore/ccsrc/vm/transform.cc b/mindspore/ccsrc/vm/transform.cc index be7aaf5baa8..92976e0ddb9 100644 --- a/mindspore/ccsrc/vm/transform.cc +++ b/mindspore/ccsrc/vm/transform.cc @@ -41,8 +41,10 @@ using TypedPrimitiveAbstractClosurePtr = std::shared_ptr nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch, prim::kPrimMakeTuple}; - -std::vector ms_nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch}; +const std::vector& GetMsNonlinearOps() { + static const std::vector ms_nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch}; + return ms_nonlinear_ops; +} CompileGraph::CompileGraph(const BackendPtr& backend, const std::vector& cut_list) : backend_(backend), cut_list_(cut_list) { diff --git a/mindspore/ccsrc/vm/transform.h b/mindspore/ccsrc/vm/transform.h index f862444a823..290af100491 100644 --- a/mindspore/ccsrc/vm/transform.h +++ b/mindspore/ccsrc/vm/transform.h @@ -42,7 +42,7 @@ extern const char kGeVm[]; // A sub namespace in ME to support compile related definition. namespace compile { extern std::vector nonlinear_ops; -extern std::vector ms_nonlinear_ops; +const std::vector& GetMsNonlinearOps(); using VmEvalFunc = std::function; using VmEvalFuncPtr = std::shared_ptr>; diff --git a/mindspore/log.py b/mindspore/log.py index 38455e2e18c..9731b04ac10 100644 --- a/mindspore/log.py +++ b/mindspore/log.py @@ -19,11 +19,13 @@ import sys import os import stat import time -import fcntl import logging from logging.handlers import RotatingFileHandler import traceback import threading +import platform +if platform.system() != "Windows": + import fcntl __all__ = ['get_level', 'get_log_config'] @@ -90,7 +92,8 @@ class _MultiCompatibleRotatingFileHandler(RotatingFileHandler): # Attain an exclusive lock with bloking mode by `fcntl` module. with open(self.baseFilename, 'a') as file_pointer: - fcntl.lockf(file_pointer.fileno(), fcntl.LOCK_EX) + if platform.system() != "Windows": + fcntl.lockf(file_pointer.fileno(), fcntl.LOCK_EX) if self.backupCount > 0: self.rolling_rename() diff --git a/third_party/securec/CMakeLists.txt b/third_party/securec/CMakeLists.txt index e360a6ebaea..b2f29c488b9 100644 --- a/third_party/securec/CMakeLists.txt +++ b/third_party/securec/CMakeLists.txt @@ -1,10 +1,18 @@ SET(CMAKE_BUILD_TYPE "Debug") -SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='") +if (WIN32) + SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer") +else() + SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='") +endif() SET(CMAKE_C_FLAGS_RELEASE "$ENV{CFLAGS} -fPIC -O3 -Wall -Wno-deprecated-declarations") set(CMAKE_EXPORT_COMPILE_COMMANDS ON) #add flags -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include -Werror") +if (WIN32) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include") +else() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include -Werror") +endif() include_directories(./include)