!450 Fix segment bug on windows

Merge pull request !450 from xiefangqi/fix_segment_on_windows

mindspore/ccsrc/CMakeLists.txt

@@ -156,7 +156,7 @@ file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
         )
 if (ENABLE_CPU)
     list(REMOVE_ITEM MINDSPORE_SRC_LIST "device/gpu/distribution/collective_init.cc")
-    if (WIN32)
+    if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
         list(REMOVE_ITEM MINDSPORE_SRC_LIST "kernel/kernel_query.cc")
     endif()
 endif()
@@ -337,19 +337,19 @@ if (ENABLE_D)
     target_link_libraries(mindspore mindspore::protobuf)
 endif()
 
-if (WIN32)
+if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
     target_link_libraries(mindspore ${PYTHON_LIBRARIES} mindspore_gvar)
 endif()
 
 # set c_expression building
-if (WIN32)
-set(PYTHON_MODULE_SOURCE ${MS_GVAR_SRC_LIST}
+if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+    set(PYTHON_MODULE_SOURCE ${MS_GVAR_SRC_LIST}
         pipeline/init.cc
         kernel/oplib/oplib.cc
         ${MINDSPORE_SRC_LIST} ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST}
         ${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST})
 else()
-set(PYTHON_MODULE_SOURCE
+    set(PYTHON_MODULE_SOURCE
         pipeline/init.cc
         kernel/oplib/oplib.cc
         ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST}
@@ -426,13 +426,5 @@ endif()
 
 if(ENABLE_MINDDATA)
     add_subdirectory(mindrecord)
-    if (WIN32)
-	set(_md_tmp_CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE})
-	set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O0 -Wl,--allow-shlib-undefined -DHALF_ENABLE_CPP11_USER_LITERALS=0")
-	set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -D_FORTIFY_SOURCE=2")
-        add_subdirectory(dataset)
-	set(CMAKE_CXX_FLAGS_RELEASE ${_md_tmp_CMAKE_CXX_FLAGS_RELEASE})
-    else()
-	add_subdirectory(dataset)
-    endif()
+    add_subdirectory(dataset)
 endif()

mindspore/ccsrc/dataset/CMakeLists.txt

@@ -12,6 +12,9 @@ endif()
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes")
 
+if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--image-base -Wl,0x10000000")
+endif()
 ############################# Options ################################
 if (ENABLE_GPUQUE)
     add_definitions(-D ENABLE_GPUQUE)
@@ -80,7 +83,7 @@ set_target_properties(_c_dataengine PROPERTIES
 ######################################################################
 
 ################# Link with external libraries ########################
-if (WIN32)
+if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
     target_link_libraries(_c_dataengine PRIVATE mindspore)
     target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module ${PYTHON_LIBRARIES} mindspore::protobuf ${SECUREC_LIBRARY})
 else()
@@ -101,7 +104,7 @@ if (ENABLE_TDTQUE)
 endif ()
 
 add_dependencies(_c_dataengine _c_mindrecord)
-if (WIN32)
+if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
     set(MINDRECORD_LINK_OBJECT ${CMAKE_BINARY_DIR}/mindspore/ccsrc/mindrecord/CMakeFiles/_c_mindrecord.dir/objects.a)
     target_link_libraries(_c_dataengine PRIVATE _c_mindrecord ${MINDRECORD_LINK_OBJECT} mindspore::sqlite)
 else()

mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc

@@ -109,11 +109,15 @@ void DatasetOp::Print(std::ostream &out, bool show_all) const {
 
 // Gets the next buffer from the given child
 Status DatasetOp::GetNextBuffer(std::unique_ptr<DataBuffer> *p_buffer, int32_t worker_id, bool retry_if_eoe) {
+#if defined(_WIN32) || defined(_WIN64)
+  RETURN_IF_NOT_OK(out_connector_->PopWithRetry(static_cast<int>(worker_id), p_buffer, retry_if_eoe));
+#else
   std::unique_ptr<DataBuffer> next_buff;
   // pop is a blocked call and will throw an interruption if the whole group shuts down.
   RETURN_IF_NOT_OK(out_connector_->PopWithRetry(static_cast<int>(worker_id), &next_buff, retry_if_eoe));
 
   *p_buffer = std::move(next_buff);
+#endif
   return Status::OK();
 }
 

mindspore/ccsrc/dataset/engine/datasetops/map_op.cc

@@ -65,6 +65,9 @@ MapOp::MapOp(const std::vector<std::string> &in_col_names, const std::vector<std
       tfuncs_(std::move(tensor_funcs)),
       in_columns_(in_col_names),
       out_columns_(out_col_names),
+#if defined(_WIN32) || defined(_WIN64)
+      eof_worker_id_(0),
+#endif
       perf_mode_(perf_mode) {
   // If caller didn't specify the out_col_names, assume they are same as the in_columns.
   if (out_columns_.empty() || out_columns_[0].empty()) {
@@ -120,6 +123,17 @@ Status MapOp::operator()() {
       RETURN_IF_NOT_OK(child_[0]->GetNextBuffer(&buff, 0));
       is_eof = buff->eof();
       RETURN_IF_NOT_OK(local_queues_[que_id]->Add(std::move(buff)));
+#if defined(_WIN32) || defined(_WIN64)
+      if (is_eof) {
+        eof_worker_id_ = que_id;
+        for (int32_t id = 0; id < num_workers_; id++) {
+          if (id != eof_worker_id_) {
+            auto eof_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
+            RETURN_IF_NOT_OK(local_queues_[id]->Add(std::move(eof_buffer)));
+          }
+        }
+      }
+#endif
       que_id = (que_id + 1) % num_workers_;
     }
   }
@@ -159,6 +173,14 @@ Status MapOp::WorkerEntry(int32_t worker_id) {
       continue;
     } else if (in_buffer->eof()) {
       // Calling base class EofReceived to forward eof buffer.
+#if defined(_WIN32) || defined(_Win64)
+      if (perf_mode_) {
+        if (eof_worker_id_ == worker_id) {
+          RETURN_IF_NOT_OK(EofReceived(worker_id));
+        }
+        break;
+      }
+#endif
       RETURN_IF_NOT_OK(EofReceived(worker_id));
       break;
     }

mindspore/ccsrc/dataset/engine/datasetops/map_op.h

@@ -193,6 +193,10 @@ class MapOp : public ParallelOp {
   // cause additional blocking because pop calls to Connector from the threads are synchronized to enforce the order.
   bool perf_mode_;
 
+#if defined(_WIN32) || defined(_WIN64)
+  // EOF worker id is only work on Performance mode, to record the worker id of queue which gets EOF
+  int32_t eof_worker_id_;
+#endif
   // Private function for worker/thread to loop continuously. It comprises the main
   // logic of MapOp: getting the data from previous Op, validating user specified column names,
   // applying a list of TensorOps to each of the data, process the results and then

mindspore/ccsrc/mindrecord/CMakeLists.txt

@@ -26,7 +26,7 @@ set_target_properties(_c_mindrecord PROPERTIES
     )
 
 # add link library
-if (WIN32)
+if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
     target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite mindspore mindspore::protobuf)
 else()
     target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite ${PYTHON_LIB} ${SECUREC_LIBRARY} mindspore mindspore_gvar mindspore::protobuf)