msprof data type

mindspore/ccsrc/plugin/device/ascend/hal/device/profiling/profiling_reporter.cc

@@ -254,20 +254,21 @@ void ProfilingReporter::BuildTensorData(MsprofGeTensorData *tensor_data, const C
   tensor_data->tensorType = tensor_type;
   std::vector<size_t> shape;
   string data_format;
+  uint32_t vm_data_type;
   if (tensor_type == MSPROF_GE_TENSOR_TYPE_INPUT) {
     auto input_node_with_index = common::AnfAlgo::GetPrevNodeOutput(node, index);
     auto input_node = input_node_with_index.first;
     auto input_index = input_node_with_index.second;
     shape = AnfAlgo::GetOutputDeviceShape(input_node, input_index);
     data_format = AnfAlgo::GetOutputFormat(input_node, input_index);
-    tensor_data->dataType = static_cast<uint32_t>(AnfAlgo::GetOutputDeviceDataType(input_node, input_index));
+    vm_data_type = static_cast<uint32_t>(AnfAlgo::GetOutputDeviceDataType(input_node, input_index));
   } else {
     shape = AnfAlgo::GetOutputDeviceShape(node, index);
     data_format = AnfAlgo::GetOutputFormat(node, index);
-    tensor_data->dataType = static_cast<uint32_t>(AnfAlgo::GetOutputDeviceDataType(node, index));
+    vm_data_type = static_cast<uint32_t>(AnfAlgo::GetOutputDeviceDataType(node, index));
   }
-
-  tensor_data->format = OpFormat2Index[data_format];
+  tensor_data->dataType = vm_data_type + MSPROF_DIFFERENCE;
+  tensor_data->format = OpFormat2Index[data_format] + MSPROF_DIFFERENCE;
   auto shape_size = std::min(static_cast<uint64_t>(MSPROF_GE_TENSOR_DATA_SHAPE_LEN), shape.size());
   (void)std::copy(shape.begin(), shape.begin() + shape_size, tensor_data->shape);
 }

mindspore/ccsrc/plugin/device/ascend/hal/device/profiling/profiling_reporter.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2021 Huawei Technologies Co., Ltd
+ * Copyright 2021-2022 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -70,6 +70,7 @@ class ProfilingReporter {
   vector<uint32_t> stream_ids_;
   vector<uint32_t> task_ids_;
   map<string, int> node_name_index_map_;
+  const uint32_t MSPROF_DIFFERENCE = 200;
 
   bool CheckStreamTaskValid() const;
   static uint32_t GetBlockDim(const CNodePtr &node);
@@ -88,14 +89,14 @@ class ProfilingReporter {
   void SetAlternativeValue(T *property, const size_t property_size, const string &value, const int32_t &device_id) {
     MS_EXCEPTION_IF_NULL(property);
     if (value.size() < property_size) {
-      property->type = static_cast<uint8_t>(MSPROF_MIX_DATA_HASH_ID);
+      property->type = static_cast<uint8_t>(MSPROF_MIX_DATA_STRING);
       const auto ret = strncpy_s(property->data.dataStr, property_size, value.c_str(), value.size());
       if (ret != 0) {
         MS_LOG(ERROR) << "[Profiling] strncpy_s value " << value.c_str() << " error!";
         return;
       }
     } else {
-      property->type = static_cast<uint8_t>(MSPROF_MIX_DATA_STRING);
+      property->type = static_cast<uint8_t>(MSPROF_MIX_DATA_HASH_ID);
       uint64_t hash_id;
       ProfilingManager::GetInstance().QueryHashId(device_id, value, &hash_id);
       property->data.hashId = hash_id;

mindspore/ccsrc/profiler/device/ascend/parallel_strategy_profiling.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2021 Huawei Technologies Co., Ltd
+ * Copyright 2021-2022 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -137,7 +137,7 @@ irpb::ProfilingParallel GetProfilingParallel(const FuncGraphPtr &func_graph) {
 }
 
 void DumpProfileParallelStrategy(const FuncGraphPtr &func_graph) {
-  if (!IsProfilingParallelStrategyEnabled()) {
+  if (has_save_parallel_strategy || !IsProfilingParallelStrategyEnabled()) {
     return;
   }
 

mindspore/python/mindspore/profiler/parser/framework_enum.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2021-2022 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -104,3 +104,6 @@ class VmFormat(IntEnum):
         """
         format_name = cls._value2member_map_.get(num)
         return 'UNKNOWN' if format_name is None else format_name.name
+
+
+MSPROF_DIFFERENCE = 200

mindspore/python/mindspore/profiler/parser/framework_parser.py

@@ -1,4 +1,4 @@
-# Copyright 2020-2021 Huawei Technologies Co., Ltd
+# Copyright 2020-2022 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,7 +27,7 @@ from mindspore import log as logger
 from mindspore.profiler.parser.framework_struct import TASK_DESC_STRUCT
 from mindspore.profiler.parser.framework_struct import TENSOR_DATA_STRUCT
 from mindspore.profiler.parser.framework_struct import STEP_INFO_STRUCT
-from mindspore.profiler.parser.framework_enum import VmDataType, VmFormat, FileDataType
+from mindspore.profiler.parser.framework_enum import VmDataType, VmFormat, FileDataType, MSPROF_DIFFERENCE
 from mindspore.profiler.common.struct_type import StructType
 from mindspore.profiler.common.util import combine_stream_task_id
 from mindspore.profiler.common.exceptions.exceptions import ProfilerDirNotFoundException
@@ -253,7 +253,7 @@ class FrameworkParser:
         # skip rsv data, rsv has 7 bytes
         skip_size = 8
         remain_size = data_size - skip_size
-        if flag == 0:
+        if flag == 1:
             unpack_data = struct.unpack(StructType.CHAR.value * remain_size,
                                         item_binary_data[cursor + skip_size:cursor + data_size])
             unpack_data = ''.join(list(map(lambda c: c.decode(), filter(lambda c: c != b'\x00', unpack_data))))
@@ -315,6 +315,20 @@ class FrameworkParser:
             point_info[tag] = full_op_name
         return point_info
 
+    @staticmethod
+    def _get_vm_data_type(msprof_data_type):
+        """Get the mapped vm data type of msprof."""
+        if msprof_data_type >= MSPROF_DIFFERENCE:
+            return msprof_data_type - MSPROF_DIFFERENCE
+        return msprof_data_type
+
+    @staticmethod
+    def _get_vm_op_format(msprof_op_format):
+        """Get the mapped op format type of msprof."""
+        if msprof_op_format >= MSPROF_DIFFERENCE:
+            return msprof_op_format - MSPROF_DIFFERENCE
+        return msprof_op_format
+
     @staticmethod
     def _construct_task_id_op_attr_dict(prof_tensor_data):
         """prof_tensor_data is a list[tensor_data], tensor_data is a dict, key is same as TENSOR_DATA_STRUCT."""
@@ -323,11 +337,11 @@ class FrameworkParser:
             task_id = combine_stream_task_id(tensor_data['streamId'], tensor_data['taskId'])
             for tensor_attr in tensor_data['tensorData']:
                 tensor_type = 'input' if tensor_attr['tensorType'] == 0 else 'output'
-                tensor_format = VmFormat.get_format_name(tensor_attr['format'])
+                tensor_format = VmFormat.get_format_name(FrameworkParser._get_vm_data_type(tensor_attr['format']))
                 op_attr = dict(
                     tensor_type=tensor_type,
                     format=tensor_format,
-                    data_type=VmDataType.get_data_type_name(tensor_attr['dataType']),
+                    data_type=VmDataType.get_data_type_name(FrameworkParser._get_vm_op_format(tensor_attr['dataType'])),
                     shape=tensor_attr['shape']
                 )
                 task_id_op_attr_dict[task_id].append(op_attr)

mindspore/python/mindspore/profiler/parser/integrator.py

@@ -587,6 +587,7 @@ class BaseTimelineGenerator:
         }
         self._device_target = str(device_target).lower()
         self._model = model
+        self._step_start_op_name = ""
         self._step_end_op_name = ""
 
     def get_thread_label_name(self):
@@ -660,8 +661,8 @@ class BaseTimelineGenerator:
         the merged result is [[1,6], [7,8]].
         """
         time_merged_segment_list = []
-        tid = self._tid_dict[display_name][0]
-        pid = self._tid_dict[display_name][1]
+        tid = self._tid_dict.get(display_name, (0, 0))[0]
+        pid = self._tid_dict.get(display_name, (0, 0))[1]
         for time_item in time_list:
             time_segment = list(map(float, time_item[self._start_time_idx:self._duration_idx + 1]))
             time_segment[1] = time_segment[0] + time_segment[1] / factor
@@ -1888,6 +1889,22 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
             timeline_list[idx][self._start_time_idx] = timeline_list[idx][self._start_time_idx] / 1000000
             timeline_list[idx][self._duration_idx] = timeline_list[idx][self._duration_idx] / 1000
 
+    def _set_step_start_and_end_op_name(self, timeline_list):
+        """Set the start and end operator full name of each step."""
+        if not timeline_list or len(timeline_list) < 2:
+            return
+
+        start_op_idx = 0
+        self._step_end_op_name = timeline_list[-1][self._op_name_idx]
+        for i, timeline in enumerate(timeline_list):
+            if timeline[self._op_name_idx] == self._step_end_op_name:
+                start_op_idx = i + 1
+                break
+
+        if start_op_idx >= len(timeline_list):
+            start_op_idx = 0
+        self._step_start_op_name = timeline_list[start_op_idx][self._op_name_idx]
+
 
 class CpuTimelineGenerator(GpuTimelineGenerator):
     """Generate cpu Timeline data from file."""