Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Profiler conditional enabled cannot be used.

This commit is contained in:
liuchuting 2022-03-28 13:38:27 +08:00
parent 44f2e803a2
commit 7d0ca5a10d
7 changed files with 84 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ccsrc/plugin/device/ascend/hal/device/profiling/profiling_manager.h
View File

@ -65,7 +65,7 @@ class ProfilingManager {
void PluginUnInit() const;
Status CallMsprofReport(NotNull<ReporterData *> reporter_data) const;
void QueryHashId(const int32_t &device_id, const std::string &src_str, uint64_t *hash_id);
const struct MsprofCallback &GetMsprofCallback() { return prof_cb_; }
const struct MsprofCallback &GetMsprofCallback() const { return prof_cb_; }
void SetMsprofCtrlCallback(MsprofCtrlCallback func) { prof_cb_.msprofCtrlCallback = func; }
void SetMsprofReporterCallback(MsprofReporterCallback func) { prof_cb_.msprofReporterCallback = func; }
void SetMsprofSetDeviceCallback(MsprofSetDeviceCallback func) { prof_cb_.msprofSetDeviceCallback = func; }

1
mindspore/python/mindspore/profiler/common/util.py
View File

@ -123,6 +123,7 @@ def get_file_join_name(input_path, file_name):
file_join_name = os.path.join(input_path, '%s.join' % file_name)
if os.path.exists(file_join_name):
os.remove(file_join_name)
file_join_name = os.path.realpath(file_join_name)
with open(file_join_name, 'ab') as bin_data:
for i in name_list:
file = input_path + os.sep + i

6
mindspore/python/mindspore/profiler/parser/flops_parser.py
View File

@ -214,7 +214,7 @@ class FlopsParser:
raise ProfilerRawFileException(
'aicore.data file does not have enough content to be parsed'
)
aicore_file_path = os.path.realpath(aicore_file_path)
try:
with open(aicore_file_path, "rb") as aicore_file:
all_log_struct = aicore_file.read(self.AICORE_LOG_SIZE * read_count)
@ -292,7 +292,7 @@ class FlopsParser:
if not os.path.exists(optime_file_path):
logger.critical(f'The {optime_file_path} file does not exist.')
raise ProfilerFileNotFoundException(optime_file_path)
optime_file_path = os.path.realpath(optime_file_path)
try:
with open(optime_file_path, 'r') as f:
lines = f.readlines()
@ -336,7 +336,7 @@ class FlopsParser:
scope_list.insert(0, "Total")
scope_depth = len(scope_list)
for idx in range(scope_depth - 1):
key_name = scope_list[idx] + " " + scope_list[idx + 1]
key_name = '{} {}'.format(scope_list[idx], scope_list[idx + 1])
self._flops_each_scope.setdefault(key_name, 0)
self._flops_each_scope[key_name] += task_fops

2
mindspore/python/mindspore/profiler/parser/memory_usage_parser.py
View File

@ -222,7 +222,7 @@ class GraphMemoryParser:
self.graph.nodes = self._parse_nodes(nodes_proto)
self._process_memory_breakdowns()
self.graph.breakdowns = self.breakdowns[self.tensor_node_id]
self.graph.breakdowns = [self.breakdowns[self.tensor_node_id]]
# update fp_start and bp_end
point_id = self._locate_fp_bp_id()

2
mindspore/python/mindspore/profiler/parser/minddata_analyzer.py
View File

@ -348,7 +348,7 @@ class MinddataProfilingAnalyzer:
for op_info in reversed(pipeline_info['op_info']):
op_id = op_info.get('op_id')
op_name = op_info.get('op_type')[0:-2]
dict_opid_pipeline_ops[op_id] = op_name + '(id=' + str(op_id) + ')'
dict_opid_pipeline_ops[op_id] = '{}(id={})'.format(op_name, str(op_id))
dict_opid_opname[op_id] = op_name
dict_opid_numworkers[op_id] = op_info.get('num_workers')

2
mindspore/python/mindspore/profiler/parser/step_trace_parser.py
View File

@ -597,7 +597,7 @@ class AscendStepTraceParser(BaseStepTraceParser):
if len(binary_data) < step_trace_size:
break
unpacked_data = StructType.unpack_binary_data(TS_TRACK_STEP_TRACE_STRUCT, binary_data)
if unpacked_data['rptType'] != STEP_TRACE_RPT_TYPE:
if unpacked_data.get('rptType') != STEP_TRACE_RPT_TYPE:
continue
ts_tracks.append(unpacked_data)
except (IOError, OSError) as err:

138
mindspore/python/mindspore/profiler/profiling.py
View File

@ -147,6 +147,7 @@ class Profiler:
self._has_started_twice = False
self.start_profile = True
self._profile_memory = False
self._stop_time = 0
# Setup and start MindData Profiling
self._md_profiler = cde.GlobalContext.profiling_manager()
@ -165,39 +166,60 @@ class Profiler:
self._cpu_profiler.init(self._output_path)
if self._device_target and self._device_target == DeviceTarget.CPU.value:
if context.get_context("mode") == context.PYNATIVE_MODE:
raise RuntimeError("Pynative model is not supported on CPU currently.")
self.start_profile = kwargs.pop("start_profile", True)
if not isinstance(self.start_profile, bool):
raise TypeError(f"For '{self.__class__.__name__}', the parameter start_profile must be bool, "
f"but got type {type(self.start_profile)}")
self._cpu_profiler_init(kwargs)
if self._device_target and self._device_target == DeviceTarget.GPU.value:
if context.get_context("mode") == context.PYNATIVE_MODE:
raise RuntimeError("Pynative model is not supported on GPU currently.")
self._parse_parameter_for_gpu(**kwargs)
gpu_profiler = c_expression.GPUProfiler
self._gpu_profiler = gpu_profiler.get_instance()
self._gpu_profiler.init(self._output_path)
if GlobalComm.WORLD_COMM_GROUP == "nccl_world_group":
self._dev_id = str(get_rank())
os.environ['DEVICE_ID'] = self._dev_id
self._gpu_profiler_init(kwargs)
elif self._device_target and self._device_target == DeviceTarget.ASCEND.value:
self._init_time = int(time.time() * 10000000)
logger.info("Profiling: profiling init time: %d", self._init_time)
self._parse_parameter_for_ascend(**kwargs)
os.environ['DEVICE_ID'] = self._dev_id
self._ascend_profiler_init(kwargs)
self._ascend_profiling_options = json.dumps(self._construct_profiling_options())
# Characters longer than 2048 are ignored, resulting in profiling option resolution errors
if len(self._ascend_profiling_options) > 2048:
msg = f"For '{self.__class__.__name__}', the environment parameter length exceeds " \
f"the limit (2048), please input valid parameters."
logger.critical(msg)
raise ValueError(msg)
def _cpu_profiler_init(self, kwargs):
"""Cpu profiler init."""
if context.get_context("mode") == context.PYNATIVE_MODE:
raise RuntimeError("Pynative model is not supported on CPU currently.")
self.start_profile = kwargs.pop("start_profile", True)
if not isinstance(self.start_profile, bool):
raise TypeError(f"For '{self.__class__.__name__}', the parameter start_profile must be bool, "
f"but got type {type(self.start_profile)}")
def _gpu_profiler_init(self, kwargs):
"""Gpu profiler init."""
if context.get_context("mode") == context.PYNATIVE_MODE:
raise RuntimeError("Pynative model is not supported on GPU currently.")
self._parse_parameter_for_gpu(kwargs)
gpu_profiler = c_expression.GPUProfiler
self._gpu_profiler = gpu_profiler.get_instance()
self._gpu_profiler.init(self._output_path)
if GlobalComm.WORLD_COMM_GROUP == "nccl_world_group":
self._dev_id = str(get_rank())
os.environ['DEVICE_ID'] = self._dev_id
def _ascend_profiler_init(self, kwargs):
"""Ascend profiler init."""
self._init_time = int(time.time() * 10000000)
logger.info("Profiling: profiling init time: %d", self._init_time)
self._parse_parameter_for_ascend(kwargs)
os.environ['DEVICE_ID'] = self._dev_id
self._ascend_profiling_options = json.dumps(self._construct_profiling_options())
# Characters longer than 2048 are ignored, resulting in profiling option resolution errors
if len(self._ascend_profiling_options) > 2048:
msg = f"For '{self.__class__.__name__}', the environment parameter length exceeds " \
f"the limit (2048), please input valid parameters."
logger.critical(msg)
raise ValueError(msg)
# use context interface to open profiling, for the new mindspore version(after 2020.5.21)
self._ascend_profiler = c_expression.AscendProfiler.get_instance()
self._ascend_profiler.init(self._output_path, int(self._dev_id), self._ascend_profiling_options)
base_profiling_container_path = os.path.join(self._output_path, "container")
container_path = os.path.join(base_profiling_container_path, self._dev_id)
data_path = os.path.join(container_path, "data")
data_path = validate_and_normalize_path(data_path)
if not os.path.exists(data_path):
os.makedirs(data_path, exist_ok=True)
def _construct_profiling_options(self):
"""
@ -227,7 +249,7 @@ class Profiler:
return profiling_options
def _parse_parameter_for_gpu(self, **kwargs):
def _parse_parameter_for_gpu(self, kwargs):
"""Parse parameter in Proflier when the device target is GPU."""
self.start_profile = kwargs.pop("start_profile", True)
@ -249,7 +271,7 @@ class Profiler:
if self._profile_memory:
raise RuntimeError(f"The parameter profile_memory is not supported on GPU currently.")
def _parse_parameter_for_ascend(self, **kwargs):
def _parse_parameter_for_ascend(self, kwargs):
"""Parse parameter in Proflier when the device target is Ascend."""
self.start_profile = kwargs.pop("start_profile", True)
if not isinstance(self.start_profile, bool):
@ -358,6 +380,28 @@ class Profiler:
else:
self._ascend_graph_analyse()
def _ascend_graph_memory_analyse(self, points):
"""Analyse memory usage info."""
if self._profile_memory:
try:
logger.info("Profiling: analyzing the memory usage info.")
self._analyse_memory_usage(points)
except (ProfilerIOException, ProfilerFileNotFoundException, ProfilerRawFileException) as err:
logger.warning(err.message)
finally:
pass
def _ascend_graph_hccl_analyse(self):
"""Analyse hccl profiler info."""
if self._profile_communication:
try:
logger.info("Profiling: analyzing the hccl profiler info.")
self._analyse_hccl_info()
except (ProfilerIOException, ProfilerFileNotFoundException, ProfilerRawFileException) as err:
logger.warning(err.message)
finally:
pass
def _ascend_graph_op_analyse(self, source_path):
"""
Ascend graph model hwts analyse.
@ -476,25 +520,8 @@ class Profiler:
finally:
pass
# analyse memory usage info
if self._profile_memory:
try:
logger.info("Profiling: analyzing the memory usage info.")
self._analyse_memory_usage(points)
except (ProfilerIOException, ProfilerFileNotFoundException, ProfilerRawFileException) as err:
logger.warning(err.message)
finally:
pass
# analyse hccl profiler info
if self._profile_communication:
try:
logger.info("Profiling: analyzing the hccl profiler info.")
self._analyse_hccl_info()
except (ProfilerIOException, ProfilerFileNotFoundException, ProfilerRawFileException) as err:
logger.warning(err.message)
finally:
pass
self._ascend_graph_memory_analyse(points)
self._ascend_graph_hccl_analyse()
# get op FLOPs from aicore.data.x.slice.0 file, and compute FLOPS, write output_op_flops_x.txt
flops_parser = FlopsParser(source_path, self._output_path, op_task_dict,
@ -581,23 +608,10 @@ class Profiler:
pynative_profiler = c_expression.PynativeProfiler
self._pynative_profiler = pynative_profiler.get_instance()
self._pynative_profiler.init(self._output_path)
self._ascend_profiler = c_expression.AscendProfiler.get_instance()
self._ascend_profiler.init(self._output_path, int(self._dev_id), self._ascend_profiling_options)
self._ascend_profiler.start()
def _ascend_graph_start(self):
"""Ascend graph mode start profiling."""
# use context interface to open profiling, for the new mindspore version(after 2020.5.21)
self._ascend_profiler = c_expression.AscendProfiler.get_instance()
self._ascend_profiler.init(self._output_path, int(self._dev_id), self._ascend_profiling_options)
base_profiling_container_path = os.path.join(self._output_path, "container")
container_path = os.path.join(base_profiling_container_path, self._dev_id)
data_path = os.path.join(container_path, "data")
data_path = validate_and_normalize_path(data_path)
if not os.path.exists(data_path):
os.makedirs(data_path, exist_ok=True)
self._ascend_profiler.start()
def stop(self):