Update deprecated api parameters and documentation
This commit is contained in:
parent
ac4365c38a
commit
dae3e26fdb
|
@ -18,8 +18,6 @@ profiler模块简介。
|
||||||
**参数:**
|
**参数:**
|
||||||
|
|
||||||
- **output_path** (str) – 表示输出数据的路径。
|
- **output_path** (str) – 表示输出数据的路径。
|
||||||
- **optypes_not_deal** (str) – (仅限Ascend)该参数已弃用,该功能已不再支持。
|
|
||||||
- **ascend_job_id** (str) – (仅限Ascend)该参数已弃用,该功能已不再支持。
|
|
||||||
- **profile_communication** (bool) – (仅限Ascend)表示是否在多设备训练中收集通信性能数据。当值为True时,收集这些数据。默认值为False。在单台设备训练中,该参数的设置无效。
|
- **profile_communication** (bool) – (仅限Ascend)表示是否在多设备训练中收集通信性能数据。当值为True时,收集这些数据。默认值为False。在单台设备训练中,该参数的设置无效。
|
||||||
- **profile_memory** (bool) – (仅限Ascend)表示是否收集Tensor内存数据。当值为True时,收集这些数据。默认值为False。
|
- **profile_memory** (bool) – (仅限Ascend)表示是否收集Tensor内存数据。当值为True时,收集这些数据。默认值为False。
|
||||||
- **start_profile** (bool) – 该参数控制是否在Profiler初始化的时候开启采集数据。默认值为True。
|
- **start_profile** (bool) – 该参数控制是否在Profiler初始化的时候开启采集数据。默认值为True。
|
||||||
|
@ -32,19 +30,6 @@ profiler模块简介。
|
||||||
|
|
||||||
收集和分析训练后或训练期间调用的性能数据。样例如上所示。
|
收集和分析训练后或训练期间调用的性能数据。样例如上所示。
|
||||||
|
|
||||||
.. py:method:: profile(network,profile_option)
|
|
||||||
|
|
||||||
获取训练网络中可训练参数的数量。
|
|
||||||
|
|
||||||
**参数:**
|
|
||||||
|
|
||||||
- **network** (Cell) - 表示训练网络。
|
|
||||||
- **profile_option** (ProfileOption) - 该参数已弃用,该功能已不再支持。
|
|
||||||
|
|
||||||
**返回:**
|
|
||||||
|
|
||||||
dict,其中key为选项名称,value为选项结果。
|
|
||||||
|
|
||||||
.. py:method:: start()
|
.. py:method:: start()
|
||||||
|
|
||||||
开启Profiler数据采集,可以按条件开启Profiler。
|
开启Profiler数据采集,可以按条件开启Profiler。
|
||||||
|
@ -62,7 +47,3 @@ profiler模块简介。
|
||||||
**异常:**
|
**异常:**
|
||||||
|
|
||||||
- **RuntimeError** – profiler没有开启。
|
- **RuntimeError** – profiler没有开启。
|
||||||
|
|
||||||
.. py:class:: mindspore.profiler.ProfileOption
|
|
||||||
|
|
||||||
这个类已经弃用,该功能已不再支持。
|
|
|
@ -18,11 +18,10 @@ Profiler Module Introduction.
|
||||||
This module provides Python APIs to enable the profiling of MindSpore neural networks.
|
This module provides Python APIs to enable the profiling of MindSpore neural networks.
|
||||||
Users can import the mindspore.profiler.Profiler, initialize the Profiler object to start profiling,
|
Users can import the mindspore.profiler.Profiler, initialize the Profiler object to start profiling,
|
||||||
and use Profiler.analyse() to stop profiling and analyse the results.
|
and use Profiler.analyse() to stop profiling and analyse the results.
|
||||||
To visualize the profiling results, users can open MindSpore Web, find the corresponding 'run'
|
Users can visualize the results using the MindInsight tool.
|
||||||
button/option and click the profile link.
|
Now, Profiler supports AICore operator, AICpu operator, HostCpu operator, memory,
|
||||||
Now, Profiler supports the AICore operator analysis.
|
correspondence, cluster, etc data analysis.
|
||||||
"""
|
"""
|
||||||
from mindspore.profiler.profiling import Profiler
|
from mindspore.profiler.profiling import Profiler
|
||||||
from mindspore.profiler.profiling import ProfileOption
|
|
||||||
|
|
||||||
__all__ = ["Profiler", "ProfileOption"]
|
__all__ = ["Profiler"]
|
||||||
|
|
|
@ -17,9 +17,7 @@ import os
|
||||||
import stat
|
import stat
|
||||||
import time
|
import time
|
||||||
import json
|
import json
|
||||||
from enum import Enum
|
|
||||||
|
|
||||||
from mindspore.nn.cell import Cell
|
|
||||||
from mindspore import log as logger, context
|
from mindspore import log as logger, context
|
||||||
from mindspore.communication.management import GlobalComm, get_rank, get_group_size
|
from mindspore.communication.management import GlobalComm, get_rank, get_group_size
|
||||||
import mindspore._c_expression as c_expression
|
import mindspore._c_expression as c_expression
|
||||||
|
@ -50,45 +48,30 @@ from mindspore.profiler.parser.op_intermediate_parser import OPIntermediateParse
|
||||||
INIT_OP_NAME = 'Default/InitDataSetQueue'
|
INIT_OP_NAME = 'Default/InitDataSetQueue'
|
||||||
|
|
||||||
|
|
||||||
def deprecated(name, version):
|
|
||||||
"""Warning notices."""
|
|
||||||
msg = f"The {name} is deprecated from MindSpore {version} and will be removed in a future version."
|
|
||||||
logger.warning(msg)
|
|
||||||
|
|
||||||
|
|
||||||
def _environment_check():
|
def _environment_check():
|
||||||
if c_expression.security.enable_security():
|
if c_expression.security.enable_security():
|
||||||
raise RuntimeError("Profiler is not supported if compiled with \'-s on\'")
|
raise RuntimeError("Profiler is not supported if compiled with \'-s on\'")
|
||||||
|
|
||||||
|
|
||||||
class ProfileOption(Enum):
|
|
||||||
"""This Class is deprecated. Profile Option Enum which be used in Profiler.profile."""
|
|
||||||
trainable_parameters = 0
|
|
||||||
|
|
||||||
|
|
||||||
class Profiler:
|
class Profiler:
|
||||||
"""
|
"""
|
||||||
Performance profiling API.
|
Performance profiling API.
|
||||||
|
|
||||||
This API enables MindSpore users to profile the performance of neural network.
|
This API enables MindSpore users to profile the performance of neural network.
|
||||||
Profiler supports Ascend and GPU, both of them are used in the same way,
|
Profiler supports Ascend and GPU, both of them are used in the same way.
|
||||||
but only output_path in args works on GPU. And it can only be initialized once.
|
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
output_path (str): Output data path.
|
output_path (str): Output data path.
|
||||||
optypes_not_deal (str): This parameter is deprecated. (Ascend only) Op type names, determine the data of
|
profile_communication (bool): (Ascend only) Whether to collect communication
|
||||||
which optype should be collected and analysed, will deal with all op if null.
|
performance data in a multi devices training, collect when True. Default is False.
|
||||||
Different op types should be separated by comma.
|
Setting this parameter has no effect during single device training.
|
||||||
ascend_job_id (str): This parameter is deprecated. (Ascend only) The directory where the profiling files
|
|
||||||
to be parsed are located. This parameter is used to support offline parsing.
|
|
||||||
profile_communication (bool): Whether to collect communication performance data in a multi devices training,
|
|
||||||
collect when True. Default is False. Setting this parameter has no effect during single device training.
|
|
||||||
profile_memory (bool): Whether to collect tensor memory data, collect when True. Default is False.
|
profile_memory (bool): Whether to collect tensor memory data, collect when True. Default is False.
|
||||||
start_profile (bool): The start_profile parameter controls whether to enable or disable performance data
|
start_profile (bool): The start_profile parameter controls whether to enable or disable performance data
|
||||||
collection based on conditions. The default value is True.
|
collection based on conditions. The default value is True.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
RuntimeError: If ascend_job_id is transferred with data that does not match the current MindSpore version.
|
RuntimeError: When the version of CANN does not match the version of MindSpore,
|
||||||
|
MindSpore cannot parse the generated ascend_job_id directory structure.
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
>>> import numpy as np
|
>>> import numpy as np
|
||||||
|
@ -147,6 +130,14 @@ class Profiler:
|
||||||
msg = "Do not init twice in the profiler."
|
msg = "Do not init twice in the profiler."
|
||||||
raise RuntimeError(msg)
|
raise RuntimeError(msg)
|
||||||
Profiler._has_initialized = True
|
Profiler._has_initialized = True
|
||||||
|
self._dev_id = None
|
||||||
|
self._cpu_profiler = None
|
||||||
|
self._gpu_profiler = None
|
||||||
|
self._init_time = None
|
||||||
|
self._ascend_job_id = ''
|
||||||
|
self._job_id_env = None
|
||||||
|
self._filt_optype_names = ''
|
||||||
|
self._output_path = ''
|
||||||
_environment_check()
|
_environment_check()
|
||||||
# get device_id and device_target
|
# get device_id and device_target
|
||||||
self._get_devid_rankid_and_devtarget()
|
self._get_devid_rankid_and_devtarget()
|
||||||
|
@ -160,7 +151,14 @@ class Profiler:
|
||||||
# Setup and start MindData Profiling
|
# Setup and start MindData Profiling
|
||||||
self._md_profiler = cde.GlobalContext.profiling_manager()
|
self._md_profiler = cde.GlobalContext.profiling_manager()
|
||||||
self._md_profiler.init()
|
self._md_profiler.init()
|
||||||
|
self._decide_device_target(kwargs)
|
||||||
|
if self.start_profile:
|
||||||
|
self.start()
|
||||||
|
elif context.get_context("mode") == context.PYNATIVE_MODE:
|
||||||
|
raise RuntimeError("Pynative model does not support conditional collection of performance data.")
|
||||||
|
|
||||||
|
def _decide_device_target(self, kwargs):
|
||||||
|
"""Complete Profiler initialization according to device_target"""
|
||||||
if self._device_target:
|
if self._device_target:
|
||||||
cpu_profiler = c_expression.CPUProfiler
|
cpu_profiler = c_expression.CPUProfiler
|
||||||
self._cpu_profiler = cpu_profiler.get_instance()
|
self._cpu_profiler = cpu_profiler.get_instance()
|
||||||
|
@ -201,11 +199,6 @@ class Profiler:
|
||||||
logger.critical(msg)
|
logger.critical(msg)
|
||||||
raise ValueError(msg)
|
raise ValueError(msg)
|
||||||
|
|
||||||
if self.start_profile:
|
|
||||||
self.start()
|
|
||||||
elif context.get_context("mode") == context.PYNATIVE_MODE:
|
|
||||||
raise RuntimeError("Pynative model does not support conditional collection of performance data.")
|
|
||||||
|
|
||||||
def _construct_profiling_options(self):
|
def _construct_profiling_options(self):
|
||||||
"""
|
"""
|
||||||
Construct profiling options to determine which profiling data should be collected.
|
Construct profiling options to determine which profiling data should be collected.
|
||||||
|
@ -258,24 +251,6 @@ class Profiler:
|
||||||
|
|
||||||
def _parse_parameter_for_ascend(self, **kwargs):
|
def _parse_parameter_for_ascend(self, **kwargs):
|
||||||
"""Parse parameter in Proflier when the device target is Ascend."""
|
"""Parse parameter in Proflier when the device target is Ascend."""
|
||||||
if 'optypes_not_deal' in kwargs:
|
|
||||||
deprecated('optypes_not_deal', '1.6')
|
|
||||||
optypes_not_deal = kwargs.pop("optypes_not_deal", "Variable")
|
|
||||||
if not isinstance(optypes_not_deal, str):
|
|
||||||
raise TypeError(f"For '{self.__class__.__name__}', the parameter optypes_not_deal "
|
|
||||||
f"must be str, but got type {type(optypes_not_deal)}")
|
|
||||||
self._filt_optype_names = optypes_not_deal.split(",") if optypes_not_deal else []
|
|
||||||
|
|
||||||
self._ascend_job_id = kwargs.pop("ascend_job_id", "")
|
|
||||||
if self._ascend_job_id:
|
|
||||||
deprecated('ascend_job_id', '1.6')
|
|
||||||
self._ascend_job_id = validate_and_normalize_path(self._ascend_job_id)
|
|
||||||
if not os.path.exists(self._ascend_job_id):
|
|
||||||
msg = f"Invalid ascend_job_id: {self._ascend_job_id}, Please pass the absolute path of the JOB dir"
|
|
||||||
logger.critical(msg)
|
|
||||||
raise ValueError(msg)
|
|
||||||
self._output_path, _ = os.path.split(self._ascend_job_id)
|
|
||||||
|
|
||||||
self.start_profile = kwargs.pop("start_profile", True)
|
self.start_profile = kwargs.pop("start_profile", True)
|
||||||
if not isinstance(self.start_profile, bool):
|
if not isinstance(self.start_profile, bool):
|
||||||
raise TypeError(f"For '{self.__class__.__name__}', the parameter start_profile must be bool, "
|
raise TypeError(f"For '{self.__class__.__name__}', the parameter start_profile must be bool, "
|
||||||
|
@ -303,6 +278,15 @@ class Profiler:
|
||||||
if task_sink and task_sink == "1":
|
if task_sink and task_sink == "1":
|
||||||
logger.warning("Profiling is not supported when task is not sink.")
|
logger.warning("Profiling is not supported when task is not sink.")
|
||||||
|
|
||||||
|
def _set_ascend_job_id(self, ascend_job_id):
|
||||||
|
"""Set output_path for offline parsing performance data."""
|
||||||
|
self._ascend_job_id = validate_and_normalize_path(ascend_job_id)
|
||||||
|
if not os.path.exists(self._ascend_job_id):
|
||||||
|
msg = f"Invalid ascend_job_id: {self._ascend_job_id}, Please pass the absolute path of the JOB dir"
|
||||||
|
logger.critical(msg)
|
||||||
|
raise ValueError(msg)
|
||||||
|
self._output_path, _ = os.path.split(self._ascend_job_id)
|
||||||
|
|
||||||
def _is_offline_parser(self):
|
def _is_offline_parser(self):
|
||||||
"""Return whether offline parser or online parser."""
|
"""Return whether offline parser or online parser."""
|
||||||
if self._device_target and self._device_target == DeviceTarget.ASCEND.value:
|
if self._device_target and self._device_target == DeviceTarget.ASCEND.value:
|
||||||
|
@ -1100,30 +1084,3 @@ class Profiler:
|
||||||
hccl_parse = HcclParser(hccl_path, self._dev_id, self._rank_id, self._output_path)
|
hccl_parse = HcclParser(hccl_path, self._dev_id, self._rank_id, self._output_path)
|
||||||
hccl_parse.parse()
|
hccl_parse.parse()
|
||||||
logger.info("Analyse hccl info successfully.")
|
logger.info("Analyse hccl info successfully.")
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def profile(network, profile_option):
|
|
||||||
"""
|
|
||||||
Get the number of trainable parameters in the training network.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
network (Cell): The training network.
|
|
||||||
profile_option (ProfileOption): The profile option.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
dict, the key is the option name, the value is the result of option.
|
|
||||||
"""
|
|
||||||
deprecated('profile', '1.6')
|
|
||||||
if not profile_option:
|
|
||||||
raise ValueError("The parameter profile_option must pass a value using ProfileOption.")
|
|
||||||
|
|
||||||
if profile_option == ProfileOption.trainable_parameters:
|
|
||||||
if not isinstance(network, Cell):
|
|
||||||
msg = "Profiling: The network should be an object of nn.Cell"
|
|
||||||
raise ValueError(msg)
|
|
||||||
param_nums = len(network.parameters_dict())
|
|
||||||
result = {"trainable_parameters": param_nums}
|
|
||||||
else:
|
|
||||||
raise ValueError("profile_option value must be ProfileOption.trainable_parameters")
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
Loading…
Reference in New Issue