Update deprecated api parameters and documentation
This commit is contained in:
mahequn123
parent
ac4365c38a
commit
dae3e26fdb
|
|
@ -18,8 +18,6 @@ profiler模块简介。
|
|||
**参数：**
|
||||
|
||||
- **output_path** (str) – 表示输出数据的路径。
|
||||
- **optypes_not_deal** (str) – （仅限Ascend）该参数已弃用，该功能已不再支持。
|
||||
- **ascend_job_id** (str) – （仅限Ascend）该参数已弃用，该功能已不再支持。
|
||||
- **profile_communication** (bool) – （仅限Ascend）表示是否在多设备训练中收集通信性能数据。当值为True时，收集这些数据。默认值为False。在单台设备训练中，该参数的设置无效。
|
||||
- **profile_memory** (bool) – （仅限Ascend）表示是否收集Tensor内存数据。当值为True时，收集这些数据。默认值为False。
|
||||
- **start_profile** (bool) – 该参数控制是否在Profiler初始化的时候开启采集数据。默认值为True。
|
||||
|
|
@ -32,19 +30,6 @@ profiler模块简介。
|
|||
|
||||
收集和分析训练后或训练期间调用的性能数据。样例如上所示。
|
||||
|
||||
.. py:method:: profile(network,profile_option)
|
||||
|
||||
获取训练网络中可训练参数的数量。
|
||||
|
||||
**参数：**
|
||||
|
||||
- **network** (Cell) - 表示训练网络。
|
||||
- **profile_option** (ProfileOption) - 该参数已弃用，该功能已不再支持。
|
||||
|
||||
**返回：**
|
||||
|
||||
dict，其中key为选项名称，value为选项结果。
|
||||
|
||||
.. py:method:: start()
|
||||
|
||||
开启Profiler数据采集，可以按条件开启Profiler。
|
||||
|
|
@ -62,7 +47,3 @@ profiler模块简介。
|
|||
**异常：**
|
||||
|
||||
- **RuntimeError** – profiler没有开启。
|
||||
|
||||
.. py:class:: mindspore.profiler.ProfileOption
|
||||
|
||||
这个类已经弃用，该功能已不再支持。
|
||||
|
|
@ -18,11 +18,10 @@ Profiler Module Introduction.
|
|||
This module provides Python APIs to enable the profiling of MindSpore neural networks.
|
||||
Users can import the mindspore.profiler.Profiler, initialize the Profiler object to start profiling,
|
||||
and use Profiler.analyse() to stop profiling and analyse the results.
|
||||
To visualize the profiling results, users can open MindSpore Web, find the corresponding 'run'
|
||||
button/option and click the profile link.
|
||||
Now, Profiler supports the AICore operator analysis.
|
||||
Users can visualize the results using the MindInsight tool.
|
||||
Now, Profiler supports AICore operator, AICpu operator, HostCpu operator, memory,
|
||||
correspondence, cluster, etc data analysis.
|
||||
"""
|
||||
from mindspore.profiler.profiling import Profiler
|
||||
from mindspore.profiler.profiling import ProfileOption
|
||||
|
||||
__all__ = ["Profiler", "ProfileOption"]
|
||||
__all__ = ["Profiler"]
|
||||
|
|
|
|||
|
|
@ -17,9 +17,7 @@ import os
|
|||
import stat
|
||||
import time
|
||||
import json
|
||||
from enum import Enum
|
||||
|
||||
from mindspore.nn.cell import Cell
|
||||
from mindspore import log as logger, context
|
||||
from mindspore.communication.management import GlobalComm, get_rank, get_group_size
|
||||
import mindspore._c_expression as c_expression
|
||||
|
|
@ -50,45 +48,30 @@ from mindspore.profiler.parser.op_intermediate_parser import OPIntermediateParse
|
|||
INIT_OP_NAME = 'Default/InitDataSetQueue'
|
||||
|
||||
|
||||
def deprecated(name, version):
|
||||
"""Warning notices."""
|
||||
msg = f"The {name} is deprecated from MindSpore {version} and will be removed in a future version."
|
||||
logger.warning(msg)
|
||||
|
||||
|
||||
def _environment_check():
|
||||
if c_expression.security.enable_security():
|
||||
raise RuntimeError("Profiler is not supported if compiled with \'-s on\'")
|
||||
|
||||
|
||||
class ProfileOption(Enum):
|
||||
"""This Class is deprecated. Profile Option Enum which be used in Profiler.profile."""
|
||||
trainable_parameters = 0
|
||||
|
||||
|
||||
class Profiler:
|
||||
"""
|
||||
Performance profiling API.
|
||||
|
||||
This API enables MindSpore users to profile the performance of neural network.
|
||||
Profiler supports Ascend and GPU, both of them are used in the same way,
|
||||
but only output_path in args works on GPU. And it can only be initialized once.
|
||||
Profiler supports Ascend and GPU, both of them are used in the same way.
|
||||
|
||||
Args:
|
||||
output_path (str): Output data path.
|
||||
optypes_not_deal (str): This parameter is deprecated. (Ascend only) Op type names, determine the data of
|
||||
which optype should be collected and analysed, will deal with all op if null.
|
||||
Different op types should be separated by comma.
|
||||
ascend_job_id (str): This parameter is deprecated. (Ascend only) The directory where the profiling files
|
||||
to be parsed are located. This parameter is used to support offline parsing.
|
||||
profile_communication (bool): Whether to collect communication performance data in a multi devices training,
|
||||
collect when True. Default is False. Setting this parameter has no effect during single device training.
|
||||
profile_communication (bool): (Ascend only) Whether to collect communication
|
||||
performance data in a multi devices training, collect when True. Default is False.
|
||||
Setting this parameter has no effect during single device training.
|
||||
profile_memory (bool): Whether to collect tensor memory data, collect when True. Default is False.
|
||||
start_profile (bool): The start_profile parameter controls whether to enable or disable performance data
|
||||
collection based on conditions. The default value is True.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If ascend_job_id is transferred with data that does not match the current MindSpore version.
|
||||
RuntimeError: When the version of CANN does not match the version of MindSpore,
|
||||
MindSpore cannot parse the generated ascend_job_id directory structure.
|
||||
|
||||
Examples:
|
||||
>>> import numpy as np
|
||||
|
|
@ -147,6 +130,14 @@ class Profiler:
|
|||
msg = "Do not init twice in the profiler."
|
||||
raise RuntimeError(msg)
|
||||
Profiler._has_initialized = True
|
||||
self._dev_id = None
|
||||
self._cpu_profiler = None
|
||||
self._gpu_profiler = None
|
||||
self._init_time = None
|
||||
self._ascend_job_id = ''
|
||||
self._job_id_env = None
|
||||
self._filt_optype_names = ''
|
||||
self._output_path = ''
|
||||
_environment_check()
|
||||
# get device_id and device_target
|
||||
self._get_devid_rankid_and_devtarget()
|
||||
|
|
@ -160,7 +151,14 @@ class Profiler:
|
|||
# Setup and start MindData Profiling
|
||||
self._md_profiler = cde.GlobalContext.profiling_manager()
|
||||
self._md_profiler.init()
|
||||
self._decide_device_target(kwargs)
|
||||
if self.start_profile:
|
||||
self.start()
|
||||
elif context.get_context("mode") == context.PYNATIVE_MODE:
|
||||
raise RuntimeError("Pynative model does not support conditional collection of performance data.")
|
||||
|
||||
def _decide_device_target(self, kwargs):
|
||||
"""Complete Profiler initialization according to device_target"""
|
||||
if self._device_target:
|
||||
cpu_profiler = c_expression.CPUProfiler
|
||||
self._cpu_profiler = cpu_profiler.get_instance()
|
||||
|
|
@ -201,11 +199,6 @@ class Profiler:
|
|||
logger.critical(msg)
|
||||
raise ValueError(msg)
|
||||
|
||||
if self.start_profile:
|
||||
self.start()
|
||||
elif context.get_context("mode") == context.PYNATIVE_MODE:
|
||||
raise RuntimeError("Pynative model does not support conditional collection of performance data.")
|
||||
|
||||
def _construct_profiling_options(self):
|
||||
"""
|
||||
Construct profiling options to determine which profiling data should be collected.
|
||||
|
|
@ -258,24 +251,6 @@ class Profiler:
|
|||
|
||||
def _parse_parameter_for_ascend(self, **kwargs):
|
||||
"""Parse parameter in Proflier when the device target is Ascend."""
|
||||
if 'optypes_not_deal' in kwargs:
|
||||
deprecated('optypes_not_deal', '1.6')
|
||||
optypes_not_deal = kwargs.pop("optypes_not_deal", "Variable")
|
||||
if not isinstance(optypes_not_deal, str):
|
||||
raise TypeError(f"For '{self.__class__.__name__}', the parameter optypes_not_deal "
|
||||
f"must be str, but got type {type(optypes_not_deal)}")
|
||||
self._filt_optype_names = optypes_not_deal.split(",") if optypes_not_deal else []
|
||||
|
||||
self._ascend_job_id = kwargs.pop("ascend_job_id", "")
|
||||
if self._ascend_job_id:
|
||||
deprecated('ascend_job_id', '1.6')
|
||||
self._ascend_job_id = validate_and_normalize_path(self._ascend_job_id)
|
||||
if not os.path.exists(self._ascend_job_id):
|
||||
msg = f"Invalid ascend_job_id: {self._ascend_job_id}, Please pass the absolute path of the JOB dir"
|
||||
logger.critical(msg)
|
||||
raise ValueError(msg)
|
||||
self._output_path, _ = os.path.split(self._ascend_job_id)
|
||||
|
||||
self.start_profile = kwargs.pop("start_profile", True)
|
||||
if not isinstance(self.start_profile, bool):
|
||||
raise TypeError(f"For '{self.__class__.__name__}', the parameter start_profile must be bool, "
|
||||
|
|
@ -303,6 +278,15 @@ class Profiler:
|
|||
if task_sink and task_sink == "1":
|
||||
logger.warning("Profiling is not supported when task is not sink.")
|
||||
|
||||
def _set_ascend_job_id(self, ascend_job_id):
|
||||
"""Set output_path for offline parsing performance data."""
|
||||
self._ascend_job_id = validate_and_normalize_path(ascend_job_id)
|
||||
if not os.path.exists(self._ascend_job_id):
|
||||
msg = f"Invalid ascend_job_id: {self._ascend_job_id}, Please pass the absolute path of the JOB dir"
|
||||
logger.critical(msg)
|
||||
raise ValueError(msg)
|
||||
self._output_path, _ = os.path.split(self._ascend_job_id)
|
||||
|
||||
def _is_offline_parser(self):
|
||||
"""Return whether offline parser or online parser."""
|
||||
if self._device_target and self._device_target == DeviceTarget.ASCEND.value:
|
||||
|
|
@ -1100,30 +1084,3 @@ class Profiler:
|
|||
hccl_parse = HcclParser(hccl_path, self._dev_id, self._rank_id, self._output_path)
|
||||
hccl_parse.parse()
|
||||
logger.info("Analyse hccl info successfully.")
|
||||
|
||||
@staticmethod
|
||||
def profile(network, profile_option):
|
||||
"""
|
||||
Get the number of trainable parameters in the training network.
|
||||
|
||||
Args:
|
||||
network (Cell): The training network.
|
||||
profile_option (ProfileOption): The profile option.
|
||||
|
||||
Returns:
|
||||
dict, the key is the option name, the value is the result of option.
|
||||
"""
|
||||
deprecated('profile', '1.6')
|
||||
if not profile_option:
|
||||
raise ValueError("The parameter profile_option must pass a value using ProfileOption.")
|
||||
|
||||
if profile_option == ProfileOption.trainable_parameters:
|
||||
if not isinstance(network, Cell):
|
||||
msg = "Profiling: The network should be an object of nn.Cell"
|
||||
raise ValueError(msg)
|
||||
param_nums = len(network.parameters_dict())
|
||||
result = {"trainable_parameters": param_nums}
|
||||
else:
|
||||
raise ValueError("profile_option value must be ProfileOption.trainable_parameters")
|
||||
|
||||
return result
|
||||
|
|
|
|||
Loading…
Reference in New Issue