Update deprecated api parameters and documentation

This commit is contained in:
mahequn123 2022-03-01 23:01:41 +08:00
parent ac4365c38a
commit dae3e26fdb
3 changed files with 34 additions and 97 deletions

View File

@ -18,8 +18,6 @@ profiler模块简介。
**参数:** **参数:**
- **output_path** (str) 表示输出数据的路径。 - **output_path** (str) 表示输出数据的路径。
- **optypes_not_deal** (str) 仅限Ascend该参数已弃用该功能已不再支持。
- **ascend_job_id** (str) 仅限Ascend该参数已弃用该功能已不再支持。
- **profile_communication** (bool) 仅限Ascend表示是否在多设备训练中收集通信性能数据。当值为True时收集这些数据。默认值为False。在单台设备训练中该参数的设置无效。 - **profile_communication** (bool) 仅限Ascend表示是否在多设备训练中收集通信性能数据。当值为True时收集这些数据。默认值为False。在单台设备训练中该参数的设置无效。
- **profile_memory** (bool) 仅限Ascend表示是否收集Tensor内存数据。当值为True时收集这些数据。默认值为False。 - **profile_memory** (bool) 仅限Ascend表示是否收集Tensor内存数据。当值为True时收集这些数据。默认值为False。
- **start_profile** (bool) 该参数控制是否在Profiler初始化的时候开启采集数据。默认值为True。 - **start_profile** (bool) 该参数控制是否在Profiler初始化的时候开启采集数据。默认值为True。
@ -32,19 +30,6 @@ profiler模块简介。
收集和分析训练后或训练期间调用的性能数据。样例如上所示。 收集和分析训练后或训练期间调用的性能数据。样例如上所示。
.. py:method:: profile(network,profile_option)
获取训练网络中可训练参数的数量。
**参数:**
- **network** (Cell) - 表示训练网络。
- **profile_option** (ProfileOption) - 该参数已弃用,该功能已不再支持。
**返回:**
dict其中key为选项名称value为选项结果。
.. py:method:: start() .. py:method:: start()
开启Profiler数据采集可以按条件开启Profiler。 开启Profiler数据采集可以按条件开启Profiler。
@ -62,7 +47,3 @@ profiler模块简介。
**异常:** **异常:**
- **RuntimeError** profiler没有开启。 - **RuntimeError** profiler没有开启。
.. py:class:: mindspore.profiler.ProfileOption
这个类已经弃用,该功能已不再支持。

View File

@ -18,11 +18,10 @@ Profiler Module Introduction.
This module provides Python APIs to enable the profiling of MindSpore neural networks. This module provides Python APIs to enable the profiling of MindSpore neural networks.
Users can import the mindspore.profiler.Profiler, initialize the Profiler object to start profiling, Users can import the mindspore.profiler.Profiler, initialize the Profiler object to start profiling,
and use Profiler.analyse() to stop profiling and analyse the results. and use Profiler.analyse() to stop profiling and analyse the results.
To visualize the profiling results, users can open MindSpore Web, find the corresponding 'run' Users can visualize the results using the MindInsight tool.
button/option and click the profile link. Now, Profiler supports AICore operator, AICpu operator, HostCpu operator, memory,
Now, Profiler supports the AICore operator analysis. correspondence, cluster, etc data analysis.
""" """
from mindspore.profiler.profiling import Profiler from mindspore.profiler.profiling import Profiler
from mindspore.profiler.profiling import ProfileOption
__all__ = ["Profiler", "ProfileOption"] __all__ = ["Profiler"]

View File

@ -17,9 +17,7 @@ import os
import stat import stat
import time import time
import json import json
from enum import Enum
from mindspore.nn.cell import Cell
from mindspore import log as logger, context from mindspore import log as logger, context
from mindspore.communication.management import GlobalComm, get_rank, get_group_size from mindspore.communication.management import GlobalComm, get_rank, get_group_size
import mindspore._c_expression as c_expression import mindspore._c_expression as c_expression
@ -50,45 +48,30 @@ from mindspore.profiler.parser.op_intermediate_parser import OPIntermediateParse
INIT_OP_NAME = 'Default/InitDataSetQueue' INIT_OP_NAME = 'Default/InitDataSetQueue'
def deprecated(name, version):
"""Warning notices."""
msg = f"The {name} is deprecated from MindSpore {version} and will be removed in a future version."
logger.warning(msg)
def _environment_check(): def _environment_check():
if c_expression.security.enable_security(): if c_expression.security.enable_security():
raise RuntimeError("Profiler is not supported if compiled with \'-s on\'") raise RuntimeError("Profiler is not supported if compiled with \'-s on\'")
class ProfileOption(Enum):
"""This Class is deprecated. Profile Option Enum which be used in Profiler.profile."""
trainable_parameters = 0
class Profiler: class Profiler:
""" """
Performance profiling API. Performance profiling API.
This API enables MindSpore users to profile the performance of neural network. This API enables MindSpore users to profile the performance of neural network.
Profiler supports Ascend and GPU, both of them are used in the same way, Profiler supports Ascend and GPU, both of them are used in the same way.
but only output_path in args works on GPU. And it can only be initialized once.
Args: Args:
output_path (str): Output data path. output_path (str): Output data path.
optypes_not_deal (str): This parameter is deprecated. (Ascend only) Op type names, determine the data of profile_communication (bool): (Ascend only) Whether to collect communication
which optype should be collected and analysed, will deal with all op if null. performance data in a multi devices training, collect when True. Default is False.
Different op types should be separated by comma. Setting this parameter has no effect during single device training.
ascend_job_id (str): This parameter is deprecated. (Ascend only) The directory where the profiling files
to be parsed are located. This parameter is used to support offline parsing.
profile_communication (bool): Whether to collect communication performance data in a multi devices training,
collect when True. Default is False. Setting this parameter has no effect during single device training.
profile_memory (bool): Whether to collect tensor memory data, collect when True. Default is False. profile_memory (bool): Whether to collect tensor memory data, collect when True. Default is False.
start_profile (bool): The start_profile parameter controls whether to enable or disable performance data start_profile (bool): The start_profile parameter controls whether to enable or disable performance data
collection based on conditions. The default value is True. collection based on conditions. The default value is True.
Raises: Raises:
RuntimeError: If ascend_job_id is transferred with data that does not match the current MindSpore version. RuntimeError: When the version of CANN does not match the version of MindSpore,
MindSpore cannot parse the generated ascend_job_id directory structure.
Examples: Examples:
>>> import numpy as np >>> import numpy as np
@ -147,6 +130,14 @@ class Profiler:
msg = "Do not init twice in the profiler." msg = "Do not init twice in the profiler."
raise RuntimeError(msg) raise RuntimeError(msg)
Profiler._has_initialized = True Profiler._has_initialized = True
self._dev_id = None
self._cpu_profiler = None
self._gpu_profiler = None
self._init_time = None
self._ascend_job_id = ''
self._job_id_env = None
self._filt_optype_names = ''
self._output_path = ''
_environment_check() _environment_check()
# get device_id and device_target # get device_id and device_target
self._get_devid_rankid_and_devtarget() self._get_devid_rankid_and_devtarget()
@ -160,7 +151,14 @@ class Profiler:
# Setup and start MindData Profiling # Setup and start MindData Profiling
self._md_profiler = cde.GlobalContext.profiling_manager() self._md_profiler = cde.GlobalContext.profiling_manager()
self._md_profiler.init() self._md_profiler.init()
self._decide_device_target(kwargs)
if self.start_profile:
self.start()
elif context.get_context("mode") == context.PYNATIVE_MODE:
raise RuntimeError("Pynative model does not support conditional collection of performance data.")
def _decide_device_target(self, kwargs):
"""Complete Profiler initialization according to device_target"""
if self._device_target: if self._device_target:
cpu_profiler = c_expression.CPUProfiler cpu_profiler = c_expression.CPUProfiler
self._cpu_profiler = cpu_profiler.get_instance() self._cpu_profiler = cpu_profiler.get_instance()
@ -201,11 +199,6 @@ class Profiler:
logger.critical(msg) logger.critical(msg)
raise ValueError(msg) raise ValueError(msg)
if self.start_profile:
self.start()
elif context.get_context("mode") == context.PYNATIVE_MODE:
raise RuntimeError("Pynative model does not support conditional collection of performance data.")
def _construct_profiling_options(self): def _construct_profiling_options(self):
""" """
Construct profiling options to determine which profiling data should be collected. Construct profiling options to determine which profiling data should be collected.
@ -258,24 +251,6 @@ class Profiler:
def _parse_parameter_for_ascend(self, **kwargs): def _parse_parameter_for_ascend(self, **kwargs):
"""Parse parameter in Proflier when the device target is Ascend.""" """Parse parameter in Proflier when the device target is Ascend."""
if 'optypes_not_deal' in kwargs:
deprecated('optypes_not_deal', '1.6')
optypes_not_deal = kwargs.pop("optypes_not_deal", "Variable")
if not isinstance(optypes_not_deal, str):
raise TypeError(f"For '{self.__class__.__name__}', the parameter optypes_not_deal "
f"must be str, but got type {type(optypes_not_deal)}")
self._filt_optype_names = optypes_not_deal.split(",") if optypes_not_deal else []
self._ascend_job_id = kwargs.pop("ascend_job_id", "")
if self._ascend_job_id:
deprecated('ascend_job_id', '1.6')
self._ascend_job_id = validate_and_normalize_path(self._ascend_job_id)
if not os.path.exists(self._ascend_job_id):
msg = f"Invalid ascend_job_id: {self._ascend_job_id}, Please pass the absolute path of the JOB dir"
logger.critical(msg)
raise ValueError(msg)
self._output_path, _ = os.path.split(self._ascend_job_id)
self.start_profile = kwargs.pop("start_profile", True) self.start_profile = kwargs.pop("start_profile", True)
if not isinstance(self.start_profile, bool): if not isinstance(self.start_profile, bool):
raise TypeError(f"For '{self.__class__.__name__}', the parameter start_profile must be bool, " raise TypeError(f"For '{self.__class__.__name__}', the parameter start_profile must be bool, "
@ -303,6 +278,15 @@ class Profiler:
if task_sink and task_sink == "1": if task_sink and task_sink == "1":
logger.warning("Profiling is not supported when task is not sink.") logger.warning("Profiling is not supported when task is not sink.")
def _set_ascend_job_id(self, ascend_job_id):
"""Set output_path for offline parsing performance data."""
self._ascend_job_id = validate_and_normalize_path(ascend_job_id)
if not os.path.exists(self._ascend_job_id):
msg = f"Invalid ascend_job_id: {self._ascend_job_id}, Please pass the absolute path of the JOB dir"
logger.critical(msg)
raise ValueError(msg)
self._output_path, _ = os.path.split(self._ascend_job_id)
def _is_offline_parser(self): def _is_offline_parser(self):
"""Return whether offline parser or online parser.""" """Return whether offline parser or online parser."""
if self._device_target and self._device_target == DeviceTarget.ASCEND.value: if self._device_target and self._device_target == DeviceTarget.ASCEND.value:
@ -1100,30 +1084,3 @@ class Profiler:
hccl_parse = HcclParser(hccl_path, self._dev_id, self._rank_id, self._output_path) hccl_parse = HcclParser(hccl_path, self._dev_id, self._rank_id, self._output_path)
hccl_parse.parse() hccl_parse.parse()
logger.info("Analyse hccl info successfully.") logger.info("Analyse hccl info successfully.")
@staticmethod
def profile(network, profile_option):
"""
Get the number of trainable parameters in the training network.
Args:
network (Cell): The training network.
profile_option (ProfileOption): The profile option.
Returns:
dict, the key is the option name, the value is the result of option.
"""
deprecated('profile', '1.6')
if not profile_option:
raise ValueError("The parameter profile_option must pass a value using ProfileOption.")
if profile_option == ProfileOption.trainable_parameters:
if not isinstance(network, Cell):
msg = "Profiling: The network should be an object of nn.Cell"
raise ValueError(msg)
param_nums = len(network.parameters_dict())
result = {"trainable_parameters": param_nums}
else:
raise ValueError("profile_option value must be ProfileOption.trainable_parameters")
return result