From dae3e26fdbdb1d3f0f0ce77f391bc1f1381c7ee8 Mon Sep 17 00:00:00 2001 From: mahequn123 Date: Tue, 1 Mar 2022 23:01:41 +0800 Subject: [PATCH] Update deprecated api parameters and documentation --- docs/api/api_python/mindspore.profiler.rst | 19 ---- .../python/mindspore/profiler/__init__.py | 9 +- .../python/mindspore/profiler/profiling.py | 103 +++++------------- 3 files changed, 34 insertions(+), 97 deletions(-) diff --git a/docs/api/api_python/mindspore.profiler.rst b/docs/api/api_python/mindspore.profiler.rst index 357879c7956..11d1d1000f0 100644 --- a/docs/api/api_python/mindspore.profiler.rst +++ b/docs/api/api_python/mindspore.profiler.rst @@ -18,8 +18,6 @@ profiler模块简介。 **参数：** - **output_path** (str) – 表示输出数据的路径。 - - **optypes_not_deal** (str) – （仅限Ascend）该参数已弃用，该功能已不再支持。 - - **ascend_job_id** (str) – （仅限Ascend）该参数已弃用，该功能已不再支持。 - **profile_communication** (bool) – （仅限Ascend）表示是否在多设备训练中收集通信性能数据。当值为True时，收集这些数据。默认值为False。在单台设备训练中，该参数的设置无效。 - **profile_memory** (bool) – （仅限Ascend）表示是否收集Tensor内存数据。当值为True时，收集这些数据。默认值为False。 - **start_profile** (bool) – 该参数控制是否在Profiler初始化的时候开启采集数据。默认值为True。 @@ -32,19 +30,6 @@ profiler模块简介。 收集和分析训练后或训练期间调用的性能数据。样例如上所示。 - .. py:method:: profile(network,profile_option) - - 获取训练网络中可训练参数的数量。 - - **参数：** - - - **network** (Cell) - 表示训练网络。 - - **profile_option** (ProfileOption) - 该参数已弃用，该功能已不再支持。 - - **返回：** - - dict，其中key为选项名称，value为选项结果。 - .. py:method:: start() 开启Profiler数据采集，可以按条件开启Profiler。 @@ -62,7 +47,3 @@ profiler模块简介。 **异常：** - **RuntimeError** – profiler没有开启。 - -.. py:class:: mindspore.profiler.ProfileOption - - 这个类已经弃用，该功能已不再支持。 \ No newline at end of file diff --git a/mindspore/python/mindspore/profiler/__init__.py b/mindspore/python/mindspore/profiler/__init__.py index 5c67de0a934..e68f45b44c1 100644 --- a/mindspore/python/mindspore/profiler/__init__.py +++ b/mindspore/python/mindspore/profiler/__init__.py @@ -18,11 +18,10 @@ Profiler Module Introduction. This module provides Python APIs to enable the profiling of MindSpore neural networks. Users can import the mindspore.profiler.Profiler, initialize the Profiler object to start profiling, and use Profiler.analyse() to stop profiling and analyse the results. -To visualize the profiling results, users can open MindSpore Web, find the corresponding 'run' -button/option and click the profile link. -Now, Profiler supports the AICore operator analysis. +Users can visualize the results using the MindInsight tool. +Now, Profiler supports AICore operator, AICpu operator, HostCpu operator, memory, +correspondence, cluster, etc data analysis. """ from mindspore.profiler.profiling import Profiler -from mindspore.profiler.profiling import ProfileOption -__all__ = ["Profiler", "ProfileOption"] +__all__ = ["Profiler"] diff --git a/mindspore/python/mindspore/profiler/profiling.py b/mindspore/python/mindspore/profiler/profiling.py index 8031d6ceba5..4a286ed282d 100644 --- a/mindspore/python/mindspore/profiler/profiling.py +++ b/mindspore/python/mindspore/profiler/profiling.py @@ -17,9 +17,7 @@ import os import stat import time import json -from enum import Enum -from mindspore.nn.cell import Cell from mindspore import log as logger, context from mindspore.communication.management import GlobalComm, get_rank, get_group_size import mindspore._c_expression as c_expression @@ -50,45 +48,30 @@ from mindspore.profiler.parser.op_intermediate_parser import OPIntermediateParse INIT_OP_NAME = 'Default/InitDataSetQueue' -def deprecated(name, version): - """Warning notices.""" - msg = f"The {name} is deprecated from MindSpore {version} and will be removed in a future version." - logger.warning(msg) - - def _environment_check(): if c_expression.security.enable_security(): raise RuntimeError("Profiler is not supported if compiled with \'-s on\'") -class ProfileOption(Enum): - """This Class is deprecated. Profile Option Enum which be used in Profiler.profile.""" - trainable_parameters = 0 - - class Profiler: """ Performance profiling API. This API enables MindSpore users to profile the performance of neural network. - Profiler supports Ascend and GPU, both of them are used in the same way, - but only output_path in args works on GPU. And it can only be initialized once. + Profiler supports Ascend and GPU, both of them are used in the same way. Args: output_path (str): Output data path. - optypes_not_deal (str): This parameter is deprecated. (Ascend only) Op type names, determine the data of - which optype should be collected and analysed, will deal with all op if null. - Different op types should be separated by comma. - ascend_job_id (str): This parameter is deprecated. (Ascend only) The directory where the profiling files - to be parsed are located. This parameter is used to support offline parsing. - profile_communication (bool): Whether to collect communication performance data in a multi devices training, - collect when True. Default is False. Setting this parameter has no effect during single device training. + profile_communication (bool): (Ascend only) Whether to collect communication + performance data in a multi devices training, collect when True. Default is False. + Setting this parameter has no effect during single device training. profile_memory (bool): Whether to collect tensor memory data, collect when True. Default is False. start_profile (bool): The start_profile parameter controls whether to enable or disable performance data collection based on conditions. The default value is True. Raises: - RuntimeError: If ascend_job_id is transferred with data that does not match the current MindSpore version. + RuntimeError: When the version of CANN does not match the version of MindSpore, + MindSpore cannot parse the generated ascend_job_id directory structure. Examples: >>> import numpy as np @@ -147,6 +130,14 @@ class Profiler: msg = "Do not init twice in the profiler." raise RuntimeError(msg) Profiler._has_initialized = True + self._dev_id = None + self._cpu_profiler = None + self._gpu_profiler = None + self._init_time = None + self._ascend_job_id = '' + self._job_id_env = None + self._filt_optype_names = '' + self._output_path = '' _environment_check() # get device_id and device_target self._get_devid_rankid_and_devtarget() @@ -160,7 +151,14 @@ class Profiler: # Setup and start MindData Profiling self._md_profiler = cde.GlobalContext.profiling_manager() self._md_profiler.init() + self._decide_device_target(kwargs) + if self.start_profile: + self.start() + elif context.get_context("mode") == context.PYNATIVE_MODE: + raise RuntimeError("Pynative model does not support conditional collection of performance data.") + def _decide_device_target(self, kwargs): + """Complete Profiler initialization according to device_target""" if self._device_target: cpu_profiler = c_expression.CPUProfiler self._cpu_profiler = cpu_profiler.get_instance() @@ -201,11 +199,6 @@ class Profiler: logger.critical(msg) raise ValueError(msg) - if self.start_profile: - self.start() - elif context.get_context("mode") == context.PYNATIVE_MODE: - raise RuntimeError("Pynative model does not support conditional collection of performance data.") - def _construct_profiling_options(self): """ Construct profiling options to determine which profiling data should be collected. @@ -258,24 +251,6 @@ class Profiler: def _parse_parameter_for_ascend(self, **kwargs): """Parse parameter in Proflier when the device target is Ascend.""" - if 'optypes_not_deal' in kwargs: - deprecated('optypes_not_deal', '1.6') - optypes_not_deal = kwargs.pop("optypes_not_deal", "Variable") - if not isinstance(optypes_not_deal, str): - raise TypeError(f"For '{self.__class__.__name__}', the parameter optypes_not_deal " - f"must be str, but got type {type(optypes_not_deal)}") - self._filt_optype_names = optypes_not_deal.split(",") if optypes_not_deal else [] - - self._ascend_job_id = kwargs.pop("ascend_job_id", "") - if self._ascend_job_id: - deprecated('ascend_job_id', '1.6') - self._ascend_job_id = validate_and_normalize_path(self._ascend_job_id) - if not os.path.exists(self._ascend_job_id): - msg = f"Invalid ascend_job_id: {self._ascend_job_id}, Please pass the absolute path of the JOB dir" - logger.critical(msg) - raise ValueError(msg) - self._output_path, _ = os.path.split(self._ascend_job_id) - self.start_profile = kwargs.pop("start_profile", True) if not isinstance(self.start_profile, bool): raise TypeError(f"For '{self.__class__.__name__}', the parameter start_profile must be bool, " @@ -303,6 +278,15 @@ class Profiler: if task_sink and task_sink == "1": logger.warning("Profiling is not supported when task is not sink.") + def _set_ascend_job_id(self, ascend_job_id): + """Set output_path for offline parsing performance data.""" + self._ascend_job_id = validate_and_normalize_path(ascend_job_id) + if not os.path.exists(self._ascend_job_id): + msg = f"Invalid ascend_job_id: {self._ascend_job_id}, Please pass the absolute path of the JOB dir" + logger.critical(msg) + raise ValueError(msg) + self._output_path, _ = os.path.split(self._ascend_job_id) + def _is_offline_parser(self): """Return whether offline parser or online parser.""" if self._device_target and self._device_target == DeviceTarget.ASCEND.value: @@ -1100,30 +1084,3 @@ class Profiler: hccl_parse = HcclParser(hccl_path, self._dev_id, self._rank_id, self._output_path) hccl_parse.parse() logger.info("Analyse hccl info successfully.") - - @staticmethod - def profile(network, profile_option): - """ - Get the number of trainable parameters in the training network. - - Args: - network (Cell): The training network. - profile_option (ProfileOption): The profile option. - - Returns: - dict, the key is the option name, the value is the result of option. - """ - deprecated('profile', '1.6') - if not profile_option: - raise ValueError("The parameter profile_option must pass a value using ProfileOption.") - - if profile_option == ProfileOption.trainable_parameters: - if not isinstance(network, Cell): - msg = "Profiling: The network should be an object of nn.Cell" - raise ValueError(msg) - param_nums = len(network.parameters_dict()) - result = {"trainable_parameters": param_nums} - else: - raise ValueError("profile_option value must be ProfileOption.trainable_parameters") - - return result