From c7ed65afa8cd54faddda7471f350e96212c446c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=8D=97?= Date: Thu, 16 Dec 2021 16:49:38 +0800 Subject: [PATCH] model amp comments --- docs/api/api_python/mindspore/mindspore.Model.rst | 7 +++++-- mindspore/train/amp.py | 4 ++-- mindspore/train/model.py | 11 +++++++---- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/api/api_python/mindspore/mindspore.Model.rst b/docs/api/api_python/mindspore/mindspore.Model.rst index 188502821fd..0b979b64ba3 100644 --- a/docs/api/api_python/mindspore/mindspore.Model.rst +++ b/docs/api/api_python/mindspore/mindspore.Model.rst @@ -17,9 +17,12 @@ - O0: 无变化。 - O2: 将网络精度转为float16，batchnorm保持float32精度，使用动态调整梯度放大系数（loss scale）的策略。 - - O3: 将网络精度转为float16，并为 `mindspore.build_train_network` 接口配置属性 `keep_batchnorm_fp32=False`。 - - auto: 为不同处理器设置专家推荐的混合精度等级，如在GPU上设为O2，在Ascend上设为O3。该设置方式不适用于所有场景，建议用户根据具体的网络模型自定义设置 `amp_level` 。在GPU上建议使用O2，在Ascend上建议使用O3。关于 `amp_level` 详见 `mindpore.build_train_network`。 + - O3: 将网络精度（包括batchnorm）转为float16，不使用梯度调整策略。 + - auto: 为不同处理器设置专家推荐的混合精度等级，如在GPU上设为O2，在Ascend上设为O3。该设置方式可能在部分场景下不适用，建议用户根据具体的网络模型自定义设置 `amp_level` 。 + 在GPU上建议使用O2，在Ascend上建议使用O3。 + 通过`kwargs`设置`keep_batchnorm_fp32`，可修改batchnorm策略，`keep_batchnorm_fp32`必须为bool类型；通过`kwargs`设置`loss_scale_manager`可修改梯度放大策略，`loss_scale_manager`必须为:class:`mindspore.LossScaleManager`的子类， + 关于 `amp_level` 详见 `mindpore.build_train_network`。 **样例:** diff --git a/mindspore/train/amp.py b/mindspore/train/amp.py index 76e7ef0fdd7..a3a1d91a124 100644 --- a/mindspore/train/amp.py +++ b/mindspore/train/amp.py @@ -153,8 +153,8 @@ def build_train_network(network, optimizer, loss_fn=None, level='O0', boost_leve level to O3 Ascend. The recommended level is chosen by the export experience, cannot always general. User should specify the level for special network. - O2 is recommended on GPU, O3 is recommended on Ascend.Property of `keep_batchnorm_fp32` , `cast_model_type` - and `loss_scale_manager` determined by `level` setting may be overwritten by settings in `kwargs` . + O2 is recommended on GPU, O3 is recommended on Ascend. Property of `keep_batchnorm_fp32`, `cast_model_type` + and `loss_scale_manager` determined by `level` setting may be overwritten by settings in `kwargs`. boost_level (str): Option for argument `level` in `mindspore.boost` , level for boost mode training. Supports ["O0", "O1", "O2"]. Default: "O0". diff --git a/mindspore/train/model.py b/mindspore/train/model.py index 782b01a3316..6476c42a1cb 100644 --- a/mindspore/train/model.py +++ b/mindspore/train/model.py @@ -140,14 +140,17 @@ class Model: - O0: Do not change. - O2: Cast network to float16, keep batchnorm run in float32, using dynamic loss scale. - - O3: Cast network to float16 and add property `keep_batchnorm_fp32=False` to - :func:`mindspore.build_train_network`. + - O3: Cast network to float16, the batchnorm is also cast to float16, loss scale will not be used. - auto: Set level to recommended level in different devices. Set level to O2 on GPU, set level to O3 on Ascend. The recommended level is chosen by the export experience, not applicable to all scenarios. User should specify the level for special network. - O2 is recommended on GPU, O3 is recommended on Ascend. The more detailed explanation of `amp_level` setting - can be found at `mindspore.build_train_network`. + O2 is recommended on GPU, O3 is recommended on Ascend. + The batchnorm strategy can be changed by `keep_batchnorm_fp32` settings in `kwargs`. `keep_batchnorm_fp32` + must be a bool. The loss scale strategy can be changed by `loss_scale_manager` setting in `kwargs`. + `loss_scale_manager` should be a subclass of :class:`mindspore.LossScaleManager`. + The more detailed explanation of `amp_level` setting can be found at `mindspore.build_train_network`. + boost_level (str): Option for argument `level` in `mindspore.boost`, level for boost mode training. Supports ["O0", "O1", "O2"]. Default: "O0".