From 568ba4091831d4f8a6af56e7baf99e554384dd6e Mon Sep 17 00:00:00 2001 From: xumengjuan1 Date: Thu, 24 Nov 2022 16:58:30 +0800 Subject: [PATCH] modify doc --- RELEASE.md | 2 +- RELEASE_CN.md | 2 +- .../mindspore.dataset.Dataset.map.rst | 6 +- docs/api/api_python/mindspore.dataset.rst | 6 +- docs/api/api_python/mindspore.rst | 137 ++++++++--------- docs/api/api_python/mindspore.train.rst | 9 ++ .../mindspore.train.ConvertModelUtils.rst} | 6 +- .../mindspore.train.ConvertNetUtils.rst} | 4 +- docs/api/api_python_en/mindspore.rst | 143 ++++++++---------- docs/api/api_python_en/mindspore.train.rst | 11 ++ .../python/mindspore/dataset/__init__.py | 9 +- .../mindspore/dataset/engine/datasets.py | 9 +- 12 files changed, 176 insertions(+), 168 deletions(-) rename docs/api/api_python/{mindspore/mindspore.ConvertModelUtils.rst => train/mindspore.train.ConvertModelUtils.rst} (95%) rename docs/api/api_python/{mindspore/mindspore.ConvertNetUtils.rst => train/mindspore.train.ConvertNetUtils.rst} (84%) diff --git a/RELEASE.md b/RELEASE.md index 2d2c97c48cf..759a80e5aa1 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -191,7 +191,7 @@ Contributions of any kind are welcome! - [STABLE] When using the map operation for dataset objects and the parameters like: num_parallel_workers > 1 and python_multiprocessing=True, the multi-process mechanism is optimized, so that the data channel and child processes are mapped one by one, avoiding excessive file handle occupation, and closing_pool interface is also deleted. - [STABLE] Add a batch of Vision, Text and Audio data augmentation operations. - [STABLE] Fix a bug where the flat_map method of the Dataset class does not flatten the result. -- [STABLE] Unify import paths of dataset augmentation APIs to provide more easier way to use. Refer to [latest api usages](https://www.mindspore.cn/docs/en/master/api_python/mindspore.dataset.vision.html). +- [STABLE] Unify import paths of dataset augmentation APIs to provide more easier way to use. Refer to [latest api usages](https://www.mindspore.cn/docs/en/r1.8/api_python/mindspore.dataset.vision.html). ### API Change diff --git a/RELEASE_CN.md b/RELEASE_CN.md index e89f8a81668..bf78f93a949 100644 --- a/RELEASE_CN.md +++ b/RELEASE_CN.md @@ -191,7 +191,7 @@ AGroupofProbiotocs, anzhengqi, askmiao, baihuawei, baiyangfan, bai-yangfan, bing - [STABLE] 对于数据集对象使用map操作时，同时num_parallel_workers>1并且python_multiprocessing=True时，进行了多进程的机制优化，使得数据通道与子进程一一映射，避免了过多的文件句柄占用，同时close_pool这个接口也被删除。 - [STABLE] 新增一批Vision、Text和Audio类数据增强操作。 - [STABLE] 修复数据集类的flat_map方法未将结果展平的错误。 -- [STABLE] 统一数据集增强API的导入路径，提供更简单的使用方法，请参阅[最新的API用法](https://www.mindspore.cn/docs/zh-CN/master/api_python/mindspore.dataset.vision.html)。 +- [STABLE] 统一数据集增强API的导入路径，提供更简单的使用方法，请参阅[最新的API用法](https://www.mindspore.cn/docs/zh-CN/r1.8/api_python/mindspore.dataset.vision.html)。 ### API变更 diff --git a/docs/api/api_python/dataset/dataset_method/operation/mindspore.dataset.Dataset.map.rst b/docs/api/api_python/dataset/dataset_method/operation/mindspore.dataset.Dataset.map.rst index c1006b209ae..0f59fd4fc27 100644 --- a/docs/api/api_python/dataset/dataset_method/operation/mindspore.dataset.Dataset.map.rst +++ b/docs/api/api_python/dataset/dataset_method/operation/mindspore.dataset.Dataset.map.rst @@ -12,9 +12,9 @@ mindspore.dataset.Dataset.map 最后一个数据增强的输出列的列名由 `output_columns` 指定，如果没有指定 `output_columns` ，输出列名与 `input_columns` 一致。 - 如果使用的是 `mindspore` `dataset` 提供的数据增强（ - `vision类 `_ ， - `nlp类 `_ ， - `audio类 `_ ），请使用如下参数： + `vision类 `_ ， + `nlp类 `_ ， + `audio类 `_ ），请使用如下参数： .. image:: map_parameter_cn.png diff --git a/docs/api/api_python/mindspore.dataset.rst b/docs/api/api_python/mindspore.dataset.rst index ba576b15be7..309e9db27cd 100644 --- a/docs/api/api_python/mindspore.dataset.rst +++ b/docs/api/api_python/mindspore.dataset.rst @@ -38,9 +38,9 @@ mindspore.dataset - 数据集操作（filter/ skip）：用户通过数据集对象方法 `.shuffle` / `.filter` / `.skip` / `.split` / `.take` / … 来实现数据集的进一步混洗、过滤、跳过、最多获取条数等操作； - 数据集样本增强操作（map）：用户可以将数据增强操作 - （`vision类 `_ ， - `nlp类 `_ ， - `audio类 `_ ） + （`vision类 `_ ， + `nlp类 `_ ， + `audio类 `_ ） 添加到map操作中执行，数据预处理过程中可以定义多个map操作，用于执行不同增强操作，数据增强操作也可以是 用户自定义增强的 `PyFunc` ； - 批（batch）：用户在样本完成增强后，使用 `.batch` 操作将多个样本组织成batch，也可以通过batch的参数 `per_batch_map` diff --git a/docs/api/api_python/mindspore.rst b/docs/api/api_python/mindspore.rst index 55dec41b6f1..77e799aa0d7 100644 --- a/docs/api/api_python/mindspore.rst +++ b/docs/api/api_python/mindspore.rst @@ -1,8 +1,11 @@ mindspore ========= +数据表达 +--------- + 张量 ------- +^^^^^ .. mscnautosummary:: :toctree: mindspore @@ -14,7 +17,7 @@ mindspore mindspore.SparseTensor 参数 ---------- +^^^^^ .. mscnautosummary:: :toctree: mindspore @@ -23,7 +26,7 @@ mindspore mindspore.ParameterTuple 数据类型 --------- +^^^^^^^^^ .. mscnautosummary:: :toctree: mindspore @@ -34,36 +37,6 @@ mindspore mindspore.pytype_to_dtype mindspore.get_py_obj_dtype -随机种子 ---------- - -.. mscnautosummary:: - :toctree: mindspore - - mindspore.set_seed - mindspore.get_seed - -自动微分 ----------------- - -.. mscnautosummary:: - :toctree: mindspore - - mindspore.grad - mindspore.value_and_grad - mindspore.jacfwd - mindspore.jacrev - mindspore.jvp - mindspore.vjp - -自动向量化 ----------------- - -.. mscnautosummary:: - :toctree: mindspore - - mindspore.vmap - 运行环境 --------- @@ -83,23 +56,14 @@ mindspore mindspore.get_algo_parameters mindspore.reset_algo_parameters -并行 -------------------- +随机种子 +--------- .. mscnautosummary:: :toctree: mindspore - mindspore.shard - -数据处理工具 -------------------- - -.. mscnautosummary:: - :toctree: mindspore - - mindspore.DatasetHelper - mindspore.connect_network_with_dataset - mindspore.data_sink + mindspore.set_seed + mindspore.get_seed 序列化 ------- @@ -125,17 +89,37 @@ mindspore mindspore.transform_checkpoint_by_rank mindspore.transform_checkpoints -调试调优 ----------- +自动微分 +---------------- .. mscnautosummary:: :toctree: mindspore - mindspore.Profiler - mindspore.SummaryCollector - mindspore.SummaryLandscape - mindspore.SummaryRecord - mindspore.set_dump + mindspore.grad + mindspore.value_and_grad + mindspore.jacfwd + mindspore.jacrev + mindspore.jvp + mindspore.vjp + +并行优化 +--------- + +自动向量化 +^^^^^^^^^^^ + +.. mscnautosummary:: + :toctree: mindspore + + mindspore.vmap + +并行 +^^^^^ + +.. mscnautosummary:: + :toctree: mindspore + + mindspore.shard 即时编译 -------- @@ -148,10 +132,36 @@ mindspore mindspore.jit_class mindspore.ms_class mindspore.ms_function + mindspore.ms_memory_recycle mindspore.mutable +工具 +----- + +数据处理工具 +^^^^^^^^^^^^^ + +.. mscnautosummary:: + :toctree: mindspore + + mindspore.DatasetHelper + mindspore.connect_network_with_dataset + mindspore.data_sink + +调试调优 +^^^^^^^^^ + +.. mscnautosummary:: + :toctree: mindspore + + mindspore.Profiler + mindspore.SummaryCollector + mindspore.SummaryLandscape + mindspore.SummaryRecord + mindspore.set_dump + 日志 ----- +^^^^^ .. mscnautosummary:: :toctree: mindspore @@ -161,26 +171,9 @@ mindspore 安装验证 --------- +^^^^^^^^^ .. mscnautosummary:: :toctree: mindspore mindspore.run_check - -内存回收 ----------- - -.. mscnautosummary:: - :toctree: mindspore - - mindspore.ms_memory_recycle - -二阶优化 ----------- - -.. mscnautosummary:: - :toctree: mindspore - - mindspore.ConvertModelUtils - mindspore.ConvertNetUtils diff --git a/docs/api/api_python/mindspore.train.rst b/docs/api/api_python/mindspore.train.rst index 697433e7d53..4d1020bd08b 100644 --- a/docs/api/api_python/mindspore.train.rst +++ b/docs/api/api_python/mindspore.train.rst @@ -73,3 +73,12 @@ mindspore.train mindspore.train.get_metric_fn mindspore.train.names mindspore.train.rearrange_inputs + +二阶优化 +---------- + +.. mscnautosummary:: + :toctree: mindspore + + mindspore.train.ConvertModelUtils + mindspore.train.ConvertNetUtils diff --git a/docs/api/api_python/mindspore/mindspore.ConvertModelUtils.rst b/docs/api/api_python/train/mindspore.train.ConvertModelUtils.rst similarity index 95% rename from docs/api/api_python/mindspore/mindspore.ConvertModelUtils.rst rename to docs/api/api_python/train/mindspore.train.ConvertModelUtils.rst index 3f30f2c85d2..7a4785fc235 100644 --- a/docs/api/api_python/mindspore/mindspore.ConvertModelUtils.rst +++ b/docs/api/api_python/train/mindspore.train.ConvertModelUtils.rst @@ -1,7 +1,7 @@ -mindspore.ConvertModelUtils -================================ +mindspore.train.ConvertModelUtils +================================== -.. py:class:: mindspore.ConvertModelUtils +.. py:class:: mindspore.train.ConvertModelUtils 该接口用于增加计算图，提升二阶算法THOR运行时的性能。 diff --git a/docs/api/api_python/mindspore/mindspore.ConvertNetUtils.rst b/docs/api/api_python/train/mindspore.train.ConvertNetUtils.rst similarity index 84% rename from docs/api/api_python/mindspore/mindspore.ConvertNetUtils.rst rename to docs/api/api_python/train/mindspore.train.ConvertNetUtils.rst index 6bd2902fdb3..0b2d7f8ad1a 100644 --- a/docs/api/api_python/mindspore/mindspore.ConvertNetUtils.rst +++ b/docs/api/api_python/train/mindspore.train.ConvertNetUtils.rst @@ -1,7 +1,7 @@ -mindspore.ConvertNetUtils +mindspore.train.ConvertNetUtils ================================ -.. py:class:: mindspore.ConvertNetUtils +.. py:class:: mindspore.train.ConvertNetUtils 将网络转换为thor层网络，用于计算并存储二阶信息矩阵。 diff --git a/docs/api/api_python_en/mindspore.rst b/docs/api/api_python_en/mindspore.rst index 49160455c37..cbc2e7cf087 100644 --- a/docs/api/api_python_en/mindspore.rst +++ b/docs/api/api_python_en/mindspore.rst @@ -1,8 +1,11 @@ mindspore ========= +Data Presentation +------------------ + Tensor ------- +^^^^^^^ .. autosummary:: :toctree: mindspore @@ -16,7 +19,7 @@ Tensor mindspore.SparseTensor Parameter ---------- +^^^^^^^^^^ .. autosummary:: :toctree: mindspore @@ -27,7 +30,7 @@ Parameter mindspore.ParameterTuple DataType --------- +^^^^^^^^^ .. class:: mindspore.dtype @@ -141,42 +144,6 @@ DataType mindspore.pytype_to_dtype mindspore.get_py_obj_dtype -Seed ----- - -.. autosummary:: - :toctree: mindspore - :nosignatures: - :template: classtemplate.rst - - mindspore.set_seed - mindspore.get_seed - -Automatic Differentiation ---------------------------------- - -.. autosummary:: - :toctree: mindspore - :nosignatures: - :template: classtemplate.rst - - mindspore.grad - mindspore.value_and_grad - mindspore.jacfwd - mindspore.jacrev - mindspore.jvp - mindspore.vjp - -Automatic Vectorization ---------------------------------- - -.. autosummary:: - :toctree: mindspore - :nosignatures: - :template: classtemplate.rst - - mindspore.vmap - Context -------- @@ -198,27 +165,16 @@ Context mindspore.get_algo_parameters mindspore.reset_algo_parameters -Parallel ---------------- +Seed +---- .. autosummary:: :toctree: mindspore :nosignatures: :template: classtemplate.rst - mindspore.shard - -Dataset Helper ---------------- - -.. autosummary:: - :toctree: mindspore - :nosignatures: - :template: classtemplate.rst - - mindspore.DatasetHelper - mindspore.connect_network_with_dataset - mindspore.data_sink + mindspore.set_seed + mindspore.get_seed Serialization ------------- @@ -246,6 +202,44 @@ Serialization mindspore.transform_checkpoint_by_rank mindspore.transform_checkpoints +Automatic Differentiation +--------------------------------- + +.. autosummary:: + :toctree: mindspore + :nosignatures: + :template: classtemplate.rst + + mindspore.grad + mindspore.value_and_grad + mindspore.jacfwd + mindspore.jacrev + mindspore.jvp + mindspore.vjp + +Parallel Optimization +----------------------- + +Automatic Vectorization +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. autosummary:: + :toctree: mindspore + :nosignatures: + :template: classtemplate.rst + + mindspore.vmap + +Parallel +^^^^^^^^^^ + +.. autosummary:: + :toctree: mindspore + :nosignatures: + :template: classtemplate.rst + + mindspore.shard + JIT --- @@ -259,31 +253,26 @@ JIT mindspore.jit_class mindspore.ms_class mindspore.ms_function + mindspore.ms_memory_recycle mindspore.mutable -Log ---- +Tool +----- + +Dataset Helper +^^^^^^^^^^^^^^^ .. autosummary:: :toctree: mindspore :nosignatures: :template: classtemplate.rst - mindspore.get_level - mindspore.get_log_config - -Installation Verification --------------------------- - -.. autosummary:: - :toctree: mindspore - :nosignatures: - :template: classtemplate.rst - - mindspore.run_check + mindspore.DatasetHelper + mindspore.connect_network_with_dataset + mindspore.data_sink Debugging and Tuning --------------------------- +^^^^^^^^^^^^^^^^^^^^^ .. autosummary:: :toctree: mindspore @@ -294,25 +283,25 @@ Debugging and Tuning mindspore.SummaryCollector mindspore.SummaryLandscape mindspore.SummaryRecord - mindspore.set_dump + mindspore.set_dump -Memory Recycle --------------------------- +Log +^^^^ .. autosummary:: :toctree: mindspore :nosignatures: :template: classtemplate.rst - mindspore.ms_memory_recycle + mindspore.get_level + mindspore.get_log_config -Thor ---------------- +Installation Verification +^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autosummary:: :toctree: mindspore :nosignatures: :template: classtemplate.rst - mindspore.ConvertModelUtils - mindspore.ConvertNetUtils + mindspore.run_check diff --git a/docs/api/api_python_en/mindspore.train.rst b/docs/api/api_python_en/mindspore.train.rst index 8120e3716a9..f8eb6847962 100644 --- a/docs/api/api_python_en/mindspore.train.rst +++ b/docs/api/api_python_en/mindspore.train.rst @@ -77,3 +77,14 @@ Utils mindspore.train.get_metric_fn mindspore.train.names mindspore.train.rearrange_inputs + +Thor +--------------- + +.. autosummary:: + :toctree: mindspore + :nosignatures: + :template: classtemplate.rst + + mindspore.train.ConvertModelUtils + mindspore.train.ConvertNetUtils diff --git a/mindspore/python/mindspore/dataset/__init__.py b/mindspore/python/mindspore/dataset/__init__.py index 8fe2b891d16..8638ab5eef5 100644 --- a/mindspore/python/mindspore/dataset/__init__.py +++ b/mindspore/python/mindspore/dataset/__init__.py @@ -55,9 +55,12 @@ The specific steps are as follows: - Dataset operation: The user uses the dataset object method `.shuffle` / `.filter` / `.skip` / `.split` / `.take` / ... to further shuffle, filter, skip, and obtain the maximum number of samples of datasets; - Dataset sample transform operation: The user can add data transform operations - ( `vision transform `_ , - `NLP transform `_ , - `audio transform `_ ) to the map + ( `vision transform `_ , + `NLP transform `_ , + `audio transform `_ ) to the map operation to perform transformations. During data preprocessing, multiple map operations can be defined to perform different transform operations to different fields. The data transform operation can also be a user-defined transform `pyfunc` (Python function); diff --git a/mindspore/python/mindspore/dataset/engine/datasets.py b/mindspore/python/mindspore/dataset/engine/datasets.py index 480879d8602..0bfd1f050d1 100644 --- a/mindspore/python/mindspore/dataset/engine/datasets.py +++ b/mindspore/python/mindspore/dataset/engine/datasets.py @@ -811,9 +811,12 @@ class Dataset: `output_columns` , and if not specified, the column name of output column is same as that of `input_columns` . - If you use transformations ( - `vision transform `_ , - `nlp transform `_ , - `audio transform `_ ) + `vision transform `_ , + `nlp transform `_ , + `audio transform `_ ) provided by mindspore dataset, please use the following parameters: .. image:: map_parameter_en.png