From fe296e457aa1bbbbb85d5ed645afd7b8cfadd845 Mon Sep 17 00:00:00 2001 From: huchunmei Date: Tue, 25 May 2021 15:28:51 +0800 Subject: [PATCH] clould --- model_zoo/official/cv/alexnet/README.md | 10 +++- model_zoo/official/cv/alexnet/README_CN.md | 8 ++- model_zoo/official/cv/alexnet/eval.py | 4 +- model_zoo/official/cv/alexnet/src/config.py | 54 ------------------- model_zoo/official/cv/alexnet/src/dataset.py | 9 ++-- model_zoo/official/cv/alexnet/train.py | 4 +- model_zoo/official/cv/lenet/README.md | 7 ++- model_zoo/official/cv/lenet/README_CN.md | 17 +++--- .../official/cv/lenet/default_config.yaml | 10 ++-- model_zoo/official/cv/lenet/export.py | 2 +- model_zoo/official/cv/lenet/postprocess.py | 8 +-- model_zoo/official/cv/lenet/src/config.py | 33 ------------ 12 files changed, 50 insertions(+), 116 deletions(-) delete mode 100644 model_zoo/official/cv/alexnet/src/config.py delete mode 100644 model_zoo/official/cv/lenet/src/config.py diff --git a/model_zoo/official/cv/alexnet/README.md b/model_zoo/official/cv/alexnet/README.md index 34584e2b3fc..50af0ab5ba0 100644 --- a/model_zoo/official/cv/alexnet/README.md +++ b/model_zoo/official/cv/alexnet/README.md @@ -84,9 +84,15 @@ sh run_standalone_eval_ascend.sh [DATA_PATH] [CKPT_NAME] ├── src │ ├──dataset.py // creating dataset │ ├──alexnet.py // alexnet architecture - │ ├──config.py // parameter configuration + │ └──model_utils + │ ├──config.py // Processing configuration parameters + │ ├──device_adapter.py // Get cloud ID + │ ├──local_adapter.py // Get local ID + │ └──moxing_adapter.py // Parameter processing + ├── default_config.yaml // Training parameter profile(cifar10) + ├── config_imagenet.yaml // Training parameter profile(imagenet) ├── train.py // training script - ├── eval.py // evaluation script + ├── eval.py // evaluation script ``` ### [Script Parameters](#contents) diff --git a/model_zoo/official/cv/alexnet/README_CN.md b/model_zoo/official/cv/alexnet/README_CN.md index 4ece36c67e3..59d4dbe49a3 100644 --- a/model_zoo/official/cv/alexnet/README_CN.md +++ b/model_zoo/official/cv/alexnet/README_CN.md @@ -88,7 +88,13 @@ sh run_standalone_eval_ascend.sh [DATA_PATH] [CKPT_NAME] ├── src │ ├──dataset.py // 创建数据集 │ ├──alexnet.py // AlexNet架构 - │ ├──config.py // 参数配置 + | └──model_utils + | ├──config.py // 训练配置 + | ├──device_adapter.py // 获取云上id + | ├──local_adapter.py // 获取本地id + | └──moxing_adapter.py // 参数处理 + ├── default_config.yaml // 训练参数配置文件 + ├── config_imagenet.yaml // 训练参数配置文件 ├── train.py // 训练脚本 ├── eval.py // 评估脚本 ``` diff --git a/model_zoo/official/cv/alexnet/eval.py b/model_zoo/official/cv/alexnet/eval.py index c5d519b559a..b5d061f1729 100644 --- a/model_zoo/official/cv/alexnet/eval.py +++ b/model_zoo/official/cv/alexnet/eval.py @@ -53,7 +53,7 @@ def eval_alexnet(): network = AlexNet(config.num_classes, phase='test') loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") opt = nn.Momentum(network.trainable_params(), config.learning_rate, config.momentum) - ds_eval = create_dataset_cifar10(config.data_path, config.batch_size, status="test", \ + ds_eval = create_dataset_cifar10(config, config.data_path, config.batch_size, status="test", \ target=config.device_target) param_dict = load_checkpoint(config.ckpt_path) print("load checkpoint from [{}].".format(config.ckpt_path)) @@ -64,7 +64,7 @@ def eval_alexnet(): elif config.dataset_name == 'imagenet': network = AlexNet(config.num_classes, phase='test') loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") - ds_eval = create_dataset_imagenet(config.data_path, config.batch_size, training=False) + ds_eval = create_dataset_imagenet(config, config.data_path, config.batch_size, training=False) param_dict = load_checkpoint(config.ckpt_path) print("load checkpoint from [{}].".format(config.ckpt_path)) load_param_into_net(network, param_dict) diff --git a/model_zoo/official/cv/alexnet/src/config.py b/model_zoo/official/cv/alexnet/src/config.py deleted file mode 100644 index 3c653b0cd7b..00000000000 --- a/model_zoo/official/cv/alexnet/src/config.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -network config setting, will be used in train.py -""" - -from easydict import EasyDict as edict - -alexnet_cifar10_cfg = edict({ - 'num_classes': 10, - 'learning_rate': 0.002, - 'momentum': 0.9, - 'epoch_size': 30, - 'batch_size': 32, - 'buffer_size': 1000, - 'image_height': 227, - 'image_width': 227, - 'save_checkpoint_steps': 1562, - 'keep_checkpoint_max': 10, - 'air_name': "alexnet.air", -}) - -alexnet_imagenet_cfg = edict({ - 'num_classes': 1000, - 'learning_rate': 0.13, - 'momentum': 0.9, - 'epoch_size': 150, - 'batch_size': 256, - 'buffer_size': None, # invalid parameter - 'image_height': 224, - 'image_width': 224, - 'save_checkpoint_steps': 625, - 'keep_checkpoint_max': 10, - 'air_name': "alexnet.air", - - # opt - 'weight_decay': 0.0001, - 'loss_scale': 1024, - - # lr - 'is_dynamic_loss_scale': 0, -}) diff --git a/model_zoo/official/cv/alexnet/src/dataset.py b/model_zoo/official/cv/alexnet/src/dataset.py index 857f6c9141b..0b1ff86017c 100644 --- a/model_zoo/official/cv/alexnet/src/dataset.py +++ b/model_zoo/official/cv/alexnet/src/dataset.py @@ -22,10 +22,9 @@ import mindspore.dataset.transforms.c_transforms as C import mindspore.dataset.vision.c_transforms as CV from mindspore.common import dtype as mstype from mindspore.communication.management import get_rank, get_group_size -from .config import alexnet_cifar10_cfg, alexnet_imagenet_cfg -def create_dataset_cifar10(data_path, batch_size=32, repeat_size=1, status="train", target="Ascend"): +def create_dataset_cifar10(cfg, data_path, batch_size=32, repeat_size=1, status="train", target="Ascend"): """ create dataset for train or test """ @@ -40,7 +39,7 @@ def create_dataset_cifar10(data_path, batch_size=32, repeat_size=1, status="trai num_shards=device_num, shard_id=rank_id) rescale = 1.0 / 255.0 shift = 0.0 - cfg = alexnet_cifar10_cfg + # cfg = alexnet_cifar10_cfg resize_op = CV.Resize((cfg.image_height, cfg.image_width)) rescale_op = CV.Rescale(rescale, shift) @@ -65,7 +64,7 @@ def create_dataset_cifar10(data_path, batch_size=32, repeat_size=1, status="trai return cifar_ds -def create_dataset_imagenet(dataset_path, batch_size=32, repeat_num=1, training=True, +def create_dataset_imagenet(cfg, dataset_path, batch_size=32, repeat_num=1, training=True, num_parallel_workers=None, shuffle=None, sampler=None, class_indexing=None): """ create a train or eval imagenet2012 dataset for resnet50 @@ -82,7 +81,7 @@ def create_dataset_imagenet(dataset_path, batch_size=32, repeat_num=1, training= """ device_num, rank_id = _get_rank_info() - cfg = alexnet_imagenet_cfg + # cfg = alexnet_imagenet_cfg num_parallel_workers = 16 if device_num == 1: diff --git a/model_zoo/official/cv/alexnet/train.py b/model_zoo/official/cv/alexnet/train.py index 252754a88f0..b864ff67da1 100644 --- a/model_zoo/official/cv/alexnet/train.py +++ b/model_zoo/official/cv/alexnet/train.py @@ -82,9 +82,9 @@ def train_alexnet(): context.set_context(device_id=get_device_id()) if config.dataset_name == "cifar10": - ds_train = create_dataset_cifar10(config.data_path, config.batch_size, target=config.device_target) + ds_train = create_dataset_cifar10(config, config.data_path, config.batch_size, target=config.device_target) elif config.dataset_name == "imagenet": - ds_train = create_dataset_imagenet(config.data_path, config.batch_size) + ds_train = create_dataset_imagenet(config, config.data_path, config.batch_size) else: raise ValueError("Unsupported dataset.") diff --git a/model_zoo/official/cv/lenet/README.md b/model_zoo/official/cv/lenet/README.md index 2725f70b07e..2d5d1070143 100644 --- a/model_zoo/official/cv/lenet/README.md +++ b/model_zoo/official/cv/lenet/README.md @@ -100,7 +100,12 @@ sh run_standalone_eval_ascend.sh [DATA_PATH] [CKPT_NAME] │ ├──aipp.cfg // aipp config │ ├──dataset.py // creating dataset │ ├──lenet.py // lenet architecture - │ ├──config.py // parameter configuration + │ └──model_utils + │ ├──config.py // Processing configuration parameters + │ ├──device_adapter.py // Get cloud ID + │ ├──local_adapter.py // Get local ID + │ └──moxing_adapter.py // Parameter processing + ├── default_config.yaml // Training parameter profile(ascend) ├── train.py // training script ├── eval.py // evaluation script ├── postprocess.py // postprocess script diff --git a/model_zoo/official/cv/lenet/README_CN.md b/model_zoo/official/cv/lenet/README_CN.md index 74e14322a07..0429d0bcfcd 100644 --- a/model_zoo/official/cv/lenet/README_CN.md +++ b/model_zoo/official/cv/lenet/README_CN.md @@ -102,12 +102,17 @@ sh run_standalone_eval_ascend.sh [DATA_PATH] [CKPT_NAME] │ ├──run_standalone_eval_ascend.sh // Ascend评估 ├── src │ ├──aipp.cfg // aipp配置文件 - │ ├──dataset.py // 创建数据集 - │ ├──lenet.py // Lenet架构 - │ ├──config.py // 参数配置 - ├── train.py // 训练脚本 - ├── eval.py // 评估脚本 - ├── postprocess.py // 310推理后处理脚本 + │ ├──dataset.py // 创建数据集 + │ ├──lenet.py // Lenet架构 + | └──model_utils + | ├──config.py // 训练配置 + | ├──device_adapter.py // 获取云上id + | ├──local_adapter.py // 获取本地id + | └──moxing_adapter.py // 参数处理 + ├── default_config.yaml // 训练参数配置文件 + ├── train.py // 训练脚本 + ├── eval.py // 评估脚本 + ├── postprocess.py // 310推理后处理脚本 ``` ## 脚本参数 diff --git a/model_zoo/official/cv/lenet/default_config.yaml b/model_zoo/official/cv/lenet/default_config.yaml index fe686ee22f4..2f2c87dde56 100644 --- a/model_zoo/official/cv/lenet/default_config.yaml +++ b/model_zoo/official/cv/lenet/default_config.yaml @@ -18,7 +18,7 @@ num_classes: 10 lr: 0.01 momentum: 0.9 epoch_size: 10 -batch_size: 15 # 32 +batch_size: 32 buffer_size: 1000 image_height: 32 image_width: 32 @@ -27,8 +27,8 @@ keep_checkpoint_max: 10 air_name: "lenet" device_id: 0 file_name: "lenet" -file_format: "AIR" - +file_format: "MINDIR" + model_name: lenet learning_rate: 0.002 dataset_name: 'mnist' @@ -37,6 +37,10 @@ dataset_sink_mode: True save_checkpoint: True save_checkpoint_epochs: 2 +# lenet acc calculation +result_path: '' # "result files path." +img_path: '' # "image file path." + --- # Config description for each option enable_modelarts: 'Whether training on modelarts, default: False' diff --git a/model_zoo/official/cv/lenet/export.py b/model_zoo/official/cv/lenet/export.py index db81d186d58..ce0ef6ee6ab 100644 --- a/model_zoo/official/cv/lenet/export.py +++ b/model_zoo/official/cv/lenet/export.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -"""export checkpoint file into air, mindir models""" +"""export checkpoint file into air, onnx, mindir models""" from src.model_utils.config import config from src.model_utils.device_adapter import get_device_id diff --git a/model_zoo/official/cv/lenet/postprocess.py b/model_zoo/official/cv/lenet/postprocess.py index e5d71735e9e..4467bb22bf5 100644 --- a/model_zoo/official/cv/lenet/postprocess.py +++ b/model_zoo/official/cv/lenet/postprocess.py @@ -14,14 +14,10 @@ # ============================================================================ """post process for 310 inference""" import os -import argparse import numpy as np +from src.model_utils.config import config batch_size = 1 -parser = argparse.ArgumentParser(description="lenet acc calculation") -parser.add_argument("--result_path", type=str, required=True, help="result files path.") -parser.add_argument("--img_path", type=str, required=True, help="image file path.") -args = parser.parse_args() def calcul_acc(labels, preds): @@ -44,4 +40,4 @@ def get_result(result_path, img_path): if __name__ == '__main__': - get_result(args.result_path, args.img_path) + get_result(config.result_path, config.img_path) diff --git a/model_zoo/official/cv/lenet/src/config.py b/model_zoo/official/cv/lenet/src/config.py deleted file mode 100644 index e191906a073..00000000000 --- a/model_zoo/official/cv/lenet/src/config.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -network config setting, will be used in train.py -""" - -from easydict import EasyDict as edict - -mnist_cfg = edict({ - 'num_classes': 10, - 'lr': 0.01, - 'momentum': 0.9, - 'epoch_size': 10, - 'batch_size': 32, - 'buffer_size': 1000, - 'image_height': 32, - 'image_width': 32, - 'save_checkpoint_steps': 1875, - 'keep_checkpoint_max': 10, - 'air_name': "lenet", -})