diff --git a/example/resnet50_cifar10/README.md b/example/resnet50_cifar10/README.md deleted file mode 100644 index abb0ba40901..00000000000 --- a/example/resnet50_cifar10/README.md +++ /dev/null @@ -1,137 +0,0 @@ -# ResNet-50 Example - -## Description - -This is an example of training ResNet-50 with CIFAR-10 dataset in MindSpore. - -## Requirements - -- Install [MindSpore](https://www.mindspore.cn/install/en). - -- Download the dataset CIFAR-10 - -> Unzip the CIFAR-10 dataset to any path you want and the folder structure should include train and eval dataset as follows: -> ``` -> . -> ├── cifar-10-batches-bin # train dataset -> └── cifar-10-verify-bin # infer dataset -> ``` - - -## Example structure - -```shell -. -├── config.py # parameter configuration -├── dataset.py # data preprocessing -├── eval.py # infer script -├── lr_generator.py # generate learning rate for each step -├── run_distribute_train.sh # launch distributed training(8 pcs) -├── run_infer.sh # launch infering -├── run_standalone_train.sh # launch standalone training(1 pcs) -└── train.py # train script -``` - - -## Parameter configuration - -Parameters for both training and inference can be set in config.py. - -``` -"class_num": 10, # dataset class num -"batch_size": 32, # batch size of input tensor -"loss_scale": 1024, # loss scale -"momentum": 0.9, # momentum -"weight_decay": 1e-4, # weight decay -"epoch_size": 90, # only valid for taining, which is always 1 for inference -"buffer_size": 100, # number of queue size in data preprocessing -"image_height": 224, # image height -"image_width": 224, # image width -"save_checkpoint": True, # whether save checkpoint or not -"save_checkpoint_steps": 195, # the step interval between two checkpoints. By default, the last checkpoint will be saved after the last step -"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint -"save_checkpoint_path": "./", # path to save checkpoint -"warmup_epochs": 5, # number of warmup epoch -"lr_decay_mode": "poly" # decay mode can be selected in steps, ploy and default -"lr_init": 0.01, # initial learning rate -"lr_end": 0.00001, # final learning rate -"lr_max": 0.1, # maximum learning rate -``` - -## Running the example - -### Train - -#### Usage - -``` -# distributed training -Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] - -# standalone training -Usage: sh run_standalone_train.sh [DATASET_PATH] -``` - - -#### Launch - -``` -# distribute training example -sh run_distribute_train.sh rank_table.json ~/cifar-10-batches-bin - -# standalone training example -sh run_standalone_train.sh ~/cifar-10-batches-bin -``` - -> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). - -#### Result - -Training result will be stored in the example path, whose folder name begins with "train" or "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. - -``` -# distribute training result(8 pcs) -epoch: 1 step: 195, loss is 1.9601055 -epoch: 2 step: 195, loss is 1.8555021 -epoch: 3 step: 195, loss is 1.6707983 -epoch: 4 step: 195, loss is 1.8162166 -epoch: 5 step: 195, loss is 1.393667 -``` - -### Infer - -#### Usage - -``` -# infer -Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH] -``` - -#### Launch - -``` -# infer example -sh run_infer.sh ~/cifar10-10-verify-bin ~/resnet50_cifar10/train_parallel0/resnet-90_195.ckpt -``` - -> checkpoint can be produced in training process. - -#### Result - -Inference result will be stored in the example path, whose folder name is "infer". Under this, you can find result like the followings in log. - -``` -result: {'acc': 0.91446314102564111} ckpt=~/resnet50_cifar10/train_parallel0/resnet-90_195.ckpt -``` - -### Running on GPU -``` -# distributed training example -mpirun -n 8 python train.py --dataset_path=~/cifar-10-batches-bin --device_target="GPU" --run_distribute=True - -# standalone training example -python train.py --dataset_path=~/cifar-10-batches-bin --device_target="GPU" - -# infer example -python eval.py --dataset_path=~/cifar10-10-verify-bin --device_target="GPU" --checkpoint_path=resnet-90_195.ckpt -``` \ No newline at end of file diff --git a/example/resnet50_cifar10/config.py b/example/resnet50_cifar10/config.py deleted file mode 100755 index 3c50a6aaed1..00000000000 --- a/example/resnet50_cifar10/config.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -network config setting, will be used in train.py and eval.py -""" -from easydict import EasyDict as ed - -config = ed({ - "class_num": 10, - "batch_size": 32, - "loss_scale": 1024, - "momentum": 0.9, - "weight_decay": 1e-4, - "epoch_size": 90, - "buffer_size": 100, - "image_height": 224, - "image_width": 224, - "save_checkpoint": True, - "save_checkpoint_epochs": 5, - "keep_checkpoint_max": 10, - "save_checkpoint_path": "./", - "warmup_epochs": 5, - "lr_decay_mode": "poly", - "lr_init": 0.01, - "lr_end": 0.00001, - "lr_max": 0.1 -}) diff --git a/example/resnet50_cifar10/dataset.py b/example/resnet50_cifar10/dataset.py deleted file mode 100755 index 8a66ec573af..00000000000 --- a/example/resnet50_cifar10/dataset.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -create train or eval dataset. -""" -import os -import mindspore.common.dtype as mstype -import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as C -import mindspore.dataset.transforms.c_transforms as C2 -from mindspore.communication.management import init, get_rank, get_group_size -from config import config - - -def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): - """ - create a train or eval dataset - - Args: - dataset_path(string): the path of dataset. - do_train(bool): whether dataset is used for train or eval. - repeat_num(int): the repeat times of dataset. Default: 1 - batch_size(int): the batch size of dataset. Default: 32 - target(str): the device target. Default: Ascend - - Returns: - dataset - """ - if target == "Ascend": - device_num = int(os.getenv("DEVICE_NUM")) - rank_id = int(os.getenv("RANK_ID")) - else: - init("nccl") - rank_id = get_rank() - device_num = get_group_size() - - if device_num == 1: - ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True) - else: - ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) - - # define map operations - trans = [] - if do_train: - trans += [ - C.RandomCrop((32, 32), (4, 4, 4, 4)), - C.RandomHorizontalFlip(prob=0.5) - ] - - trans += [ - C.Resize((config.image_height, config.image_width)), - C.Rescale(1.0 / 255.0, 0.0), - C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), - C.HWC2CHW() - ] - - type_cast_op = C2.TypeCast(mstype.int32) - - ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) - ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) - - # apply batch operations - ds = ds.batch(batch_size, drop_remainder=True) - - # apply dataset repeat operation - ds = ds.repeat(repeat_num) - - return ds diff --git a/example/resnet50_cifar10/eval.py b/example/resnet50_cifar10/eval.py deleted file mode 100755 index f7d71c8d29c..00000000000 --- a/example/resnet50_cifar10/eval.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -eval. -""" -import os -import argparse -from dataset import create_dataset -from config import config -from mindspore import context -from mindspore.model_zoo.resnet import resnet50 -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits -from mindspore.train.model import Model, ParallelMode -from mindspore.train.serialization import load_checkpoint, load_param_into_net -from mindspore.communication.management import init, get_group_size - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=False, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=True, help='Do eval or not.') -parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') -args_opt = parser.parse_args() - -if __name__ == '__main__': - target = args_opt.device_target - context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) - if not args_opt.do_eval and args_opt.run_distribute: - if target == "Ascend": - device_id = int(os.getenv('DEVICE_ID')) - context.set_context(device_id=device_id) - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([140]) - init() - elif target == "GPU": - init("nccl") - context.set_auto_parallel_context(device_num=get_group_size(), parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - - epoch_size = config.epoch_size - net = resnet50(class_num=config.class_num) - loss = SoftmaxCrossEntropyWithLogits(sparse=True) - - if args_opt.do_eval: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size, - target=target) - step_size = dataset.get_dataset_size() - - if args_opt.checkpoint_path: - param_dict = load_checkpoint(args_opt.checkpoint_path) - load_param_into_net(net, param_dict) - net.set_train(False) - - model = Model(net, loss_fn=loss, metrics={'acc'}) - res = model.eval(dataset) - print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/example/resnet50_cifar10/lr_generator.py b/example/resnet50_cifar10/lr_generator.py deleted file mode 100755 index 37c8e907d75..00000000000 --- a/example/resnet50_cifar10/lr_generator.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""learning rate generator""" -import numpy as np - - -def get_lr(global_step, lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch, lr_decay_mode): - """ - generate learning rate array - - Args: - global_step(int): total steps of the training - lr_init(float): init learning rate - lr_end(float): end learning rate - lr_max(float): max learning rate - warmup_epochs(int): number of warmup epochs - total_epochs(int): total epoch of training - steps_per_epoch(int): steps of one epoch - lr_decay_mode(string): learning rate decay mode, including steps, poly or default - - Returns: - np.array, learning rate array - """ - lr_each_step = [] - total_steps = steps_per_epoch * total_epochs - warmup_steps = steps_per_epoch * warmup_epochs - if lr_decay_mode == 'steps': - decay_epoch_index = [0.3 * total_steps, 0.6 * total_steps, 0.8 * total_steps] - for i in range(total_steps): - if i < decay_epoch_index[0]: - lr = lr_max - elif i < decay_epoch_index[1]: - lr = lr_max * 0.1 - elif i < decay_epoch_index[2]: - lr = lr_max * 0.01 - else: - lr = lr_max * 0.001 - lr_each_step.append(lr) - elif lr_decay_mode == 'poly': - if warmup_steps != 0: - inc_each_step = (float(lr_max) - float(lr_init)) / float(warmup_steps) - else: - inc_each_step = 0 - for i in range(total_steps): - if i < warmup_steps: - lr = float(lr_init) + inc_each_step * float(i) - else: - base = (1.0 - (float(i) - float(warmup_steps)) / (float(total_steps) - float(warmup_steps))) - lr = float(lr_max) * base * base - if lr < 0.0: - lr = 0.0 - lr_each_step.append(lr) - else: - for i in range(total_steps): - if i < warmup_steps: - lr = lr_init + (lr_max - lr_init) * i / warmup_steps - else: - lr = lr_max - (lr_max - lr_end) * (i - warmup_steps) / (total_steps - warmup_steps) - lr_each_step.append(lr) - - current_step = global_step - lr_each_step = np.array(lr_each_step).astype(np.float32) - learning_rate = lr_each_step[current_step:] - - return learning_rate diff --git a/example/resnet50_cifar10/run_distribute_train.sh b/example/resnet50_cifar10/run_distribute_train.sh deleted file mode 100755 index e4bdd775b35..00000000000 --- a/example/resnet50_cifar10/run_distribute_train.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 2 ] -then - echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH]" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) - -if [ ! -f "$PATH1" ] -then - echo "error: MINDSPORE_HCCL_CONFIG_PATH=$PATH1 is not a file" -exit 1 -fi - -if [ ! -d "$PATH2" ] -then - echo "error: DATASET_PATH=$PATH2 is not a directory" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=8 -export RANK_SIZE=8 -export MINDSPORE_HCCL_CONFIG_PATH=$PATH1 - -for((i=0; i<${DEVICE_NUM}; i++)) -do - export DEVICE_ID=$i - export RANK_ID=$i - rm -rf ./train_parallel$i - mkdir ./train_parallel$i - cp *.py ./train_parallel$i - cp *.sh ./train_parallel$i - cd ./train_parallel$i || exit - echo "start training for rank $RANK_ID, device $DEVICE_ID" - env > env.log - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 &> log & - cd .. -done diff --git a/example/resnet50_cifar10/run_infer.sh b/example/resnet50_cifar10/run_infer.sh deleted file mode 100755 index 14d7faf981e..00000000000 --- a/example/resnet50_cifar10/run_infer.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 2 ] -then - echo "Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH]" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) - - -if [ ! -d $PATH1 ] -then - echo "error: DATASET_PATH=$1 is not a directory" -exit 1 -fi - -if [ ! -f $PATH2 ] -then - echo "error: CHECKPOINT_PATH=$2 is not a file" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=1 -export DEVICE_ID=0 -export RANK_SIZE=$DEVICE_NUM -export RANK_ID=0 - -if [ -d "infer" ]; -then - rm -rf ./infer -fi -mkdir ./infer -cp *.py ./infer -cp *.sh ./infer -cd ./infer || exit -env > env.log -echo "start infering for device $DEVICE_ID" -python eval.py --do_eval=True --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & -cd .. diff --git a/example/resnet50_cifar10/run_standalone_train.sh b/example/resnet50_cifar10/run_standalone_train.sh deleted file mode 100755 index cb08cde6c94..00000000000 --- a/example/resnet50_cifar10/run_standalone_train.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 1 ] -then - echo "Usage: sh run_standalone_train.sh [DATASET_PATH]" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) - -if [ ! -d "$PATH1" ] -then - echo "error: DATASET_PATH=$PATH1 is not a directory" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=1 -export DEVICE_ID=0 -export RANK_ID=0 - -if [ -d "train" ]; -then - rm -rf ./train -fi -mkdir ./train -cp *.py ./train -cp *.sh ./train -cd ./train || exit -echo "start training for device $DEVICE_ID" -env > env.log -python train.py --do_train=True --dataset_path=$PATH1 &> log & -cd .. diff --git a/example/resnet50_cifar10/train.py b/example/resnet50_cifar10/train.py deleted file mode 100755 index 323695ae291..00000000000 --- a/example/resnet50_cifar10/train.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""train_imagenet.""" -import os -import argparse -import numpy as np -from dataset import create_dataset -from lr_generator import get_lr -from config import config -from mindspore import context -from mindspore import Tensor -from mindspore.model_zoo.resnet import resnet50 -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.nn.optim.momentum import Momentum -from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits - -from mindspore.train.model import Model, ParallelMode - -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor -from mindspore.train.loss_scale_manager import FixedLossScaleManager -from mindspore.communication.management import init, get_rank, get_group_size - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') -args_opt = parser.parse_args() - - -if __name__ == '__main__': - target = args_opt.device_target - ckpt_save_dir = config.save_checkpoint_path - context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) - np.random.seed(1) - if not args_opt.do_eval and args_opt.run_distribute: - if target == "Ascend": - device_id = int(os.getenv('DEVICE_ID')) - context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id, - enable_auto_mixed_precision=True) - init() - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([107, 160]) - ckpt_save_dir = config.save_checkpoint_path - elif target == "GPU": - context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) - init("nccl") - context.set_auto_parallel_context(device_num=get_group_size(), parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - ckpt_save_dir = config.save_checkpoint_path + "ckpt_" + str(get_rank()) + "/" - epoch_size = config.epoch_size - net = resnet50(class_num=config.class_num) - - if args_opt.do_train: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, - repeat_num=epoch_size, batch_size=config.batch_size, target=target) - step_size = dataset.get_dataset_size() - - loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) - lr = Tensor(get_lr(global_step=0, lr_init=config.lr_init, lr_end=config.lr_end, lr_max=config.lr_max, - warmup_epochs=config.warmup_epochs, total_epochs=epoch_size, steps_per_epoch=step_size, - lr_decay_mode='poly')) - opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, - config.weight_decay, config.loss_scale) - if target == 'GPU': - loss = SoftmaxCrossEntropyWithLogits(sparse=True, is_grad=False, reduction='mean') - opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum) - model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'}) - else: - loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') - model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}, - amp_level="O2", keep_batchnorm_fp32=False) - - time_cb = TimeMonitor(data_size=step_size) - loss_cb = LossMonitor() - cb = [time_cb, loss_cb] - if config.save_checkpoint: - config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs*step_size, - keep_checkpoint_max=config.keep_checkpoint_max) - ckpt_cb = ModelCheckpoint(prefix="resnet", directory=ckpt_save_dir, config=config_ck) - cb += [ckpt_cb] - model.train(epoch_size, dataset, callbacks=cb) diff --git a/example/resnet50_imagenet2012/README.md b/example/resnet50_imagenet2012/README.md deleted file mode 100644 index 6baf863544e..00000000000 --- a/example/resnet50_imagenet2012/README.md +++ /dev/null @@ -1,150 +0,0 @@ -# ResNet-50 Example - -## Description - -This is an example of training ResNet-50 with ImageNet2012 dataset in MindSpore. - -## Requirements - -- Install [MindSpore](https://www.mindspore.cn/install/en). - -- Download the dataset ImageNet2012 - -> Unzip the ImageNet2012 dataset to any path you want and the folder structure should include train and eval dataset as follows: -> ``` -> . -> ├── ilsvrc # train dataset -> └── ilsvrc_eval # infer dataset -> ``` - - -## Example structure - -```shell -. -├── crossentropy.py # CrossEntropy loss function -├── config.py # parameter configuration -├── dataset.py # data preprocessing -├── eval.py # infer script -├── lr_generator.py # generate learning rate for each step -├── run_distribute_train.sh # launch distributed training(8 pcs) -├── run_infer.sh # launch infering -├── run_standalone_train.sh # launch standalone training(1 pcs) -└── train.py # train script -``` - - -## Parameter configuration - -Parameters for both training and inference can be set in config.py. - -``` -"class_num": 1001, # dataset class number -"batch_size": 32, # batch size of input tensor -"loss_scale": 1024, # loss scale -"momentum": 0.9, # momentum optimizer -"weight_decay": 1e-4, # weight decay -"epoch_size": 90, # only valid for taining, which is always 1 for inference -"pretrained_epoch_size": 1, # epoch size that model has been trained before load pretrained checkpoint -"buffer_size": 1000, # number of queue size in data preprocessing -"image_height": 224, # image height -"image_width": 224, # image width -"save_checkpoint": True, # whether save checkpoint or not -"save_checkpoint_epochs": 1, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch -"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint -"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path -"warmup_epochs": 0, # number of warmup epoch -"lr_decay_mode": "cosine", # decay mode for generating learning rate -"label_smooth": True, # label smooth -"label_smooth_factor": 0.1, # label smooth factor -"lr_init": 0, # initial learning rate -"lr_max": 0.1, # maximum learning rate -``` - -## Running the example - -### Train - -#### Usage - -``` -# distributed training -Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional) - -# standalone training -Usage: sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional) - -``` - - -#### Launch - -```bash -# distributed training example(8 pcs) -sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc - -# If you want to load pretrained ckpt file -sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc ./pretrained.ckpt - -# standalone training example(1 pcs) -sh run_standalone_train.sh dataset/ilsvrc - -# If you want to load pretrained ckpt file -sh run_standalone_train.sh dataset/ilsvrc ./pretrained.ckpt -``` - -> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). - -#### Result - -Training result will be stored in the example path, whose folder name begins with "train" or "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. - -``` -# distribute training result(8 pcs) -epoch: 1 step: 5004, loss is 4.8995576 -epoch: 2 step: 5004, loss is 3.9235563 -epoch: 3 step: 5004, loss is 3.833077 -epoch: 4 step: 5004, loss is 3.2795618 -epoch: 5 step: 5004, loss is 3.1978393 -``` - -### Infer - -#### Usage - -``` -# infer -Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH] -``` - -#### Launch - -```bash -# infer with checkpoint -sh run_infer.sh dataset/ilsvrc_eval train_parallel0/resnet-90_5004.ckpt -``` - -> checkpoint can be produced in training process. - -#### Result - -Inference result will be stored in the example path, whose folder name is "infer". Under this, you can find result like the followings in log. - -``` -result: {'acc': 0.7671054737516005} ckpt=train_parallel0/resnet-90_5004.ckpt -``` - -### Running on GPU -``` -# distributed training example -mpirun -n 8 python train.py --dataset_path=dataset/ilsvrc/train --device_target="GPU" --run_distribute=True - -# standalone training example -python train.py --dataset_path=dataset/ilsvrc/train --device_target="GPU" - -# standalone training example with pretrained checkpoint -python train.py --dataset_path=dataset/ilsvrc/train --device_target="GPU" --pre_trained=pretrained.ckpt - -# infer example -python eval.py --dataset_path=dataset/ilsvrc/val --device_target="GPU" --checkpoint_path=resnet-90_5004ss.ckpt -``` \ No newline at end of file diff --git a/example/resnet50_imagenet2012/crossentropy.py b/example/resnet50_imagenet2012/crossentropy.py deleted file mode 100644 index b078b29f6fe..00000000000 --- a/example/resnet50_imagenet2012/crossentropy.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""define loss function for network""" -from mindspore.nn.loss.loss import _Loss -from mindspore.ops import operations as P -from mindspore.ops import functional as F -from mindspore import Tensor -from mindspore.common import dtype as mstype -import mindspore.nn as nn - - -class CrossEntropy(_Loss): - """the redefined loss function with SoftmaxCrossEntropyWithLogits""" - - def __init__(self, smooth_factor=0, num_classes=1001): - super(CrossEntropy, self).__init__() - self.onehot = P.OneHot() - self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) - self.off_value = Tensor(1.0 * smooth_factor / (num_classes - 1), mstype.float32) - self.ce = nn.SoftmaxCrossEntropyWithLogits() - self.mean = P.ReduceMean(False) - - def construct(self, logit, label): - one_hot_label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) - loss = self.ce(logit, one_hot_label) - loss = self.mean(loss, 0) - return loss diff --git a/example/resnet50_imagenet2012/dataset.py b/example/resnet50_imagenet2012/dataset.py deleted file mode 100755 index 0691985e0b1..00000000000 --- a/example/resnet50_imagenet2012/dataset.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -create train or eval dataset. -""" -import os -import mindspore.common.dtype as mstype -import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as C -import mindspore.dataset.transforms.c_transforms as C2 -from mindspore.communication.management import init, get_rank, get_group_size - -def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): - """ - create a train or eval dataset - - Args: - dataset_path(string): the path of dataset. - do_train(bool): whether dataset is used for train or eval. - repeat_num(int): the repeat times of dataset. Default: 1 - batch_size(int): the batch size of dataset. Default: 32 - target(str): the device target. Default: Ascend - - Returns: - dataset - """ - if target == "Ascend": - device_num = int(os.getenv("DEVICE_NUM")) - rank_id = int(os.getenv("RANK_ID")) - else: - init("nccl") - rank_id = get_rank() - device_num = get_group_size() - - if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) - else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) - - image_size = 224 - mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] - std = [0.229 * 255, 0.224 * 255, 0.225 * 255] - - # define map operations - if do_train: - trans = [ - C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), - C.RandomHorizontalFlip(prob=0.5), - C.Normalize(mean=mean, std=std), - C.HWC2CHW() - ] - else: - trans = [ - C.Decode(), - C.Resize((256, 256)), - C.CenterCrop(image_size), - C.Normalize(mean=mean, std=std), - C.HWC2CHW() - ] - - type_cast_op = C2.TypeCast(mstype.int32) - - ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) - ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) - - # apply batch operations - ds = ds.batch(batch_size, drop_remainder=True) - - # apply dataset repeat operation - ds = ds.repeat(repeat_num) - - return ds diff --git a/example/resnet50_imagenet2012/eval.py b/example/resnet50_imagenet2012/eval.py deleted file mode 100755 index 3f7961e7866..00000000000 --- a/example/resnet50_imagenet2012/eval.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -eval. -""" -import os -import argparse -from dataset import create_dataset -from config import config -from mindspore import context -from mindspore.model_zoo.resnet import resnet50 -from mindspore.train.model import Model -from mindspore.train.serialization import load_checkpoint, load_param_into_net -from crossentropy import CrossEntropy - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=False, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=True, help='Do eval or not.') -parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') -args_opt = parser.parse_args() -target = args_opt.device_target -context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) -if target == "Ascend": - device_id = int(os.getenv('DEVICE_ID')) - context.set_context(device_id=device_id) - -if __name__ == '__main__': - - net = resnet50(class_num=config.class_num) - if not config.use_label_smooth: - config.label_smooth_factor = 0.0 - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - - if args_opt.do_eval: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size, - target=target) - step_size = dataset.get_dataset_size() - - if args_opt.checkpoint_path: - param_dict = load_checkpoint(args_opt.checkpoint_path) - load_param_into_net(net, param_dict) - net.set_train(False) - - model = Model(net, loss_fn=loss, metrics={'acc'}) - res = model.eval(dataset) - print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/example/resnet50_imagenet2012/run_distribute_train.sh b/example/resnet50_imagenet2012/run_distribute_train.sh deleted file mode 100755 index 22157608e60..00000000000 --- a/example/resnet50_imagenet2012/run_distribute_train.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 2 ] && [ $# != 3 ] -then - echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) -if [ $# == 3 ] -then - PATH3=$(get_real_path $3) -fi - -if [ ! -f "$PATH1" ] -then - echo "error: MINDSPORE_HCCL_CONFIG_PATH=$PATH1 is not a file" -exit 1 -fi - -if [ ! -d "$PATH2" ] -then - echo "error: DATASET_PATH=$PATH2 is not a directory" -exit 1 -fi - -if [ $# == 3 ] && [ ! -f "$PATH3" ] -then - echo "error: PRETRAINED_CKPT_PATH=$PATH3 is not a file" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=8 -export RANK_SIZE=8 -export MINDSPORE_HCCL_CONFIG_PATH=$PATH1 -export RANK_TABLE_FILE=$PATH1 - -for((i=0; i<${DEVICE_NUM}; i++)) -do - export DEVICE_ID=$i - export RANK_ID=$i - rm -rf ./train_parallel$i - mkdir ./train_parallel$i - cp *.py ./train_parallel$i - cp *.sh ./train_parallel$i - cd ./train_parallel$i || exit - echo "start training for rank $RANK_ID, device $DEVICE_ID" - env > env.log - if [ $# == 2 ] - then - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 &> log & - else - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 --pre_trained=$PATH3 &> log & - fi - cd .. -done diff --git a/example/resnet50_imagenet2012/run_infer.sh b/example/resnet50_imagenet2012/run_infer.sh deleted file mode 100755 index 1482b63f5f6..00000000000 --- a/example/resnet50_imagenet2012/run_infer.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 2 ] -then - echo "Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH]" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) - - -if [ ! -d $PATH1 ] -then - echo "error: DATASET_PATH=$PATH1 is not a directory" -exit 1 -fi - -if [ ! -f $PATH2 ] -then - echo "error: CHECKPOINT_PATH=$PATH2 is not a file" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=1 -export DEVICE_ID=0 -export RANK_SIZE=$DEVICE_NUM -export RANK_ID=0 - -if [ -d "infer" ]; -then - rm -rf ./infer -fi -mkdir ./infer -cp *.py ./infer -cp *.sh ./infer -cd ./infer || exit -env > env.log -echo "start infering for device $DEVICE_ID" -python eval.py --do_eval=True --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & -cd .. diff --git a/example/resnet50_imagenet2012/run_standalone_train.sh b/example/resnet50_imagenet2012/run_standalone_train.sh deleted file mode 100755 index e0eb5efaf0e..00000000000 --- a/example/resnet50_imagenet2012/run_standalone_train.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 1 ] && [ $# != 2 ] -then - echo "Usage: sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) -if [ $# == 2 ] -then - PATH2=$(get_real_path $2) -fi - -if [ ! -d "$PATH1" ] -then - echo "error: DATASET_PATH=$PATH1 is not a directory" -exit 1 -fi - -if [ $# == 2 ] && [ ! -f "$PATH2" ] -then - echo "error: PRETRAINED_CKPT_PATH=$PATH2 is not a file" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=1 -export DEVICE_ID=0 -export RANK_ID=0 - -if [ -d "train" ]; -then - rm -rf ./train -fi -mkdir ./train -cp *.py ./train -cp *.sh ./train -cd ./train || exit -echo "start training for device $DEVICE_ID" -env > env.log -if [ $# == 1 ] -then - python train.py --do_train=True --dataset_path=$PATH1 &> log & -else - python train.py --do_train=True --dataset_path=$PATH1 --pre_trained=$PATH2 &> log & -fi -cd .. diff --git a/example/resnet50_imagenet2012/train.py b/example/resnet50_imagenet2012/train.py deleted file mode 100755 index 6896320ecea..00000000000 --- a/example/resnet50_imagenet2012/train.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""train_imagenet.""" -import os -import argparse -import numpy as np -from dataset import create_dataset -from lr_generator import get_lr -from config import config -from mindspore import context -from mindspore import Tensor -from mindspore.model_zoo.resnet import resnet50 -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.nn.optim.momentum import Momentum - -from mindspore.train.model import Model, ParallelMode - -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor -from mindspore.train.loss_scale_manager import FixedLossScaleManager -from mindspore.train.serialization import load_checkpoint, load_param_into_net -from mindspore.communication.management import init, get_rank, get_group_size -import mindspore.nn as nn -import mindspore.common.initializer as weight_init -from crossentropy import CrossEntropy - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') -parser.add_argument('--pre_trained', type=str, default=None, help='Pretrained checkpoint path') -args_opt = parser.parse_args() - -if __name__ == '__main__': - target = args_opt.device_target - ckpt_save_dir = config.save_checkpoint_path - context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) - np.random.seed(1) - if not args_opt.do_eval and args_opt.run_distribute: - if target == "Ascend": - device_id = int(os.getenv('DEVICE_ID')) - context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id, - enable_auto_mixed_precision=True) - init() - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([107, 160]) - ckpt_save_dir = config.save_checkpoint_path - elif target == "GPU": - context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) - init("nccl") - context.set_auto_parallel_context(device_num=get_group_size(), parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - ckpt_save_dir = config.save_checkpoint_path + "ckpt_" + str(get_rank()) + "/" - - epoch_size = config.epoch_size - net = resnet50(class_num=config.class_num) - - # weight init - if args_opt.pre_trained: - param_dict = load_checkpoint(args_opt.pre_trained) - load_param_into_net(net, param_dict) - epoch_size = config.epoch_size - config.pretrained_epoch_size - else: - for _, cell in net.cells_and_names(): - if isinstance(cell, nn.Conv2d): - cell.weight.default_input = weight_init.initializer(weight_init.XavierUniform(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() - if isinstance(cell, nn.Dense): - cell.weight.default_input = weight_init.initializer(weight_init.TruncatedNormal(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() - if not config.use_label_smooth: - config.label_smooth_factor = 0.0 - - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - - if args_opt.do_train: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, - repeat_num=epoch_size, batch_size=config.batch_size, target=target) - step_size = dataset.get_dataset_size() - - loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) - lr = get_lr(lr_init=config.lr_init, lr_end=0.0, lr_max=config.lr_max, warmup_epochs=config.warmup_epochs, - total_epochs=config.epoch_size, steps_per_epoch=step_size, lr_decay_mode='cosine') - if args_opt.pre_trained: - lr = lr[config.pretrained_epoch_size * step_size:] - lr = Tensor(lr) - - opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, - config.weight_decay, config.loss_scale) - if target == "Ascend": - model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}, - amp_level="O2", keep_batchnorm_fp32=False) - elif target == "GPU": - model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}) - - - time_cb = TimeMonitor(data_size=step_size) - loss_cb = LossMonitor() - cb = [time_cb, loss_cb] - if config.save_checkpoint: - config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs*step_size, - keep_checkpoint_max=config.keep_checkpoint_max) - ckpt_cb = ModelCheckpoint(prefix="resnet", directory=ckpt_save_dir, config=config_ck) - cb += [ckpt_cb] - model.train(epoch_size, dataset, callbacks=cb) diff --git a/example/resnet50_imagenet2012_THOR/README.md b/example/resnet50_imagenet2012_THOR/README.md deleted file mode 100644 index 6003d8d7b7a..00000000000 --- a/example/resnet50_imagenet2012_THOR/README.md +++ /dev/null @@ -1,118 +0,0 @@ -# ResNet-50-THOR Example - -## Description - -This is an example of training ResNet-50 V1.5 with ImageNet2012 dataset by second-order optimizer THOR. THOR is a novel approximate seond-order optimization method in MindSpore. With fewer iterations, THOR can finish ResNet-50 V1.5 training in 72 minutes to top-1 accuracy of 75.9% using 8 Ascend 910, which is much faster than SGD with Momentum. - -## Requirements - -- Install [MindSpore](https://www.mindspore.cn/install/en). - -- Download the dataset ImageNet2012 - -> Unzip the ImageNet2012 dataset to any path you want and the folder structure should include train and eval dataset as follows: -> ``` -> . -> ├── ilsvrc # train dataset -> └── ilsvrc_eval # infer dataset -> ``` - - -## Example structure - -```shell -. -├── crossentropy.py # CrossEntropy loss function -├── config.py # parameter configuration -├── dataset_imagenet.py # data preprocessing -├── eval.py # infer script -├── model # include model file of the optimizer -├── run_distribute_train.sh # launch distributed training(8 pcs) -├── run_infer.sh # launch infering -└── train.py # train script -``` - - -## Parameter configuration - -Parameters for both training and inference can be set in config.py. - -``` -"class_num": 1000, # dataset class number -"batch_size": 32, # batch size of input tensor -"loss_scale": 128, # loss scale -"momentum": 0.9, # momentum of THOR optimizer -"weight_decay": 5e-4, # weight decay -"epoch_size": 45, # only valid for taining, which is always 1 for inference -"buffer_size": 1000, # number of queue size in data preprocessing -"image_height": 224, # image height -"image_width": 224, # image width -"save_checkpoint": True, # whether save checkpoint or not -"save_checkpoint_steps": 5004, # the step interval between two checkpoints. By default, the checkpoint will be saved every epoch -"keep_checkpoint_max": 20, # only keep the last keep_checkpoint_max checkpoint -"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path -"label_smooth": True, # label smooth -"label_smooth_factor": 0.1, # label smooth factor -"frequency": 834, # the step interval to update second-order information matrix -``` - -## Running the example - -### Train - -#### Usage - -``` -# distributed training -Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [DEVICE_NUM] -``` - - -#### Launch - -```bash -# distributed training example(8 pcs) -sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc -``` - -> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). - -#### Result - -Training result will be stored in the example path, whose folder name begins with "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. - -``` -# distribute training result(8 pcs) -epoch: 1 step: 5004, loss is 4.4182425 -epoch: 2 step: 5004, loss is 3.740064 -epoch: 3 step: 5004, loss is 4.0546017 -epoch: 4 step: 5004, loss is 3.7598825 -epoch: 5 step: 5004, loss is 3.3744206 -...... -``` - -### Infer - -#### Usage - -``` -# infer -Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH] -``` - -#### Launch - -```bash -# infer with checkpoint -sh run_infer.sh dataset/ilsvrc_eval train_parallel0/resnet-42_5004.ckpt -``` - -> checkpoint can be produced in training process. - -#### Result - -Inference result will be stored in the example path, whose folder name is "infer". Under this, you can find result like the followings in log. - -``` -result: {'acc': 0.759503041} ckpt=train_parallel0/resnet-42_5004.ckpt -``` diff --git a/example/resnet50_imagenet2012_THOR/config.py b/example/resnet50_imagenet2012_THOR/config.py deleted file mode 100644 index cd0a81d5e6f..00000000000 --- a/example/resnet50_imagenet2012_THOR/config.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -network config setting, will be used in train.py and eval.py -""" -from easydict import EasyDict as ed - -config = ed({ - "class_num": 1000, - "batch_size": 32, - "loss_scale": 128, - "momentum": 0.9, - "weight_decay": 5e-4, - "epoch_size": 45, - "buffer_size": 1000, - "image_height": 224, - "image_width": 224, - "save_checkpoint": True, - "save_checkpoint_steps": 5004, - "keep_checkpoint_max": 20, - "save_checkpoint_path": "./", - "label_smooth": 1, - "label_smooth_factor": 0.1, - "frequency": 834 -}) diff --git a/example/resnet50_imagenet2012_THOR/crossentropy.py b/example/resnet50_imagenet2012_THOR/crossentropy.py deleted file mode 100644 index e8681ff4972..00000000000 --- a/example/resnet50_imagenet2012_THOR/crossentropy.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CrossEntropy""" -import mindspore.nn as nn -from mindspore import Tensor -from mindspore.common import dtype as mstype -from mindspore.nn.loss.loss import _Loss -from mindspore.ops import functional as F -from mindspore.ops import operations as P - - -class CrossEntropy(_Loss): - """CrossEntropy""" - def __init__(self, smooth_factor=0., num_classes=1000): - super(CrossEntropy, self).__init__() - self.onehot = P.OneHot() - self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) - self.off_value = Tensor(1.0 * smooth_factor / (num_classes - 1), mstype.float32) - # self.cast = P.Cast() - self.ce = nn.SoftmaxCrossEntropyWithLogits() - self.mean = P.ReduceMean(False) - - def construct(self, logit, label): - # one_hot_label = self.onehot(self.cast(label, mstype.int32), - # F.shape(logit)[1], self.on_value, self.off_value)、 - one_hot_label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) - loss = self.ce(logit, one_hot_label) - loss = self.mean(loss, 0) - return loss diff --git a/example/resnet50_imagenet2012_THOR/dataset_imagenet.py b/example/resnet50_imagenet2012_THOR/dataset_imagenet.py deleted file mode 100644 index 296b675136d..00000000000 --- a/example/resnet50_imagenet2012_THOR/dataset_imagenet.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -create train or eval dataset. -""" -import os - -import mindspore.common.dtype as mstype -import mindspore.dataset.engine as de -import mindspore.dataset.transforms.c_transforms as C2 -import mindspore.dataset.transforms.vision.c_transforms as V_C - - -def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): - """ - create a train or eval dataset - Args: - dataset_path(string): the path of dataset. - do_train(bool): whether dataset is used for train or eval. - repeat_num(int): the repeat times of dataset. Default: 1 - batch_size(int): the batch size of dataset. Default: 32 - Returns: - dataset - """ - - device_num = int(os.getenv("RANK_SIZE")) - rank_id = int(os.getenv("RANK_ID")) - - if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False) - else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) - - image_size = 224 - mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] - std = [0.229 * 255, 0.224 * 255, 0.225 * 255] - if do_train: - transform_img = [ - V_C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), - V_C.RandomHorizontalFlip(prob=0.5), - V_C.Normalize(mean=mean, std=std), - V_C.HWC2CHW() - ] - else: - transform_img = [ - V_C.Decode(), - V_C.Resize((256, 256)), - V_C.CenterCrop(image_size), - V_C.Normalize(mean=mean, std=std), - V_C.HWC2CHW() - ] - # type_cast_op = C2.TypeCast(mstype.float16) - type_cast_op = C2.TypeCast(mstype.int32) - - ds = ds.map(input_columns="image", operations=transform_img, num_parallel_workers=8) - ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) - - # apply shuffle operations - # ds = ds.shuffle(buffer_size=config.buffer_size) - - # apply batch operations - ds = ds.batch(batch_size, drop_remainder=True) - - # apply dataset repeat operation - ds = ds.repeat(repeat_num) - - return ds diff --git a/example/resnet50_imagenet2012_THOR/eval.py b/example/resnet50_imagenet2012_THOR/eval.py deleted file mode 100755 index db82b9fcacb..00000000000 --- a/example/resnet50_imagenet2012_THOR/eval.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -eval. -""" -import os -import argparse -from dataset_imagenet import create_dataset -from config import config -from mindspore import context -from mindspore.model_zoo.resnet import resnet50 -from mindspore.train.model import Model -from mindspore.train.serialization import load_checkpoint, load_param_into_net -from crossentropy import CrossEntropy - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=False, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=True, help='Do eval or not.') -parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -args_opt = parser.parse_args() - -device_id = int(os.getenv('DEVICE_ID')) - -context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False) -context.set_context(device_id=device_id) - -if __name__ == '__main__': - - net = resnet50(class_num=config.class_num) - if not config.label_smooth: - config.label_smooth_factor = 0.0 - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - - if args_opt.do_eval: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size) - step_size = dataset.get_dataset_size() - - if args_opt.checkpoint_path: - param_dict = load_checkpoint(args_opt.checkpoint_path) - load_param_into_net(net, param_dict) - net.set_train(False) - - model = Model(net, loss_fn=loss, metrics={'acc'}) - res = model.eval(dataset) - print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/example/resnet50_imagenet2012_THOR/model/dataset_helper.py b/example/resnet50_imagenet2012_THOR/model/dataset_helper.py deleted file mode 100644 index 77f67344c2b..00000000000 --- a/example/resnet50_imagenet2012_THOR/model/dataset_helper.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""Dataset help for minddata dataset""" -from mindspore._checkparam import check_bool -from mindspore.parallel._utils import _get_device_num, _get_parallel_mode -from mindspore.train.dataset_helper import _send_data -from mindspore.train._utils import _exec_datagraph, _get_types_and_shapes, \ - _to_full_shapes -from mindspore.train.parallel_utils import ParallelMode - - -class DatasetHelper: - """ - Help function to use the Minddata dataset. - - According to different context, change the iter of dataset, to use the same for loop in different context. - - Note: - The iter of DatasetHelper will give one epoch data. - - Args: - dataset (DataSet): The dataset. - dataset_sink_mode (bool): If true use GetNext to fetch the data, or else feed the data from host. - Default: True. - - Examples: - >>> dataset_helper = DatasetHelper(dataset) - >>> for inputs in dataset_helper: - >>> outputs = network(*inputs) - """ - - def __init__(self, dataset, dataset_sink_mode=True, iter_first_order=0): - check_bool(dataset_sink_mode) - self.iter = _DatasetIterMSLoopSink(dataset, iter_first_order) - - def __iter__(self): - return self.iter.__iter__() - - # A temp solution for loop sink. Delete later - def types_shapes(self): - """Get the types and shapes from dataset on current config.""" - return self.iter.types_shapes() - - def loop_size(self): - """Get loop_size for every iteration.""" - return self.iter.loop_size - - -class _DatasetIter: - """Base iter for dataset help""" - - def __init__(self, dataset): - self.loop_size = 1 - if not hasattr(dataset, '__ME_INITED__'): - if not hasattr(dataset, '__loop_size__'): - self.loop_size = dataset.get_dataset_size() - else: - self.loop_size = dataset.__loop_size__ - dataset.__TRANSFER_DATASET__ = _exec_datagraph(dataset, self.loop_size) - dataset.__ME_INITED__ = dataset.__TRANSFER_DATASET__.queue_name - - if not hasattr(dataset, '__no_send__'): - _send_data(dataset) - else: - _send_data(dataset) - - self.ind = 0 - self.dataset = dataset - dataset_types, dataset_shapes = _get_types_and_shapes(dataset) - self.dataset_types, self.dataset_shapes = dataset_types, dataset_shapes - - def __iter__(self): - self.ind = 0 - return self - - def __next__(self): - if self.ind >= self.loop_count: - raise StopIteration() - self.ind += 1 - return self.op() - - def types_shapes(self): - return self.dataset_types, self.dataset_shapes - - def get_loop_count(self, dataset): - loop_count = 1 - if hasattr(dataset, '__loop_size__'): - loop_size = dataset.__loop_size__ - if dataset.get_dataset_size() % loop_size != 0: - raise ValueError(f'Dataset size {dataset.get_dataset_size()} and ' - f'loop_size {loop_size} are not matched.') - loop_count = int(dataset.get_dataset_size() / loop_size) - return loop_count - - -class _DatasetIterMSLoopSink(_DatasetIter): - """Iter for context (device_target=Ascend)""" - - def __init__(self, dataset, iter_first_order): - super(_DatasetIterMSLoopSink, self).__init__(dataset) - loop_size = dataset.__loop_size__ + iter_first_order - self.loop_count = int(dataset.get_dataset_size() / loop_size) * 2 - # for self._parallel_mode equal to semi_auto_parallel or auto_parallel, use a complete tensor to - # compile, and slice tensor to run. The batch dimension of tensors for compile is device_number - # times the batch dimension of tensors for run. Now only support LoopSink. - if _get_parallel_mode() in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): - device_num = _get_device_num() - self.dataset_shapes = _to_full_shapes(self.dataset_shapes, device_num) - - def op(): - return tuple() - - self.op = op diff --git a/example/resnet50_imagenet2012_THOR/model/grad_reducer_thor.py b/example/resnet50_imagenet2012_THOR/model/grad_reducer_thor.py deleted file mode 100644 index ad8d8dd8e4c..00000000000 --- a/example/resnet50_imagenet2012_THOR/model/grad_reducer_thor.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""grad_reducer_thor""" -import mindspore.common.dtype as mstype -from mindspore.communication.management import GlobalComm, get_group_size -from mindspore.nn.cell import Cell -from mindspore.ops import functional as F, composite as C, operations as P -from mindspore.ops.operations.comm_ops import AllReduce, ReduceOp - -reduce_opt = C.MultitypeFuncGraph("reduce_opt") - -_all_reduce_A = AllReduce() - - -def _init_optimizer_allreduce(group): - global _all_reduce_A - _all_reduce_A = AllReduce(ReduceOp.SUM, GlobalComm.WORLD_COMM_GROUP) - _all_reduce_A.add_prim_attr('fusion', group) - - -@reduce_opt.register("Function", "Number", "Tensor") -def _tensors_allreduce_mean(mul, degree, grad): - degree = F.scalar_cast(degree, F.dtype(grad)) - grad = _all_reduce_A(grad) - cast_op = P.Cast() - return mul(grad, cast_op(F.scalar_to_array(1.0 / degree), F.dtype(grad))) - - -@reduce_opt.register("Bool", "Tensor") -def _tensors_allreduce(allreduce_filter, grad): - if allreduce_filter: - return _all_reduce_A(grad) - return grad - - -_get_datatype = C.MultitypeFuncGraph("_get_datatype") - - -@_get_datatype.register("Tensor") -def _tensors_get_datatype(grad): - """ - Acquire gradient datatype. - - Args: - grad (Tensor): The gradient tensor before operation. - - Returns: - mstype, the datatype of gradient. - """ - return F.dtype(grad) - - -_cast_datatype = C.MultitypeFuncGraph("_cast_datatype") - - -@_cast_datatype.register("TypeType", "Tensor") -def _tensors_cast_datatype(datatype, grad): - """ - Cast gradient to datatype. - - Args: - datatype (mstype): the destination datatype of gradient. - grad (Tensor): The gradient tensor before operation. - - Returns: - Tensor, the gradient tensor after operation. - """ - return F.cast(grad, datatype) - - -class DistributedGradReducerThor(Cell): - """ - A distributed optimizer. - - Constructs a gradient reducer Cell, which applies communication and average operations on - single-process gradient values. - - Args: - parameters (list): the parameters to be updated. - mean (bool): When mean is true, the mean coefficient (degree) would apply on gradients. Default: False. - degree (int): The mean coefficient. Usually it equals to device number. Default: None. - - Raises: - ValueError: If degree is not a int or less than 0. - - Examples: - >>> from mindspore.communication import init, get_group_size - >>> from mindspore.ops import composite as C - >>> from mindspore.ops import operations as P - >>> from mindspore.ops import functional as F - >>> from mindspore import context - >>> from mindspore import nn - >>> from mindspore import ParallelMode, ParameterTuple - >>> - >>> device_id = int(os.environ["DEVICE_ID"]) - >>> context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=True, - >>> device_id=int(device_id), enable_hccl=True) - >>> init() - >>> context.reset_auto_parallel_context() - >>> context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL) - >>> - >>> - >>> class TrainingWrapper(nn.Cell): - >>> def __init__(self, network, optimizer, sens=1.0): - >>> super(TrainingWrapper, self).__init__(auto_prefix=False) - >>> self.network = network - >>> self.network.add_flags(defer_inline=True) - >>> self.weights = ParameterTuple(network.trainable_params()) - >>> self.optimizer = optimizer - >>> self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) - >>> self.sens = sens - >>> self.reducer_flag = False - >>> self.grad_reducer = None - >>> self.parallel_mode = context.get_auto_parallel_context("parallel_mode") - >>> if self.parallel_mode in [ParallelMode.DATA_PARALLEL, - >>> ParallelMode.HYBRID_PARALLEL]: - >>> self.reducer_flag = True - >>> if self.reducer_flag: - >>> mean = context.get_auto_parallel_context("mirror_mean") - >>> if mean.get_device_num_is_set(): - >>> degree = context.get_auto_parallel_context("device_num") - >>> else: - >>> degree = get_group_size() - >>> self.grad_reducer = nn.DistributedGradReducer(optimizer.parameters, mean, degree) - >>> - >>> def construct(self, *args): - >>> weights = self.weights - >>> loss = self.network(*args) - >>> sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) - >>> grads = self.grad(self.network, weights)(*args, sens) - >>> if self.reducer_flag: - >>> # apply grad reducer on grads - >>> grads = self.grad_reducer(grads) - >>> return F.depend(loss, self.optimizer(grads)) - >>> - >>> network = Net() - >>> optimizer = nn.Momentum(network.trainable_params(), learning_rate=0.1, momentum=0.9) - >>> train_cell = TrainingWrapper(network, optimizer) - >>> inputs = Tensor(np.ones([16, 16]).astype(np.float32)) - >>> label = Tensor(np.zeros([16, 16]).astype(np.float32)) - >>> grads = train_cell(inputs, label) - """ - - def __init__(self, parameters, group, mean=True, degree=None): - super(DistributedGradReducerThor, self).__init__(auto_prefix=False) - self.hyper_map = C.HyperMap() - self.mul = P.Mul() - if degree is None: - self.degree = get_group_size() - else: - if not isinstance(degree, int) or degree <= 0: - raise ValueError("Parameter 'degree' in DistributedGradReducer should large than 0 and be int") - self.degree = degree - self.mean = mean - self.allreduce_filter = tuple(x.layerwise_parallel is False for x in parameters) - _init_optimizer_allreduce(group) - - def construct(self, grads): - # In some circumstances, the data precision of grads could be mixed with float16 and float32. Thus, the - # result of AllReduce is unreliable. To solve the problem, grads should be cast to float32 before AllReduce, - # and cast back after the operation. - datatypes = self.hyper_map(F.partial(_get_datatype), grads) - grads = self.hyper_map(F.partial(_cast_datatype, mstype.float32), grads) - - if self.mean: - new_grad = self.hyper_map(F.partial(reduce_opt, self.mul, self.degree), grads) - else: - new_grad = self.hyper_map(F.partial(reduce_opt), self.allreduce_filter, grads) - - new_grad = self.hyper_map(F.partial(_cast_datatype), datatypes, new_grad) - return new_grad diff --git a/example/resnet50_imagenet2012_THOR/model/model_thor.py b/example/resnet50_imagenet2012_THOR/model/model_thor.py deleted file mode 100644 index 25e3dd7f823..00000000000 --- a/example/resnet50_imagenet2012_THOR/model/model_thor.py +++ /dev/null @@ -1,725 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""Model.""" - -import numpy as np -from mindspore import context -from mindspore import log as logger -from mindspore import nn -from mindspore._c_expression import init_exec_dataset -from mindspore._checkparam import check_input_data, check_output_data, check_int_positive, check_bool -from mindspore.common import dtype as mstype -from mindspore.common.dtype import pytype_to_dtype -from mindspore.common.tensor import Tensor -from mindspore.nn.metrics import Loss -from mindspore.nn.metrics import get_metrics -from mindspore.nn.wrap.cell_wrapper import _VirtualDatasetCell -from mindspore.parallel._utils import _get_parallel_mode, _get_device_num, _get_global_rank, \ - _get_parameter_broadcast, _device_number_check, _parameter_broadcast_check -from mindspore.train import amp -from mindspore.train.callback import _InternalCallbackParam, RunContext, _CallbackManager -from mindspore.train.parallel_utils import ParallelMode - -from model.dataset_helper import DatasetHelper - - -def _convert_type(types): - """ - Convert from numpy type to tensor type. - - Args: - types (list): Numpy type list of element in dataset. - - Returns: - list, list of element in dataset. - """ - ms_types = [] - for np_type in types: - ms_type = pytype_to_dtype(np_type) - ms_types.append(ms_type) - return ms_types - - -def _get_types_and_shapes(dataset): - """Get dataset types and shapes.""" - dataset_types = _convert_type(dataset.output_types()) - dataset_shapes = dataset.output_shapes() - return dataset_types, dataset_shapes - - -def _exec_datagraph(exec_dataset, dataset_size, phase='dataset'): - """Initialize and execute the dataset graph.""" - batch_size = exec_dataset.get_batch_size() - input_indexs = exec_dataset.input_indexs - - # transform data format - dataset_types, dataset_shapes = _get_types_and_shapes(exec_dataset) - init_exec_dataset(exec_dataset.__ME_INITED__, - dataset_size, - batch_size, - dataset_types, - dataset_shapes, - input_indexs, - phase=phase, - need_run=False) - - -class Model: - """ - High-Level API for Training or Testing. - - `Model` groups layers into an object with training and inference features. - - Args: - network (Cell): The training or testing network. - loss_fn (Cell): Objective function, if loss_fn is None, the - network should contain the logic of loss and grads calculation, and the logic - of parallel if needed. Default: None. - optimizer (Cell): Optimizer for updating the weights. Default: None. - metrics (Union[dict, set]): Dict or set of metrics to be evaluated by the model during - training and testing. eg: {'accuracy', 'recall'}. Default: None. - eval_network (Cell): Network for evaluation. If not defined, `network` and `loss_fn` would be wrapped as - `eval_network`. Default: None. - eval_indexes (list): In case of defining the `eval_network`, if `eval_indexes` is None, all outputs of - `eval_network` would be passed to metrics, otherwise `eval_indexes` must contain three - elements, representing the positions of loss value, predict value and label, the loss - value would be passed to `Loss` metric, predict value and label would be passed to other - metric. Default: None. - amp_level (str): Option for argument `level` in `mindspore.amp.build_train_network`, level for mixed - precision training. Supports [O0, O2]. Default: "O0". - - - O0: Do not change. - - O2: Cast network to float16, keep batchnorm run in float32, using dynamic loss scale. - - loss_scale_manager (Union[None, LossScaleManager]): If None, not scale the loss, or else - scale the loss by LossScaleManager. If it is set, overwrite the level setting. It's a eyword argument. - e.g. Use `loss_scale_manager=None` to set the value. - keep_batchnorm_fp32 (bool): Keep Batchnorm run in `float32`. If set, overwrite the level setting. Default: True. - - Examples: - >>> class Net(nn.Cell): - >>> def __init__(self): - >>> super(Net, self).__init__() - >>> self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal') - >>> self.bn = nn.BatchNorm2d(64) - >>> self.relu = nn.ReLU() - >>> self.flatten = nn.Flatten() - >>> self.fc = nn.Dense(64*224*224, 12) # padding=0 - >>> - >>> def construct(self, x): - >>> x = self.conv(x) - >>> x = self.bn(x) - >>> x = self.relu(x) - >>> x = self.flatten(x) - >>> out = self.fc(x) - >>> return out - >>> - >>> net = Net() - >>> loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) - >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) - >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) - >>> dataset = get_dataset() - >>> model.train(2, dataset) - """ - - def __init__(self, network, loss_fn=None, optimizer=None, metrics=None, eval_network=None, - eval_indexes=None, amp_level="O0", frequency=278, stop_epoch=100, **kwargs): - self._network = network - self._loss_fn = loss_fn - self._optimizer = optimizer - self._loss_scale_manager = None - self._loss_scale_manager_set = False - self._keep_bn_fp32 = True - self._check_kwargs(kwargs) - self._amp_level = amp_level - self._process_amp_args(kwargs) - self._parallel_mode = _get_parallel_mode() - self._device_number = _get_device_num() - self._global_rank = _get_global_rank() - self._parameter_broadcast = _get_parameter_broadcast() - self._frequency = frequency - self._stop_epoch = stop_epoch - - self._train_network = self._build_train_network() - self._build_eval_network(metrics, eval_network, eval_indexes) - self._build_predict_network() - - def _process_amp_args(self, kwargs): - if self._amp_level == "O0": - self._keep_bn_fp32 = False - if 'keep_batchnorm_fp32' in kwargs: - self._keep_bn_fp32 = kwargs['keep_batchnorm_fp32'] - if 'loss_scale_manager' in kwargs: - self._loss_scale_manager = kwargs['loss_scale_manager'] - self._loss_scale_manager_set = True - - def _check_kwargs(self, kwargs): - for arg in kwargs: - if arg not in ['loss_scale_manager', 'keep_batchnorm_fp32']: - raise ValueError(f"Unsupport arg '{arg}'") - - def _build_train_network(self): - """Build train network""" - network = self._network - if self._optimizer: - if self._loss_scale_manager_set: - network = amp.build_train_network(network, - self._optimizer, - self._loss_fn, - level=self._amp_level, - loss_scale_manager=self._loss_scale_manager, - keep_batchnorm_fp32=self._keep_bn_fp32) - else: - network = amp.build_train_network(network, - self._optimizer, - self._loss_fn, - level=self._amp_level, - keep_batchnorm_fp32=self._keep_bn_fp32) - elif self._loss_fn: - network = nn.WithLossCell(network, self._loss_fn) - # If need to check if loss_fn is not None, but optimizer is None - - if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): - network.set_auto_parallel() - return network - - def _build_eval_network(self, metrics, eval_network, eval_indexes): - """Build the network for evaluation.""" - self._metric_fns = get_metrics(metrics) - if not self._metric_fns: - return - - if eval_network is not None: - if eval_indexes is not None and not (isinstance(eval_indexes, list) and len(eval_indexes) == 3): - raise ValueError("Eval_indexes must be a list or None. If eval_indexes is a list, length of it \ - must be three. But got {}".format(eval_indexes)) - - self._eval_network = eval_network - self._eval_indexes = eval_indexes - else: - if self._loss_fn is None: - raise ValueError("loss_fn can not be None.") - self._eval_network = nn.WithEvalCell(self._network, self._loss_fn, self._amp_level == "O2") - self._eval_indexes = [0, 1, 2] - - if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): - self._eval_network.set_auto_parallel() - - def _build_predict_network(self): - """Build the network for prediction.""" - self._predict_network = self._network - if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): - self._predict_network = _VirtualDatasetCell(self._network) - self._predict_network.set_auto_parallel() - - def _clear_metrics(self): - """Clear metrics local values.""" - for metric in self._metric_fns.values(): - metric.clear() - - def _update_metrics(self, outputs): - """Update metrics local values.""" - if not isinstance(outputs, tuple): - raise ValueError("The `outputs` is not tuple.") - - if self._eval_indexes is not None and len(outputs) < 3: - raise ValueError("The length of `outputs` must be greater than or equal to 3, \ - but got {}".format(len(outputs))) - - for metric in self._metric_fns.values(): - if self._eval_indexes is None: - metric.update(*outputs) - else: - if isinstance(metric, Loss): - metric.update(outputs[self._eval_indexes[0]]) - else: - metric.update(outputs[self._eval_indexes[1]], outputs[self._eval_indexes[2]]) - - def _get_metrics(self): - """Get metrics local values.""" - metrics = dict() - for key, value in self._metric_fns.items(): - metrics[key] = value.eval() - return metrics - - def _get_scaling_sens(self): - """get the scaling sens""" - scaling_sens = 1 - if self._loss_scale_manager is not None: - scaling_sens = self._loss_scale_manager.get_loss_scale() - if self._parallel_mode == ParallelMode.DATA_PARALLEL: - scaling_sens /= self._device_number - return scaling_sens - - def _exec_preprocess(self, network, is_train, phase, dataset, dataset_sink_mode, iter_first_order): - """Initializes dataset.""" - need_wrap = False - if dataset_sink_mode: - # remove later to deal with loop sink - if not hasattr(dataset, '__ME_INITED__') and context.get_context("device_target") == "Ascend" \ - and not context.get_context("enable_ge"): - need_wrap = True - - if not is_train: - dataset.__loop_size__ = 1 - - dataset_helper = DatasetHelper(dataset, dataset_sink_mode, iter_first_order) - - # remove later to deal with loop sink - if need_wrap: - network = nn.DataWrapper(network, *(dataset_helper.types_shapes()), dataset.__ME_INITED__) - network.set_train(is_train) - network.phase = phase - - return dataset_helper, network - - def init(self, train_dataset=None, valid_dataset=None): - """ - Initializes compute graphs and data graphs with sink mode. - - Note: - Pre-init process only supports `GRAPH_MODE` and `Ascend` target currently. - - Args: - train_dataset (Dataset): A training dataset iterator. If define `train_dataset`, training graphs will be - initialized. Default: None. - valid_dataset (Dataset): A evaluating dataset iterator. If define `valid_dataset`, evaluation graphs will - be initialized, and `metrics` in `Model` can not be None. Default: None. - - Examples: - >>> train_dataset = get_train_dataset() - >>> valid_dataset = get_valid_dataset() - >>> net = Net() - >>> loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) - >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) - >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics={'acc'}) - >>> model.init(train_dataset, valid_dataset) - >>> model.train(2, train_dataset) - >>> model.eval(valid_dataset) - """ - if context.get_context("mode") != context.GRAPH_MODE or context.get_context("device_target") != "Ascend": - raise RuntimeError('Pre-init process only supports GRAPH MODE and Ascend target currently.') - - if not train_dataset and not valid_dataset: - raise ValueError('Both train_dataset and valid_dataset can not be None or empty.') - - _device_number_check(self._parallel_mode, self._device_number) - - if train_dataset: - _parameter_broadcast_check(self._parallel_mode, self._parameter_broadcast) - self._train_network.set_train() - self._train_network.phase = 'train' - - if self._parameter_broadcast: - self._train_network.set_broadcast_flag() - - train_dataset_helper, train_network = self._exec_preprocess(self._train_network, - is_train=True, - phase='train', - dataset=train_dataset, - dataset_sink_mode=True) - self._train_network = train_network - for inputs in train_dataset_helper: - self._train_network.compile(*inputs) - break - - if valid_dataset: - if not self._metric_fns: - raise RuntimeError('If define `valid_dataset`, metric fn can not be None or empty.') - - self._eval_network.set_train(False) - self._eval_network.phase = 'eval' - valid_dataset_helper, eval_network = self._exec_preprocess(self._eval_network, - is_train=False, - phase='eval', - dataset=valid_dataset, - dataset_sink_mode=True) - self._eval_network = eval_network - for inputs in valid_dataset_helper: - self._eval_network.compile(*inputs) - break - - def _train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=True): - """ - Training. - - Args: - epoch (int): Total number of iterations on the data. - train_dataset (Dataset): A training dataset iterator. If there is no - loss_fn, a tuple with multiply data (data1, data2, data3, ...) will be - returned and passed to the network. Otherwise, a tuple (data, label) will - be returned, and the data and label are passed to the network and loss - function respectively. - callbacks (list): List of callback object. Callbacks which should be executed while training. Default: None. - dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. - Configure pynative mode, the training process will be performed with - dataset not sink. - """ - epoch = check_int_positive(epoch) - self._train_network.set_train() - - if self._parameter_broadcast: - self._train_network.set_broadcast_flag() - - # build callback list - cb_params = _InternalCallbackParam() - cb_params.train_network = self._train_network - cb_params.epoch_num = epoch - cb_params.batch_num = train_dataset.get_dataset_size() - cb_params.mode = "train" - cb_params.loss_fn = self._loss_fn - cb_params.optimizer = self._optimizer - cb_params.parallel_mode = self._parallel_mode - cb_params.device_number = self._device_number - cb_params.train_dataset = train_dataset - cb_params.list_callback = callbacks - - with _CallbackManager(callbacks) as list_callback: - if not dataset_sink_mode: - self._train_process(epoch, train_dataset, list_callback, cb_params) - elif context.get_context("mode") == context.PYNATIVE_MODE: - logger.warning("The pynative mode cannot support dataset sink mode currently." - "So the training process will be performed with dataset not sink.") - self._train_process(epoch, train_dataset, list_callback, cb_params) - else: - self._train_dataset_sink_process(epoch, train_dataset, list_callback, cb_params) - - def _train_dataset_sink_process(self, epoch, train_dataset, list_callback=None, cb_params=None): - """ - Training process. The data would be passed to network through dataset channel. - - Args: - epoch (int): Total number of iterations on the data. - train_dataset (Dataset): A training dataset iterator. If there is no - loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be - returned and passed to the network. Otherwise, a tuple (data, label) should - be returned, and the data and label are passed to the network and loss - function respectively. - list_callback (Callback): Executor of callback list. Default: None. - cb_params (_InternalCallbackParam): Callback parameters. Default: None. - """ - iter_first_order = self._frequency - 1 - iter_second_order = 1 - train_dataset.__loop_size__ = iter_second_order - dataset_helper, train_network = self._exec_preprocess(self._train_network, - is_train=True, - phase='train', - dataset=train_dataset, - dataset_sink_mode=True, - iter_first_order=iter_first_order) - self._train_network = train_network - cb_params.train_network = self._train_network - cb_params.cur_step_num = 0 - - loop_size = dataset_helper.loop_size() - run_context = RunContext(cb_params) - list_callback.begin(run_context) - - # used to stop training for early stop, such as stopAtTIme or stopATStep - should_stop = False - has_do_dataset_init = False - switch_branch_one = True - for i in range(epoch): - cb_params.cur_epoch_num = i + 1 - list_callback.epoch_begin(run_context) - - # for data sink dataset_helper only iter once, other wise iter epoch_size times. - for inputs in dataset_helper: - list_callback.step_begin(run_context) - if switch_branch_one: - cb_params.cur_step_num += loop_size - self._train_network.add_flags_recursive(thor=True) - self._train_network.phase = 'train0' - else: - cb_params.cur_step_num += iter_first_order - self._train_network.add_flags_recursive(thor=False) - self._train_network.phase = 'train1' - if not has_do_dataset_init: - _exec_datagraph(train_dataset, iter_first_order, phase='train1_dataset') - has_do_dataset_init = True - switch_branch_one = not switch_branch_one - outputs = self._train_network(*inputs) - cb_params.net_outputs = outputs - list_callback.step_end(run_context) - - list_callback.epoch_end(run_context) - should_stop = should_stop or run_context.get_stop_requested() - if should_stop: - break - - list_callback.end(run_context) - - def _train_process(self, epoch, train_dataset, list_callback=None, cb_params=None): - """ - Training process. The data would be passed to network directly. - - Args: - epoch (int): Total number of iterations on the data. - train_dataset (Dataset): A training dataset iterator. If there is no - loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be - returned and passed to the network. Otherwise, a tuple (data, label) should - be returned, and the data and label are passed to the network and loss - function respectively. - list_callback (Callback): Executor of callback list. Default: None. - cb_params (_InternalCallbackParam): Callback parameters. Default: None. - """ - dataset_helper, _ = self._exec_preprocess(self._train_network, - is_train=True, - phase='train', - dataset=train_dataset, - dataset_sink_mode=False) - cb_params.cur_step_num = 0 - run_context = RunContext(cb_params) - list_callback.begin(run_context) - # used to stop training for early stop, such as stopAtTIme or stopATStep - should_stop = False - - for i in range(epoch): - cb_params.cur_epoch_num = i + 1 - - list_callback.epoch_begin(run_context) - - for next_element in dataset_helper: - len_element = len(next_element) - if self._loss_fn and len_element != 2: - raise ValueError("when loss_fn is not None, train_dataset should" - "return two elements, but got {}".format(len_element)) - cb_params.cur_step_num += 1 - list_callback.step_begin(run_context) - - overflow = False - if self._loss_scale_manager and self._loss_scale_manager.get_drop_overflow_update(): - scaling_sens = self._get_scaling_sens() - next_element = tuple(next_element) + (Tensor(scaling_sens, mstype.float32),) - - outputs = self._train_network(*next_element) - cb_params.net_outputs = outputs - if self._loss_scale_manager and self._loss_scale_manager.get_drop_overflow_update(): - _, overflow, _ = outputs - overflow = np.all(overflow.asnumpy()) - self._loss_scale_manager.update_loss_scale(overflow) - - list_callback.step_end(run_context) - should_stop = should_stop or run_context.get_stop_requested() - if should_stop: - break - - train_dataset.reset() - - list_callback.epoch_end(run_context) - should_stop = should_stop or run_context.get_stop_requested() - if should_stop: - break - - list_callback.end(run_context) - - def train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=True): - """ - Training API where the iteration is controlled by python front-end. - - When setting pynative mode, the training process will be performed with dataset not sink. - - Note: - CPU is not supported when dataset_sink_mode is true. - If dataset_sink_mode is True, epoch of training should be equal to the count of repeat - operation in dataset processing. Otherwise, errors could occur since the amount of data - is not the amount training requires. - If dataset_sink_mode is True, data will be sent to device. If device is Ascend, features - of data will be transferred one by one. The limitation of data transmission per time is 256M. - - Args: - epoch (int): Total number of iterations on the data. - train_dataset (Dataset): A training dataset iterator. If there is no - loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be - returned and passed to the network. Otherwise, a tuple (data, label) should - be returned, and the data and label are passed to the network and loss - function respectively. - callbacks (list): List of callback object. Callbacks which should be excuted while training. Default: None. - dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. - Configure pynative mode, the training process will be performed with - dataset not sink. - - - Examples: - >>> dataset = get_dataset() - >>> net = Net() - >>> loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) - >>> loss_scale_manager = FixedLossScaleManager() - >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) - >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None, loss_scale_manager=loss_scale_manager) - >>> model.train(2, dataset) - """ - repeat_count = train_dataset.get_repeat_count() - if epoch != repeat_count and dataset_sink_mode is True: - logger.warning(f"The epoch_size {epoch} is not the same with dataset repeat_count {repeat_count}") - check_bool(dataset_sink_mode) - _device_number_check(self._parallel_mode, self._device_number) - _parameter_broadcast_check(self._parallel_mode, self._parameter_broadcast) - - self._train(epoch, - train_dataset, - callbacks=callbacks, - dataset_sink_mode=dataset_sink_mode) - - def _eval_dataset_sink_process(self, valid_dataset, list_callback=None, cb_params=None): - """ - Evaluation. The data would be passed to network through dataset channel. - - Args: - valid_dataset (Dataset): Dataset to evaluate the model. - list_callback (Callback): Executor of callback list. Default: None. - cb_params (_InternalCallbackParam): Callback parameters. Default: None. - - Returns: - Dict, returns the loss value & metrics values for the model in test mode. - """ - run_context = RunContext(cb_params) - - dataset_helper, eval_network = self._exec_preprocess(self._eval_network, - is_train=False, - phase='eval', - dataset=valid_dataset, - dataset_sink_mode=True) - self._eval_network = eval_network - cb_params.eval_network = self._eval_network - list_callback.begin(run_context) - - for inputs in dataset_helper: - cb_params.cur_step_num += 1 - list_callback.step_begin(run_context) - - outputs = self._eval_network(*inputs) - - cb_params.net_outputs = outputs - list_callback.step_end(run_context) - self._update_metrics(outputs) - - metrics = self._get_metrics() - cb_params.metrics = metrics - list_callback.end(run_context) - - return metrics - - def _eval_process(self, valid_dataset, list_callback=None, cb_params=None): - """ - Evaluation. The data would be passed to network directly. - - Args: - valid_dataset (Dataset): Dataset to evaluate the model. - list_callback (Callback): Executor of callback list. Default: None. - cb_params (_InternalCallbackParam): Callback parameters. Default: None. - - Returns: - Dict, returns the loss value & metrics values for the model in test mode. - """ - run_context = RunContext(cb_params) - list_callback.begin(run_context) - - dataset_helper, _ = self._exec_preprocess(self._eval_network, - is_train=False, - phase='eval', - dataset=valid_dataset, - dataset_sink_mode=False) - for next_element in dataset_helper: - cb_params.cur_step_num += 1 - list_callback.step_begin(run_context) - outputs = self._eval_network(*next_element) - cb_params.net_outputs = outputs - list_callback.step_end(run_context) - self._update_metrics(outputs) - - metrics = self._get_metrics() - cb_params.metrics = metrics - list_callback.end(run_context) - return metrics - - def eval(self, valid_dataset, callbacks=None, dataset_sink_mode=True): - """ - Evaluation API where the iteration is controlled by python front-end. - - Configure to pynative mode, the evaluation will be performed with dataset non-sink mode. - - Note: - CPU is not supported when dataset_sink_mode is true. - If dataset_sink_mode is True, data will be sent to device. If device is Ascend, features - of data will be transferred one by one. The limitation of data transmission per time is 256M. - - Args: - valid_dataset (Dataset): Dataset to evaluate the model. - callbacks (list): List of callback object. Callbacks which should be excuted - while training. Default: None. - dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. - - Returns: - Dict, returns the loss value & metrics values for the model in test mode. - - Examples: - >>> dataset = get_dataset() - >>> net = Net() - >>> loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) - >>> model = Model(net, loss_fn=loss, optimizer=None, metrics={'acc'}) - >>> model.eval(dataset) - """ - check_bool(dataset_sink_mode) - _device_number_check(self._parallel_mode, self._device_number) - if not self._metric_fns: - raise ValueError("metric fn can not be None or empty.") - - cb_params = _InternalCallbackParam() - cb_params.eval_network = self._eval_network - cb_params.valid_dataset = valid_dataset - cb_params.batch_num = valid_dataset.get_dataset_size() - cb_params.mode = "eval" - cb_params.cur_step_num = 0 - - self._eval_network.set_train(mode=False) - self._eval_network.phase = 'eval' - - self._clear_metrics() - - with _CallbackManager(callbacks) as list_callback: - if dataset_sink_mode: - return self._eval_dataset_sink_process(valid_dataset, list_callback, cb_params) - return self._eval_process(valid_dataset, list_callback, cb_params) - - def predict(self, *predict_data): - """ - Generates output predictions for the input samples. - - Data could be single tensor, or list of tensor, tuple of tensor. - - Note: - Batch data should be put together in one tensor. - - Args: - predict_data (Tensor): Tensor of predict data. can be array, list or tuple. - - Returns: - Tensor, array(s) of predictions. - - Examples: - >>> input_data = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]), mindspore.float32) - >>> model = Model(Net()) - >>> model.predict(input_data) - """ - self._predict_network.set_train(False) - check_input_data(*predict_data, data_class=Tensor) - result = self._predict_network(*predict_data) - - check_output_data(result) - return result - - -__all__ = ["Model"] diff --git a/example/resnet50_imagenet2012_THOR/model/resnet.py b/example/resnet50_imagenet2012_THOR/model/resnet.py deleted file mode 100644 index f3305022e87..00000000000 --- a/example/resnet50_imagenet2012_THOR/model/resnet.py +++ /dev/null @@ -1,359 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""ResNet.""" -import math -import numpy as np -import mindspore.nn as nn -from mindspore.common.tensor import Tensor -from mindspore.ops import operations as P - -from model.thor_layer import Conv2d_Thor, Dense_Thor - - -def calculate_gain(nonlinearity, param=None): - """calculate_gain""" - linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d'] - res = 0 - if nonlinearity in linear_fns or nonlinearity == 'sigmoid': - res = 1 - elif nonlinearity == 'tanh': - res = 5.0 / 3 - elif nonlinearity == 'relu': - res = math.sqrt(2.0) - elif nonlinearity == 'leaky_relu': - if param is None: - negative_slope = 0.01 - elif not isinstance(param, bool) and isinstance(param, int) or isinstance(param, float): - # True/False are instances of int, hence check above - negative_slope = param - else: - raise ValueError("negative_slope {} not a valid number".format(param)) - res = math.sqrt(2.0 / (1 + negative_slope ** 2)) - else: - raise ValueError("Unsupported nonlinearity {}".format(nonlinearity)) - return res - - -def _calculate_fan_in_and_fan_out(tensor): - """_calculate_fan_in_and_fan_out""" - dimensions = len(tensor) - if dimensions < 2: - raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions") - if dimensions == 2: # Linear - fan_in = tensor[1] - fan_out = tensor[0] - else: - num_input_fmaps = tensor[1] - num_output_fmaps = tensor[0] - receptive_field_size = 1 - if dimensions > 2: - receptive_field_size = tensor[2] * tensor[3] - fan_in = num_input_fmaps * receptive_field_size - fan_out = num_output_fmaps * receptive_field_size - return fan_in, fan_out - - -def _calculate_correct_fan(tensor, mode): - mode = mode.lower() - valid_modes = ['fan_in', 'fan_out'] - if mode not in valid_modes: - raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes)) - fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) - return fan_in if mode == 'fan_in' else fan_out - - -def kaiming_normal(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu'): - fan = _calculate_correct_fan(inputs_shape, mode) - gain = calculate_gain(nonlinearity, a) - std = gain / math.sqrt(fan) - return np.random.normal(0, std, size=inputs_shape).astype(np.float32) - - -def kaiming_uniform(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu'): - fan = _calculate_correct_fan(inputs_shape, mode) - gain = calculate_gain(nonlinearity, a) - std = gain / math.sqrt(fan) - bound = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation - return np.random.uniform(-bound, bound, size=inputs_shape).astype(np.float32) - - -def _conv3x3(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): - weight_shape = (out_channel, in_channel, 3, 3) - weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) - return Conv2d_Thor(in_channel, out_channel, - kernel_size=3, stride=stride, padding=0, pad_mode='same', weight_init=weight, - damping=damping, loss_scale=loss_scale, frequency=frequency) - - -def _conv1x1(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): - weight_shape = (out_channel, in_channel, 1, 1) - weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) - return Conv2d_Thor(in_channel, out_channel, - kernel_size=1, stride=stride, padding=0, pad_mode='same', weight_init=weight, - damping=damping, loss_scale=loss_scale, frequency=frequency) - - -def _conv7x7(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): - weight_shape = (out_channel, in_channel, 7, 7) - weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) - return Conv2d_Thor(in_channel, out_channel, - kernel_size=7, stride=stride, padding=0, pad_mode='same', weight_init=weight, - damping=damping, loss_scale=loss_scale, frequency=frequency) - - -def _bn(channel): - return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, - gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) - - -def _bn_last(channel): - return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, - gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) - - -def _fc(in_channel, out_channel, damping, loss_scale, frequency): - weight_shape = (out_channel, in_channel) - weight = Tensor(kaiming_uniform(weight_shape, a=math.sqrt(5))) - return Dense_Thor(in_channel, out_channel, has_bias=False, weight_init=weight, - bias_init=0, damping=damping, loss_scale=loss_scale, frequency=frequency) - - -class ResidualBlock(nn.Cell): - """ - ResNet V1 residual block definition. - - Args: - in_channel (int): Input channel. - out_channel (int): Output channel. - stride (int): Stride size for the first convolutional layer. Default: 1. - - Returns: - Tensor, output tensor. - - Examples: - >>> ResidualBlock(3, 256, stride=2) - """ - expansion = 4 - - def __init__(self, - in_channel, - out_channel, - stride=1, - damping=0.03, - loss_scale=1, - frequency=278): - super(ResidualBlock, self).__init__() - - channel = out_channel // self.expansion - self.conv1 = _conv1x1(in_channel, channel, stride=1, damping=damping, loss_scale=loss_scale, - frequency=frequency) - self.bn1 = _bn(channel) - - self.conv2 = _conv3x3(channel, channel, stride=stride, damping=damping, loss_scale=loss_scale, - frequency=frequency) - self.bn2 = _bn(channel) - - self.conv3 = _conv1x1(channel, out_channel, stride=1, damping=damping, loss_scale=loss_scale, - frequency=frequency) - self.bn3 = _bn_last(out_channel) - - self.relu = nn.ReLU() - - self.down_sample = False - - if stride != 1 or in_channel != out_channel: - self.down_sample = True - self.down_sample_layer = None - - if self.down_sample: - self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channel, out_channel, stride, - damping=damping, loss_scale=loss_scale, - frequency=frequency), - _bn(out_channel)]) - self.add = P.TensorAdd() - - def construct(self, x): - identity = x - - out = self.conv1(x) - out = self.bn1(out) - out = self.relu(out) - - out = self.conv2(out) - out = self.bn2(out) - out = self.relu(out) - - out = self.conv3(out) - out = self.bn3(out) - - if self.down_sample: - identity = self.down_sample_layer(identity) - - out = self.add(out, identity) - out = self.relu(out) - - return out - - -class ResNet(nn.Cell): - """ - ResNet architecture. - - Args: - block (Cell): Block for network. - layer_nums (list): Numbers of block in different layers. - in_channels (list): Input channel in each layer. - out_channels (list): Output channel in each layer. - strides (list): Stride size in each layer. - num_classes (int): The number of classes that the training images are belonging to. - Returns: - Tensor, output tensor. - - Examples: - >>> ResNet(ResidualBlock, - >>> [3, 4, 6, 3], - >>> [64, 256, 512, 1024], - >>> [256, 512, 1024, 2048], - >>> [1, 2, 2, 2], - >>> 10) - """ - - def __init__(self, - block, - layer_nums, - in_channels, - out_channels, - strides, - num_classes, - damping, - loss_scale, - frequency): - super(ResNet, self).__init__() - - if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: - raise ValueError("the length of layer_num, in_channels, out_channels list must be 4!") - - self.conv1 = _conv7x7(3, 64, stride=2, damping=damping, loss_scale=loss_scale, frequency=frequency) - self.bn1 = _bn(64) - self.relu = P.ReLU() - self.maxpool = P.MaxPoolWithArgmax(padding="same", ksize=3, strides=2) - - self.layer1 = self._make_layer(block, - layer_nums[0], - in_channel=in_channels[0], - out_channel=out_channels[0], - stride=strides[0], - damping=damping, - loss_scale=loss_scale, - frequency=frequency) - self.layer2 = self._make_layer(block, - layer_nums[1], - in_channel=in_channels[1], - out_channel=out_channels[1], - stride=strides[1], - damping=damping, - loss_scale=loss_scale, - frequency=frequency) - self.layer3 = self._make_layer(block, - layer_nums[2], - in_channel=in_channels[2], - out_channel=out_channels[2], - stride=strides[2], damping=damping, - loss_scale=loss_scale, - frequency=frequency) - self.layer4 = self._make_layer(block, - layer_nums[3], - in_channel=in_channels[3], - out_channel=out_channels[3], - stride=strides[3], - damping=damping, - loss_scale=loss_scale, - frequency=frequency) - - self.mean = P.ReduceMean(keep_dims=True) - self.flatten = nn.Flatten() - self.end_point = _fc(out_channels[3], num_classes, damping=damping, loss_scale=loss_scale, frequency=frequency) - - def _make_layer(self, block, layer_num, in_channel, out_channel, stride, - damping, loss_scale, frequency): - """ - Make stage network of ResNet. - - Args: - block (Cell): Resnet block. - layer_num (int): Layer number. - in_channel (int): Input channel. - out_channel (int): Output channel. - stride (int): Stride size for the first convolutional layer. - - Returns: - SequentialCell, the output layer. - - Examples: - >>> _make_layer(ResidualBlock, 3, 128, 256, 2) - """ - layers = [] - - resnet_block = block(in_channel, out_channel, stride=stride, - damping=damping, loss_scale=loss_scale, frequency=frequency) - layers.append(resnet_block) - - for _ in range(1, layer_num): - resnet_block = block(out_channel, out_channel, stride=1, - damping=damping, loss_scale=loss_scale, frequency=frequency) - layers.append(resnet_block) - - return nn.SequentialCell(layers) - - def construct(self, x): - x = self.conv1(x) - x = self.bn1(x) - x = self.relu(x) - c1, _ = self.maxpool(x) - - c2 = self.layer1(c1) - c3 = self.layer2(c2) - c4 = self.layer3(c3) - c5 = self.layer4(c4) - - out = self.mean(c5, (2, 3)) - out = self.flatten(out) - out = self.end_point(out) - - return out - - -def resnet50(class_num=10, damping=0.03, loss_scale=1, frequency=278): - """ - Get ResNet50 neural network. - - Args: - class_num (int): Class number. - - Returns: - Cell, cell instance of ResNet50 neural network. - - Examples: - >>> net = resnet50(10) - """ - return ResNet(ResidualBlock, - [3, 4, 6, 3], - [64, 256, 512, 1024], - [256, 512, 1024, 2048], - [1, 2, 2, 2], - class_num, - damping, - loss_scale, - frequency) diff --git a/example/resnet50_imagenet2012_THOR/model/thor.py b/example/resnet50_imagenet2012_THOR/model/thor.py deleted file mode 100644 index 6786cb74857..00000000000 --- a/example/resnet50_imagenet2012_THOR/model/thor.py +++ /dev/null @@ -1,199 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""momentum""" -import mindspore.common.dtype as mstype -from mindspore.common.initializer import initializer -from mindspore.common.parameter import Parameter -from mindspore.common.parameter import ParameterTuple -from mindspore.common.tensor import Tensor -from mindspore.nn.optim.optimizer import Optimizer -from mindspore.ops import functional as F, composite as C, operations as P -from mindspore.parallel._utils import _get_device_num, _get_mirror_mean -from model.grad_reducer_thor import DistributedGradReducerThor - -momentum_opt = C.MultitypeFuncGraph("momentum_opt") - - -@momentum_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor") -def _tensor_run_opt_ext(opt, learning_rate, momentum, gradient, weight, moment): - """Apply momentum optimizer to the weight parameter using Tensor.""" - success = True - success = F.depend(success, opt(weight, moment, learning_rate, gradient, momentum)) - return success - - -op_add = P.AddN() -apply_decay = C.MultitypeFuncGraph("apply_decay") - - -@apply_decay.register("Number", "Bool", "Tensor", "Tensor") -def _tensor_apply_decay(weight_decay, if_apply, weight, gradient): - """Get grad with weight_decay.""" - if if_apply: - return op_add((weight * weight_decay, gradient)) - return gradient - - -class THOR(Optimizer): - """THOR""" - def __init__(self, params, learning_rate, momentum, matrix_A, matrix_G, A_inv_max, G_inv_max, weight_decay=0.0, - loss_scale=1.0, - decay_filter=lambda x: x.name not in []): - super(THOR, self).__init__(learning_rate, params, weight_decay, loss_scale) - if isinstance(momentum, float) and momentum < 0.0: - raise ValueError("momentum should be at least 0.0, but got momentum {}".format(momentum)) - self.momentum = Parameter(Tensor(momentum, mstype.float32), name="momentum") - self.params = self.parameters - self.moments = self.params.clone(prefix="moments", init='zeros') - self.hyper_map = C.HyperMap() - self.opt = P.ApplyMomentum() - self.matrix_A = ParameterTuple(matrix_A) - self.matrix_G = ParameterTuple(matrix_G) - self.A_inv_max = ParameterTuple(A_inv_max) - self.G_inv_max = ParameterTuple(G_inv_max) - self.cube_matmul_left = P.CusMatMulCubeFraczLeftCast() - self.cube_matmul_left_fc = P.CusMatMulCubeDenseLeft() - self.cube_matmul_right_fc = P.CusMatMulCubeDenseRight() - self.cube_matmul_right_mul = P.CusMatMulCubeFraczRightMul() - self.transpose = P.Transpose() - self.shape = P.Shape() - self.reshape = P.Reshape() - self.mul = P.Mul() - self.weight_idx = [] - for i in range(len(self.params)): - if "conv" in self.params[i].name or "end_point" in self.params[i].name: - self.weight_idx.append(i) - self.weight_idx.append(len(self.params)) - self.feature_map = [1.0 / 12544, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, - 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, - 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, - 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, - 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, - 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, - 1.0 / 196, 1.0 / 196, 1.0 / 196, - 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, - 1.0] - mean = _get_mirror_mean() - degree = _get_device_num() - self.grad_reducer_Amax = DistributedGradReducerThor(self.parameters, 2, mean, degree) - self.grad_reducer_Gmax = DistributedGradReducerThor(self.parameters, 5, mean, degree) - self.grad_reducer_A = DistributedGradReducerThor(self.parameters, 3, mean, degree) - self.grad_reducer_G = DistributedGradReducerThor(self.parameters, 4, mean, degree) - self.matrix_A_inv = () - self.matrix_G_inv = () - self.matrix_max_inv = () - - for i in range(54): - self.matrix_max_inv = self.matrix_max_inv + ( - Parameter(initializer(1, [1], mstype.float32), name="matrix_max" + str(i), requires_grad=False),) - self.log = P.Log() - self.exp = P.Exp() - self.sqrt = P.Sqrt() - self.matrix_max_inv = ParameterTuple(self.matrix_max_inv) - self.assign = P.Assign() - self.cast = P.Cast() - self.thor = True - self.weight_decay = weight_decay * loss_scale - self.decay_flags = tuple(decay_filter(x) for x in self.parameters) - - def construct(self, gradients): - params = self.params - moments = self.moments - if self.thor: - matrix_A_allreduce = () - matrix_G_allreduce = () - matrix_A_max_allreduce = () - matrix_G_max_allreduce = () - for i in range(54): - g = gradients[i * 3] - matrix_A = self.matrix_A[i] - matrix_G = self.matrix_G[i] - A_max = self.A_inv_max[i] - G_max = self.G_inv_max[i] - matrix_A = F.depend(matrix_A, g) - matrix_G = F.depend(matrix_G, g) - A_max = F.depend(A_max, g) - G_max = F.depend(G_max, g) - matrix_A_allreduce = matrix_A_allreduce + (matrix_A,) - matrix_G_allreduce = matrix_G_allreduce + (matrix_G,) - matrix_A_max_allreduce = matrix_A_max_allreduce + (A_max,) - matrix_G_max_allreduce = matrix_G_max_allreduce + (G_max,) - matrix_A_allreduce = self.grad_reducer_A(matrix_A_allreduce) - matrix_G_allreduce = self.grad_reducer_G(matrix_G_allreduce) - matrix_A_max_allreduce = self.grad_reducer_Amax(matrix_A_max_allreduce) - matrix_G_max_allreduce = self.grad_reducer_Gmax(matrix_G_max_allreduce) - new_grads = () - for i in range(54): - g = gradients[i * 3] - temp_a = matrix_A_allreduce[i] - temp_g = matrix_G_allreduce[i] - temp_a = self.cast(temp_a, mstype.float32) - temp_g = self.cast(temp_g, mstype.float32) - matrix_A_inv_max = self.log(matrix_A_max_allreduce[i]) - matrix_A_inv_max = self.mul(matrix_A_inv_max, -1) - matrix_A_inv_max = self.exp(matrix_A_inv_max) - temp_a = self.mul(temp_a, matrix_A_inv_max) - matrix_G_inv_max = self.log(matrix_G_max_allreduce[i]) - matrix_G_inv_max = self.mul(matrix_G_inv_max, -1) - matrix_G_inv_max = self.exp(matrix_G_inv_max) - temp_g = self.mul(temp_g, matrix_G_inv_max) - temp_max = self.mul(matrix_A_max_allreduce[i], matrix_G_max_allreduce[i]) - temp_max = self.mul(temp_max, self.feature_map[i]) - temp_a = self.cast(temp_a, mstype.float16) - temp_g = self.cast(temp_g, mstype.float16) - if i == 53: - g = self.cube_matmul_left_fc(temp_g, g) - g = self.cube_matmul_right_fc(g, temp_a, temp_max) - else: - g = self.cube_matmul_left(temp_g, g) - g = self.cube_matmul_right_mul(g, temp_a, temp_max) - fake_A = self.assign(self.matrix_A[i], temp_a) - fake_G = self.assign(self.matrix_G[i], temp_g) - fake_max = self.assign(self.matrix_max_inv[i], temp_max) - g = F.depend(g, fake_A) - g = F.depend(g, fake_G) - g = F.depend(g, fake_max) - if i == 53: - new_grads = new_grads + (g,) - else: - new_grads = new_grads + (g, gradients[i * 3 + 1], gradients[i * 3 + 2]) - gradients = new_grads - else: - new_grads = () - for i in range(54): - g = gradients[i * 3] - matrix_A = self.matrix_A[i] - matrix_G = self.matrix_G[i] - matrix_max = self.matrix_max_inv[i] - matrix_A = F.depend(matrix_A, g) - matrix_G = F.depend(matrix_G, g) - matrix_max = F.depend(matrix_max, g) - if i == 53: - g = self.cube_matmul_left_fc(matrix_G, g) - g = self.cube_matmul_right_fc(g, matrix_A, matrix_max) - new_grads = new_grads + (g,) - else: - g = self.cube_matmul_left(matrix_G, g) - g = self.cube_matmul_right_mul(g, matrix_A, matrix_max) - new_grads = new_grads + (g, gradients[i * 3 + 1], gradients[i * 3 + 2]) - gradients = new_grads - - if self.weight_decay > 0: - gradients = self.hyper_map(F.partial(apply_decay, self.weight_decay), self.decay_flags, - params, gradients) - gradients = self.scale_grad(gradients) - lr = self.get_lr() - success = self.hyper_map(F.partial(momentum_opt, self.opt, lr, self.momentum), gradients, params, moments) - return success diff --git a/example/resnet50_imagenet2012_THOR/model/thor_layer.py b/example/resnet50_imagenet2012_THOR/model/thor_layer.py deleted file mode 100644 index d84cbf7a939..00000000000 --- a/example/resnet50_imagenet2012_THOR/model/thor_layer.py +++ /dev/null @@ -1,477 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""thor_layer""" -import numpy as np - -import mindspore as ms -import mindspore.common.dtype as mstype -from mindspore._checkparam import check_bool, twice, check_int_positive -from mindspore._extends import cell_attr_register -from mindspore.common.initializer import initializer -from mindspore.common.parameter import Parameter -from mindspore.common.tensor import Tensor -from mindspore.nn.cell import Cell -from mindspore.nn.layer.activation import get_activation -from mindspore.ops import operations as P -C0 = 16 - -def caculate_device_shape(matrix_dim, channel, is_A): - ll = (0) - if is_A: - if channel // C0 == 0: - matrix_dim = (matrix_dim / channel) * C0 - ll = (int(matrix_dim // C0), int(matrix_dim // C0), C0, C0), int(matrix_dim) - else: - ll = (int(matrix_dim // C0), int(matrix_dim // C0), C0, C0), int(matrix_dim) - return ll - -class _Conv(Cell): - r"""Applies a N-D convolution over an input signal composed of several input - planes. - """ - - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride, - pad_mode, - padding, - dilation, - group, - data_format, - has_bias, - weight_init, - bias_init, - ): - super(_Conv, self).__init__() - self.in_channels = in_channels - self.out_channels = out_channels - self.kernel_size = kernel_size - self.stride = stride - self.pad_mode = pad_mode - self.padding = padding - self.dilation = dilation - self.group = group - self.data_format = data_format - self.has_bias = has_bias - if not (isinstance(in_channels, int) and in_channels > 0): - raise ValueError('Attr \'in_channels\' of \'Conv2D\' Op passed ' - + str(in_channels) + ', should be a int and greater than 0.') - if (not isinstance(kernel_size, tuple)) or len(kernel_size) != 2 or \ - (not isinstance(kernel_size[0], int)) or (not isinstance(kernel_size[1], int)) or \ - kernel_size[0] < 1 or kernel_size[1] < 1: - raise ValueError('Attr \'kernel_size\' of \'Conv2D\' Op passed ' - + str(self.kernel_size) + ', should be a int or tuple and equal to or greater than 1.') - if in_channels % group != 0: - raise ValueError('Attr \'in_channels\' of \'Conv2D\' Op must be divisible by ' - 'attr \'group\' of \'Conv2D\' Op.') - if out_channels % group != 0: - raise ValueError('Attr \'out_channels\' of \'Conv2D\' Op must be divisible by ' - 'attr \'group\' of \'Conv2D\' Op.') - - self.weight = Parameter(initializer( - weight_init, [out_channels, in_channels // group, *kernel_size]), name='weight') - - if check_bool(has_bias): - self.bias = Parameter(_initializer( - bias_init, [out_channels]), name='bias') - else: - if bias_init != 'zeros': - logger.warning("Value of 'has_bias' is False, value of 'bias_init' will be ignored.") - self.bias = None - - def construct(self, *inputs): - raise NotImplementedError - - -class Conv2d_Thor(_Conv): - """Conv2d_Thor""" - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - pad_mode='same', - padding=0, - dilation=1, - group=1, - data_format='NCHW', - has_bias=False, - weight_init='normal', - damping=0.03, - loss_scale=1, - frequency=278, - bias_init='zeros'): - self.thor = True - ksizes = (1, kernel_size, kernel_size, 1) - self.hw = kernel_size * kernel_size - strides = (1, stride, stride, 1) - kernel_size = twice(kernel_size) - super(Conv2d_Thor, self).__init__( - in_channels, - out_channels, - kernel_size, - stride, - pad_mode, - padding, - dilation, - group, - data_format, - has_bias, - weight_init, - bias_init, - ) - self.conv2d = P.Conv2D(out_channel=self.out_channels, - kernel_size=self.kernel_size, - mode=1, - pad_mode=self.pad_mode, - pad=self.padding, - stride=self.stride, - dilation=self.dilation, - group=self.group - ) - - self.img2col = P.CusImg2Col(ksizes=ksizes, strides=strides) - self.cube_matmul = P.CusMatMulCube(transpose_a=True) - self.matrix_combine = P.CusMatrixCombine() - self.cholesky = P.CusCholeskyTrsm() - self.transpose02314 = P.CusTranspose02314() - self.matrix_A_dim = self.in_channels * self.kernel_size[0] * self.kernel_size[1] - self.matrix_G_dim = self.out_channels - self.matrix_A_device_shape, self.matrix_A_device_dim = caculate_device_shape(self.matrix_A_dim, - self.in_channels, True) - self.matrix_G_device_shape, self.matrix_G_device_dim = caculate_device_shape(self.matrix_G_dim, - self.in_channels, False) - self.matrix_A_device_temp_shape = ( - self.matrix_A_device_shape[0], self.matrix_A_device_shape[2], self.matrix_A_device_shape[1], - self.matrix_A_device_shape[3]) - self.matrix_G_device_temp_shape = ( - self.matrix_G_device_shape[0], self.matrix_G_device_shape[2], self.matrix_G_device_shape[1], - self.matrix_G_device_shape[3]) - self.matrix_A_inv = Parameter( - Tensor(np.reshape(np.identity(self.matrix_A_device_dim).astype(np.float16), self.matrix_A_device_shape)), - name='matrix_A_inv', requires_grad=False) - self.A_inv_max = Parameter(initializer(0, [1], mstype.float32), name="A_inv_max", requires_grad=False) - self.matrix_G_inv = Parameter( - Tensor(np.reshape(np.identity(self.matrix_G_device_dim).astype(np.float16), self.matrix_G_device_shape)), - name="matrix_G_inv", requires_grad=False) - - self.G_inv_max = Parameter(initializer(0, [1], mstype.float32), name="G_inv_max", requires_grad=False) - self.fake_G = Tensor( - np.reshape(np.identity(self.matrix_G_device_dim).astype(np.float16), self.matrix_G_device_shape)) - - self.shape = P.Shape() - self.reshape = P.Reshape() - self.transpose = P.Transpose() - self.cov_step = Parameter(initializer(0, [1], mstype.int32), name="cov_step", requires_grad=False) - self.mul = P.Mul() - self.cast = P.Cast() - self.damping = Tensor(damping) - self.vector_matmul = P.CusBatchMatMul() - self.diag_block_dim = 128 - self.channels_slice_flag = False - if self.in_channels % C0 != 0: - self.channels_slice_flag = True - - self.padA_flag = False - if (self.matrix_A_dim // self.diag_block_dim) * self.diag_block_dim != self.matrix_A_dim \ - and self.matrix_A_dim > self.diag_block_dim: - self.padA_flag = True - pad_dim = self.diag_block_dim - self.matrix_A_dim % self.diag_block_dim - self.padA = P.Pad(((0, pad_dim), (0, pad_dim))) - self.device_shape_pad_flag = False - if self.matrix_A_dim != self.matrix_A_device_dim: - self.device_shape_pad_flag = True - self.device_shape_pad = P.Pad(((0, 0), (0, C0 - self.in_channels), (0, 0), (0, C0 - self.in_channels))) - self.slice = P.Slice() - self.gather = P.GatherV2() - self.freq = Tensor(frequency, mstype.int32) - self.loss_scale = Tensor(1 / loss_scale, mstype.float16) - self.axis = 0 - - dampingA_dim = self.matrix_A_dim - if (self.matrix_A_dim % self.diag_block_dim) != 0 and self.matrix_A_dim > self.diag_block_dim: - dampingA_dim = (self.matrix_A_dim // self.diag_block_dim + 1) * self.diag_block_dim - dampingG_dim = self.matrix_G_dim - if (self.matrix_G_dim % self.diag_block_dim) != 0 and self.matrix_G_dim > self.diag_block_dim: - dampingG_dim = (self.matrix_G_dim // self.diag_block_dim + 1) * self.diag_block_dim - - self.dampingA = Tensor(np.identity(dampingA_dim), mstype.float32) - self.dampingG = Tensor(np.identity(dampingG_dim), mstype.float32) - self.fused_abs_max1 = P.CusFusedAbsMax1([self.matrix_A_dim, self.matrix_A_dim]) - self.fused_abs_max2 = P.CusFusedAbsMax1() - self.log = P.Log() - self.exp = P.Exp() - self.sqrt = P.Sqrt() - self.getG = P.InsertGradientOf(self.save_gradient) - - def save_gradient(self, dout): - """save_gradient""" - out = dout - dout = self.mul(dout, self.loss_scale) - dout = self.mul(dout, 32.0) - dout = self.transpose02314(dout) - dout_shape = self.shape(dout) - normalizer = dout_shape[0] - - matrix_G = self.cube_matmul(dout, dout) - normalizer = self.cast(normalizer, ms.float32) - matrix_G = self.mul(matrix_G, 1.0 / normalizer) - damping_step = self.gather(self.damping, self.cov_step, 0) - self.cov_step = self.cov_step + self.freq - damping_step = self.cast(damping_step, mstype.float32) - damping = self.mul(damping_step, 32.0 / normalizer) - damping = self.sqrt(damping) - dampingG = self.cast(self.dampingG, mstype.float32) - matrix_G = matrix_G + damping * dampingG - - matrix_G_inv = self.cholesky(matrix_G) - matrix_G_inv = self.vector_matmul(matrix_G_inv, matrix_G_inv) - matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv) - matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv_max) - self.G_inv_max = matrix_G_inv_max - matrix_G_inv = self.matrix_combine(matrix_G_inv) - matrix_G_inv = self.reshape(matrix_G_inv, self.matrix_G_device_temp_shape) - matrix_G_inv = self.transpose(matrix_G_inv, (2, 0, 1, 3)) - matrix_G = self.cast(matrix_G_inv, mstype.float16) - self.matrix_G_inv = matrix_G - return out - - def construct(self, x): - if self.thor: - matrix_A = self.img2col(x) - matrix_A_shape = self.shape(matrix_A) - normalizer = matrix_A_shape[0] - matrix_A = self.cube_matmul(matrix_A, matrix_A) - - if self.channels_slice_flag: - matrix_A = self.reshape(matrix_A, (self.hw, C0, self.hw, C0)) - matrix_A = self.slice(matrix_A, (0, 0, 0, 0), (self.hw, self.in_channels, self.hw, self.in_channels)) - matrix_A = self.reshape(matrix_A, (self.matrix_A_dim, self.matrix_A_dim)) - normalizer = self.cast(normalizer, ms.float32) - matrix_A = self.mul(matrix_A, 1.0 / normalizer) - if self.padA_flag: - matrix_A = self.padA(matrix_A) - damping_step = self.gather(self.damping, self.cov_step, self.axis) - damping_step = self.cast(damping_step, mstype.float32) - damping = self.mul(damping_step, 32.0 / normalizer) - damping = self.sqrt(damping) - damping_A = self.cast(self.dampingA, mstype.float32) - matrix_A = matrix_A + damping * damping_A - matrix_A_inv = self.cholesky(matrix_A) - matrix_A_inv = self.vector_matmul(matrix_A_inv, matrix_A_inv) - matrix_A_inv_max = self.fused_abs_max1(matrix_A_inv) - matrix_A_inv_max = self.fused_abs_max2(matrix_A_inv_max) - self.A_inv_max = matrix_A_inv_max - matrix_A_inv = self.matrix_combine(matrix_A_inv) - matrix_A_inv = self.cast(matrix_A_inv, mstype.float16) - if self.padA_flag: - matrix_A_inv = self.slice(matrix_A_inv, (0, 0), (self.matrix_A_dim, self.matrix_A_dim)) - - if self.device_shape_pad_flag: - matrix_A_inv = self.reshape(matrix_A_inv, (self.hw, self.in_channels, self.hw, self.in_channels)) - matrix_A_inv = self.device_shape_pad(matrix_A_inv) - matrix_A_inv = self.reshape(matrix_A_inv, self.matrix_A_device_temp_shape) - matrix_A_inv = self.transpose(matrix_A_inv, (2, 0, 1, 3)) - self.matrix_A_inv = matrix_A_inv - self.matrix_G_inv = self.fake_G - out = self.conv2d(x, self.weight) - out = self.getG(out) - else: - out = self.conv2d(x, self.weight) - - return out - - def extra_repr(self): - """extra_repr""" - s = 'input_channels={}, output_channels={}, kernel_size={},' \ - 'stride={}, pad_mode={}, padding={}, dilation={}, ' \ - 'group={}, data_format={}, has_bias={},' \ - 'weight_init={}, bias_init={}'.format( - self.in_channels, - self.out_channels, - self.kernel_size, - self.stride, - self.pad_mode, - self.padding, - self.dilation, - self.group, - self.data_format, - self.has_bias, - self.weight, - self.bias) - - if self.has_bias: - s += ', bias={}'.format(self.bias) - return s - - -class Dense_Thor(Cell): - """Dense_Thor""" - @cell_attr_register(attrs=['has_bias', 'activation']) - def __init__(self, - in_channels, - out_channels, - weight_init='normal', - bias_init='zeros', - damping=0.03, - loss_scale=1, - frequency=278, - has_bias=True, - activation=None): - super(Dense_Thor, self).__init__() - self.in_channels = check_int_positive(in_channels) - self.out_channels = check_int_positive(out_channels) - self.has_bias = check_bool(has_bias) - self.thor = True - if isinstance(weight_init, Tensor): - if weight_init.dim() != 2 or weight_init.shape[0] != out_channels or \ - weight_init.shape[1] != in_channels: - raise ValueError("weight_init shape error") - - self.weight = Parameter(initializer(weight_init, [out_channels, in_channels]), name="weight") - - if self.has_bias: - if isinstance(bias_init, Tensor): - if bias_init.dim() != 1 or bias_init.shape[0] != out_channels: - raise ValueError("bias_init shape error") - - self.bias = Parameter(initializer(bias_init, [out_channels]), name="bias") - - self.matmul = P.MatMul(transpose_b=True) - self.bias_add = P.BiasAdd() - - self.activation = get_activation(activation) - self.activation_flag = self.activation is not None - - self.matrix_A_inv = Parameter(Tensor(np.zeros([128, 128, 16, 16]).astype(np.float16)), name='matrix_A_inv', - requires_grad=False) - self.matrix_G_inv = Parameter(Tensor(np.zeros([63, 63, 16, 16]).astype(np.float16)), name="matrix_G_inv", - requires_grad=False) - self.fake_G = Tensor(np.zeros([63, 63, 16, 16]).astype(np.float16)) - - self.matmul = P.MatMul(transpose_b=True) - self.cube_matmul = P.CusMatMulCube(transpose_a=True) - self.matrix_combine = P.CusMatrixCombine() - self.cholesky = P.CusCholeskyTrsm() - self.shape = P.Shape() - self.reshape = P.Reshape() - self.transpose = P.Transpose() - self.cov_step = Parameter(initializer(0, [1], mstype.int32), name="cov_step", requires_grad=False) - self.mul = P.Mul() - self.cast = P.Cast() - self.damping = Tensor(damping) - self.loss_scale = Tensor(1 / loss_scale, mstype.float16) - self.vector_matmul = P.CusBatchMatMul() - self.pad = P.Pad(((0, 24), (0, 24))) - self.pad1 = P.Pad(((0, 8), (0, 8))) - self.slice = P.Slice() - self.gather = P.GatherV2() - self.assignadd = P.AssignAdd() - self.freq = Tensor(frequency, mstype.int32) - self.axis = 0 - self.A_inv_max = Parameter(initializer(0, [1], mstype.float32), name="A_inv_max", requires_grad=False) - self.G_inv_max = Parameter(initializer(0, [1], mstype.float32), name="G_inv_max", requires_grad=False) - self.fused_abs_max1 = P.CusFusedAbsMax1([1000, 1000]) - self.fused_abs_max2 = P.CusFusedAbsMax1() - self.log = P.Log() - self.exp = P.Exp() - self.dampingA = Tensor(np.identity(2048), mstype.float32) - self.dampingG = Tensor(np.identity(1024), mstype.float32) - self.add = P.TensorAdd() - self.sqrt = P.Sqrt() - self.getG = P.InsertGradientOf(self.save_gradient) - - def save_gradient(self, dout): - """save_gradient""" - out = dout - dout = self.mul(dout, self.loss_scale) - dout = self.mul(dout, 32.0) - normalizer = 32 - matrix_G = self.cube_matmul(dout, dout) - normalizer = self.cast(normalizer, ms.float32) - matrix_G = self.mul(matrix_G, 1.0 / normalizer) - matrix_G = self.pad(matrix_G) - damping_step = self.gather(self.damping, self.cov_step, 0) - damping_step = self.cast(damping_step, mstype.float32) - self.cov_step = self.cov_step + self.freq - damping = self.sqrt(damping_step) - dampingG = self.cast(self.dampingG, mstype.float32) - matrix_G = matrix_G + damping * dampingG - matrix_G_inv = self.cholesky(matrix_G) - matrix_G_inv = self.vector_matmul(matrix_G_inv, matrix_G_inv) - matrix_G_inv_max = self.fused_abs_max1(matrix_G_inv) - matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv_max) - self.G_inv_max = matrix_G_inv_max - matrix_G_inv = self.matrix_combine(matrix_G_inv) - matrix_G_inv = self.slice(matrix_G_inv, (0, 0), (1000, 1000)) - matrix_G_inv = self.pad1(matrix_G_inv) - matrix_G_inv_shape = self.shape(matrix_G_inv) - matrix_G_inv = self.reshape(matrix_G_inv, (matrix_G_inv_shape[0] / 16, 16, matrix_G_inv_shape[0] / 16, 16)) - matrix_G_inv = self.transpose(matrix_G_inv, (2, 0, 1, 3)) - matrix_G_inv = self.cast(matrix_G_inv, mstype.float16) - self.matrix_G_inv = matrix_G_inv - return out - - def construct(self, x): - """construct""" - if self.thor: - inputs = self.cube_matmul(x, x) - normalizer = 32 - normalizer = self.cast(normalizer, ms.float32) - matrix_A = self.mul(inputs, 1.0 / normalizer) - - damping_step = self.gather(self.damping, self.cov_step, self.axis) - damping_step = self.cast(damping_step, mstype.float32) - damping = self.sqrt(damping_step) - dampingA = self.cast(self.dampingA, mstype.float32) - matrix_A = matrix_A + damping * dampingA - matrix_A_inv = self.cholesky(matrix_A) - matrix_A_inv = self.vector_matmul(matrix_A_inv, matrix_A_inv) - - matrix_A_inv_max = self.fused_abs_max2(matrix_A_inv) - matrix_A_inv_max = self.fused_abs_max2(matrix_A_inv_max) - self.A_inv_max = matrix_A_inv_max - - matrix_A_inv = self.matrix_combine(matrix_A_inv) - matrix_A_inv_shape = self.shape(matrix_A_inv) - matrix_A_inv = self.reshape(matrix_A_inv, (matrix_A_inv_shape[0] / 16, 16, matrix_A_inv_shape[0] / 16, 16)) - matrix_A_inv = self.transpose(matrix_A_inv, (2, 0, 1, 3)) - matrix_A_inv = self.cast(matrix_A_inv, mstype.float16) - self.matrix_A_inv = matrix_A_inv - self.matrix_G_inv = self.fake_G - output = self.matmul(x, self.weight) - output = self.getG(output) - else: - output = self.matmul(x, self.weight) - - if self.has_bias: - output = self.bias_add(output, self.bias) - if self.activation_flag: - return self.activation(output) - return output - - def extend_repr(self): - """extend_repr""" - str_info = 'in_channels={}, out_channels={}, weight={}, has_bias={}' \ - .format(self.in_channels, self.out_channels, self.weight, self.has_bias) - if self.has_bias: - str_info = str_info + ', bias={}'.format(self.bias) - - if self.activation_flag: - str_info = str_info + ', activation={}'.format(self.activation) - - return str_info diff --git a/example/resnet50_imagenet2012_THOR/run_distribute_train.sh b/example/resnet50_imagenet2012_THOR/run_distribute_train.sh deleted file mode 100644 index e39034a9127..00000000000 --- a/example/resnet50_imagenet2012_THOR/run_distribute_train.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 3 ] -then - echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [DEVICE_NUM]" -exit 1 -fi - -if [ ! -f $1 ] -then - echo "error: DMINDSPORE_HCCL_CONFIG_PATH=$1 is not a file" -exit 1 -fi - -if [ ! -d $2 ] -then - echo "error: DATASET_PATH=$2 is not a directory" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=$3 -export RANK_SIZE=$3 -export MINDSPORE_HCCL_CONFIG_PATH=$1 - -for((i=0; i<${DEVICE_NUM}; i++)) -do - export DEVICE_ID=$i - export RANK_ID=$i - rm -rf ./train_parallel$i - mkdir ./train_parallel$i - cp *.py ./train_parallel$i - cp *.sh ./train_parallel$i - cp -r model ./train_parallel$i - cd ./train_parallel$i || exit - echo "start training for rank $RANK_ID, device $DEVICE_ID" - - env > env.log - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$2 > log 2>&1 & - cd .. -done diff --git a/example/resnet50_imagenet2012_THOR/run_infer.sh b/example/resnet50_imagenet2012_THOR/run_infer.sh deleted file mode 100755 index 14d7faf981e..00000000000 --- a/example/resnet50_imagenet2012_THOR/run_infer.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 2 ] -then - echo "Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH]" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) - - -if [ ! -d $PATH1 ] -then - echo "error: DATASET_PATH=$1 is not a directory" -exit 1 -fi - -if [ ! -f $PATH2 ] -then - echo "error: CHECKPOINT_PATH=$2 is not a file" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=1 -export DEVICE_ID=0 -export RANK_SIZE=$DEVICE_NUM -export RANK_ID=0 - -if [ -d "infer" ]; -then - rm -rf ./infer -fi -mkdir ./infer -cp *.py ./infer -cp *.sh ./infer -cd ./infer || exit -env > env.log -echo "start infering for device $DEVICE_ID" -python eval.py --do_eval=True --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & -cd .. diff --git a/example/resnet50_imagenet2012_THOR/train.py b/example/resnet50_imagenet2012_THOR/train.py deleted file mode 100644 index 309018da57b..00000000000 --- a/example/resnet50_imagenet2012_THOR/train.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""train_imagenet.""" -import argparse -import os -import random - -import numpy as np - -from mindspore import Tensor -from mindspore import context -from mindspore.communication.management import init -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor -from mindspore.train.loss_scale_manager import FixedLossScaleManager -from mindspore.train.model import ParallelMode -from model.model_thor import Model -from model.resnet import resnet50 -from model.thor import THOR - -from config import config -from crossentropy import CrossEntropy -from dataset_imagenet import create_dataset - -random.seed(1) -np.random.seed(1) - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') - -args_opt = parser.parse_args() -device_id = int(os.getenv('DEVICE_ID')) - -context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id) - - -def get_model_lr(global_step, lr_init, decay, total_epochs, steps_per_epoch): - """get_model_lr""" - lr_each_step = [] - total_steps = steps_per_epoch * total_epochs - for i in range(total_steps): - epoch = (i + 1) / steps_per_epoch - base = (1.0 - float(epoch) / total_epochs) ** decay - lr_local = lr_init * base - if epoch >= 39: - lr_local = lr_local * 0.5 - if epoch >= 40: - lr_local = lr_local * 0.5 - lr_each_step.append(lr_local) - current_step = global_step - lr_each_step = np.array(lr_each_step).astype(np.float32) - learning_rate = lr_each_step[current_step:] - return learning_rate - - -def get_model_damping(global_step, damping_init, decay_rate, total_epochs, steps_per_epoch): - """get_model_damping""" - damping_each_step = [] - total_steps = steps_per_epoch * total_epochs - for step in range(total_steps): - epoch = (step + 1) / steps_per_epoch - damping_here = damping_init * (decay_rate ** (epoch / 10)) - damping_each_step.append(damping_here) - - current_step = global_step - damping_each_step = np.array(damping_each_step).astype(np.float32) - damping_now = damping_each_step[current_step:] - return damping_now - - -if __name__ == '__main__': - if not args_opt.do_eval and args_opt.run_distribute: - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True, parameter_broadcast=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([107], "hccl_world_groupsum1") - auto_parallel_context().set_all_reduce_fusion_split_indices([27], "hccl_world_groupsum2") - auto_parallel_context().set_all_reduce_fusion_split_indices([27], "hccl_world_groupsum3") - auto_parallel_context().set_all_reduce_fusion_split_indices([27], "hccl_world_groupsum4") - auto_parallel_context().set_all_reduce_fusion_split_indices([27], "hccl_world_groupsum5") - - init() - - epoch_size = config.epoch_size - damping = get_model_damping(0, 0.03, 0.87, 50, 5004) - net = resnet50(class_num=config.class_num, damping=damping, loss_scale=config.loss_scale, - frequency=config.frequency) - - if not config.label_smooth: - config.label_smooth_factor = 0.0 - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - if args_opt.do_train: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, - repeat_num=epoch_size, batch_size=config.batch_size) - step_size = dataset.get_dataset_size() - - loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) - lr = Tensor(get_model_lr(0, 0.045, 6, 70, 5004)) - opt = THOR(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, - filter(lambda x: 'matrix_A' in x.name, net.get_parameters()), - filter(lambda x: 'matrix_G' in x.name, net.get_parameters()), - filter(lambda x: 'A_inv_max' in x.name, net.get_parameters()), - filter(lambda x: 'G_inv_max' in x.name, net.get_parameters()), - config.weight_decay, config.loss_scale) - - model = Model(net, loss_fn=loss, optimizer=opt, amp_level='O2', loss_scale_manager=loss_scale, - keep_batchnorm_fp32=False, metrics={'acc'}, frequency=config.frequency) - - time_cb = TimeMonitor(data_size=step_size) - loss_cb = LossMonitor() - cb = [time_cb, loss_cb] - if config.save_checkpoint: - config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_steps, - keep_checkpoint_max=config.keep_checkpoint_max) - ckpt_cb = ModelCheckpoint(prefix="resnet", directory=config.save_checkpoint_path, config=config_ck) - cb += [ckpt_cb] - - model.train(epoch_size, dataset, callbacks=cb) diff --git a/model_zoo/resnet/README.md b/model_zoo/resnet/README.md new file mode 100644 index 00000000000..ad934536027 --- /dev/null +++ b/model_zoo/resnet/README.md @@ -0,0 +1,251 @@ +# ResNet Example + +## Description + +These are examples of training ResNet-50/ResNet-101 with CIFAR-10/ImageNet2012 dataset in MindSpore. +(Training ResNet-101 with dataset CIFAR-10 is unsupported now.) + +## Requirements + +- Install [MindSpore](https://www.mindspore.cn/install/en). + +- Download the dataset CIFAR-10 or ImageNet2012 + +CIFAR-10 + +> Unzip the CIFAR-10 dataset to any path you want and the folder structure should include train and eval dataset as follows: +> ``` +> . +> └─dataset +> ├─ cifar-10-batches-bin # train dataset +> └─ cifar-10-verify-bin # evaluate dataset +> ``` + +ImageNet2012 + +> Unzip the ImageNet2012 dataset to any path you want and the folder should include train and eval dataset as follows: +> +> ``` +> . +> └─dataset +> ├─ilsvrc # train dataset +> └─validation_preprocess # evaluate dataset +> ``` + + + +## Structure + +```shell +. +└──resnet + ├── README.md + ├── script + ├── run_distribute_train.sh # launch distributed training(8 pcs) + ├── run_eval.sh # launch evaluation + └── run_standalone_train.sh # launch standalone training(1 pcs) + ├── src + ├── config.py # parameter configuration + ├── dataset.py # data preprocessing + ├── crossentropy.py # loss definition for ImageNet2012 dataset + ├── lr_generator.py # generate learning rate for each step + └── resnet.py # resnet backbone, including resnet50 and resnet101 + ├── eval.py # eval net + └── train.py # train net +``` + + +## Parameter configuration + +Parameters for both training and evaluation can be set in config.py. + +- config for ResNet-50, CIFAR-10 dataset + +``` +"class_num": 10, # dataset class num +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum +"weight_decay": 1e-4, # weight decay +"epoch_size": 90, # only valid for taining, which is always 1 for inference +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_steps": 195, # the step interval between two checkpoints. By default, the last checkpoint will be saved after the last step +"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint +"warmup_epochs": 5, # number of warmup epoch +"lr_decay_mode": "poly" # decay mode can be selected in steps, ploy and default +"lr_init": 0.01, # initial learning rate +"lr_end": 0.00001, # final learning rate +"lr_max": 0.1, # maximum learning rate +``` + +- config for ResNet-50, ImageNet2012 dataset + +``` +"class_num": 1001, # dataset class number +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum optimizer +"weight_decay": 1e-4, # weight decay +"epoch_size": 90, # only valid for taining, which is always 1 for inference +"pretrained_epoch_size": 1, # epoch size that model has been trained before load pretrained checkpoint +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_epochs": 1, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch +"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path +"warmup_epochs": 0, # number of warmup epoch +"lr_decay_mode": "cosine", # decay mode for generating learning rate +"label_smooth": True, # label smooth +"label_smooth_factor": 0.1, # label smooth factor +"lr_init": 0, # initial learning rate +"lr_max": 0.1, # maximum learning rate +``` + +- config for ResNet-101, ImageNet2012 dataset + +``` +"class_num": 1001, # dataset class number +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum optimizer +"weight_decay": 1e-4, # weight decay +"epoch_size": 120, # epoch sizes for training +"pretrain_epoch_size": 0, # epoch size of pretrain checkpoint +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_epochs": 1, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch +"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path +"warmup_epochs": 0, # number of warmup epoch +"lr_decay_mode": "cosine" # decay mode for generating learning rate +"label_smooth": 1, # label_smooth +"label_smooth_factor": 0.1, # label_smooth_factor +"lr": 0.1 # base learning rate +``` + + + +## Running the example + +### Train + +#### Usage + +``` +# distributed training +Usage: sh run_distribute_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] + [PRETRAINED_CKPT_PATH](optional) + +# standalone training +Usage: sh run_standalone_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] + [PRETRAINED_CKPT_PATH](optional) +``` + + +#### Launch + +``` +# distribute training example +sh run_distribute_train.sh resnet50 cifar10 rank_table.json ~/cifar-10-batches-bin + +# standalone training example +sh run_standalone_train.sh resnet50 cifar10 ~/cifar-10-batches-bin +``` + +> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). + +#### Result + +Training result will be stored in the example path, whose folder name begins with "train" or "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. + +- training ResNet-50 with CIFAR-10 dataset + +``` +# distribute training result(8 pcs) +epoch: 1 step: 195, loss is 1.9601055 +epoch: 2 step: 195, loss is 1.8555021 +epoch: 3 step: 195, loss is 1.6707983 +epoch: 4 step: 195, loss is 1.8162166 +epoch: 5 step: 195, loss is 1.393667 +... +``` + +- training ResNet-50 with ImageNet2012 dataset + +``` +# distribute training result(8 pcs) +epoch: 1 step: 5004, loss is 4.8995576 +epoch: 2 step: 5004, loss is 3.9235563 +epoch: 3 step: 5004, loss is 3.833077 +epoch: 4 step: 5004, loss is 3.2795618 +epoch: 5 step: 5004, loss is 3.1978393 +... +``` + +- training ResNet-101 with ImageNet2012 dataset + +``` +# distribute training result(8p) +epoch: 1 step: 5004, loss is 4.805483 +epoch: 2 step: 5004, loss is 3.2121816 +epoch: 3 step: 5004, loss is 3.429647 +epoch: 4 step: 5004, loss is 3.3667371 +epoch: 5 step: 5004, loss is 3.1718972 +... +epoch: 67 step: 5004, loss is 2.2768745 +epoch: 68 step: 5004, loss is 1.7223864 +epoch: 69 step: 5004, loss is 2.0665488 +epoch: 70 step: 5004, loss is 1.8717369 +... +``` + +### Evaluation + +#### Usage + +``` +# evaluation +Usage: sh run_eval.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [CHECKPOINT_PATH] +``` + +#### Launch + +``` +# evaluation example +sh run_eval.sh resnet50 cifar10 ~/cifar10-10-verify-bin ~/resnet50_cifar10/train_parallel0/resnet-90_195.ckpt +``` + +> checkpoint can be produced in training process. + +#### Result + +Evaluation result will be stored in the example path, whose folder name is "eval". Under this, you can find result like the followings in log. + +- evaluating ResNet-50 with CIFAR-10 dataset + +``` +result: {'acc': 0.91446314102564111} ckpt=~/resnet50_cifar10/train_parallel0/resnet-90_195.ckpt +``` + +- evaluating ResNet-50 with ImageNet2012 dataset + +``` +result: {'acc': 0.7671054737516005} ckpt=train_parallel0/resnet-90_5004.ckpt +``` + +- evaluating ResNet-101 with ImageNet2012 dataset + +``` +result: {'top_5_accuracy': 0.9429417413572343, 'top_1_accuracy': 0.7853513124199744} ckpt=train_parallel0/resnet-120_5004.ckpt +``` + +### Running on GPU +``` +# distributed training example +mpirun -n 8 python train.py ---net=resnet50 --dataset=cifar10 -dataset_path=~/cifar-10-batches-bin --device_target="GPU" --run_distribute=True + +# standalone training example +python train.py --net=resnet50 --dataset=cifar10 --dataset_path=~/cifar-10-batches-bin --device_target="GPU" + +# infer example +python eval.py --net=resnet50 --dataset=cifar10 --dataset_path=~/cifar10-10-verify-bin --device_target="GPU" --checkpoint_path=resnet-90_195.ckpt +``` diff --git a/model_zoo/resnet/eval.py b/model_zoo/resnet/eval.py new file mode 100755 index 00000000000..426b8c9f3de --- /dev/null +++ b/model_zoo/resnet/eval.py @@ -0,0 +1,90 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""train resnet.""" +import os +import random +import argparse +import numpy as np +from mindspore import context +from mindspore import dataset as de +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.train.model import Model +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from src.crossentropy import CrossEntropy + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--net', type=str, default=None, help='Resnet Model, either resnet50 or resnet101') +parser.add_argument('--dataset', type=str, default=None, help='Dataset, either cifar10 or imagenet2012') + +parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') +args_opt = parser.parse_args() + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +if args_opt.net == "resnet50": + from src.resnet import resnet50 as resnet + + if args_opt.dataset == "cifar10": + from src.config import config1 as config + from src.dataset import create_dataset1 as create_dataset + else: + from src.config import config2 as config + from src.dataset import create_dataset2 as create_dataset +else: + from src.resnet import resnet101 as resnet + from src.config import config3 as config + from src.dataset import create_dataset3 as create_dataset + +if __name__ == '__main__': + target = args_opt.device_target + + # init context + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False, device_id=device_id) + + # create dataset + if args_opt.net == "resnet50": + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size, + target=target) + else: + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size) + step_size = dataset.get_dataset_size() + + # define net + net = resnet(class_num=config.class_num) + + # load checkpoint + param_dict = load_checkpoint(args_opt.checkpoint_path) + load_param_into_net(net, param_dict) + net.set_train(False) + + # define loss, model + if args_opt.dataset == "imagenet2012": + if not config.use_label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) + else: + loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + + # define model + model = Model(net, loss_fn=loss, metrics={'top_1_accuracy', 'top_5_accuracy'}) + + # eval model + res = model.eval(dataset) + print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/model_zoo/resnet101/scripts/run_distribute_train.sh b/model_zoo/resnet/scripts/run_distribute_train.sh similarity index 58% rename from model_zoo/resnet101/scripts/run_distribute_train.sh rename to model_zoo/resnet/scripts/run_distribute_train.sh index 65790b88c1f..efcb620cd86 100755 --- a/model_zoo/resnet101/scripts/run_distribute_train.sh +++ b/model_zoo/resnet/scripts/run_distribute_train.sh @@ -14,12 +14,31 @@ # limitations under the License. # ============================================================================ -if [ $# != 2 ] && [ $# != 3 ] +if [ $# != 4 ] && [ $# != 5 ] then - echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_PATH](optional)" + echo "Usage: sh run_distribute_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" exit 1 fi +if [ $1 != "resnet50" ] && [ $1 != "resnet101" ] +then + echo "error: the selected net is neither resnet50 nor resnet101" +exit 1 +fi + +if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ] +then + echo "error: the selected dataset is neither cifar10 nor imagenet2012" +exit 1 +fi + +if [ $1 == "resnet101" ] && [ $2 == "cifar10" ] +then + echo "error: training resnet101 with cifar10 dataset is unsupported now!" +exit 1 +fi + + get_real_path(){ if [ "${1:0:1}" == "/" ]; then echo "$1" @@ -27,14 +46,13 @@ get_real_path(){ echo "$(realpath -m $PWD/$1)" fi } -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) -echo $PATH1 -echo $PATH2 -if [ $# == 3 ] + +PATH1=$(get_real_path $3) +PATH2=$(get_real_path $4) + +if [ $# == 5 ] then - PATH3=$(get_real_path $3) - echo $PATH3 + PATH3=$(get_real_path $5) fi if [ ! -f $PATH1 ] @@ -49,9 +67,9 @@ then exit 1 fi -if [ $# == 3 ] && [ ! -f $PATH3 ] +if [ $# == 5 ] && [ ! -f $PATH3 ] then - echo "error: PRETRAINED_PATH=$PATH3 is not a file" + echo "error: PRETRAINED_CKPT_PATH=$PATH3 is not a file" exit 1 fi @@ -73,14 +91,14 @@ do cd ./train_parallel$i || exit echo "start training for rank $RANK_ID, device $DEVICE_ID" env > env.log - if [ $# == 2 ] + if [ $# == 4 ] then - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 &> log & + python train.py --net=$1 --dataset=$2 --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 &> log & fi - if [ $# == 3 ] + if [ $# == 5 ] then - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 --pre_trained=$PATH3 &> log & + python train.py --net=$1 --dataset=$2 --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 --pre_trained=$PATH3 &> log & fi cd .. diff --git a/model_zoo/resnet101/scripts/run_eval.sh b/model_zoo/resnet/scripts/run_eval.sh similarity index 62% rename from model_zoo/resnet101/scripts/run_eval.sh rename to model_zoo/resnet/scripts/run_eval.sh index 88f5d364ce2..496b3c1e2b4 100755 --- a/model_zoo/resnet101/scripts/run_eval.sh +++ b/model_zoo/resnet/scripts/run_eval.sh @@ -14,12 +14,31 @@ # limitations under the License. # ============================================================================ -if [ $# != 2 ] +if [ $# != 4 ] then - echo "Usage: sh run_eval.sh [DATASET_PATH] [CHECKPOINT_PATH]" + echo "Usage: sh run_eval.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [CHECKPOINT_PATH]" exit 1 fi +if [ $1 != "resnet50" ] && [ $1 != "resnet101" ] +then + echo "error: the selected net is neither resnet50 nor resnet101" +exit 1 +fi + +if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ] +then + echo "error: the selected dataset is neither cifar10 nor imagenet2012" +exit 1 +fi + +if [ $1 == "resnet101" ] && [ $2 == "cifar10" ] +then + echo "error: evaluating resnet101 with cifar10 dataset is unsupported now!" +exit 1 +fi + + get_real_path(){ if [ "${1:0:1}" == "/" ]; then echo "$1" @@ -27,10 +46,10 @@ get_real_path(){ echo "$(realpath -m $PWD/$1)" fi } -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) -echo $PATH1 -echo $PATH2 + +PATH1=$(get_real_path $3) +PATH2=$(get_real_path $4) + if [ ! -d $PATH1 ] then @@ -60,6 +79,6 @@ cp *.sh ./eval cp -r ../src ./eval cd ./eval || exit env > env.log -echo "start infering for device $DEVICE_ID" -python eval.py --do_eval=True --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & +echo "start evaluation for device $DEVICE_ID" +python eval.py --net=$1 --dataset=$2 --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & cd .. diff --git a/model_zoo/resnet101/scripts/run_standalone_train.sh b/model_zoo/resnet/scripts/run_standalone_train.sh similarity index 56% rename from model_zoo/resnet101/scripts/run_standalone_train.sh rename to model_zoo/resnet/scripts/run_standalone_train.sh index 7214d114d55..2272dbd88b5 100755 --- a/model_zoo/resnet101/scripts/run_standalone_train.sh +++ b/model_zoo/resnet/scripts/run_standalone_train.sh @@ -14,12 +14,31 @@ # limitations under the License. # ============================================================================ -if [ $# != 1 ] && [ $# != 2 ] +if [ $# != 3 ] && [ $# != 4 ] then - echo "Usage: sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_PATH](optional)" + echo "Usage: sh run_standalone_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" exit 1 fi +if [ $1 != "resnet50" ] && [ $1 != "resnet101" ] +then + echo "error: the selected net is neither resnet50 nor resnet101" +exit 1 +fi + +if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ] +then + echo "error: the selected dataset is neither cifar10 nor imagenet2012" +exit 1 +fi + +if [ $1 == "resnet101" ] && [ $2 == "cifar10" ] +then + echo "error: training resnet101 with cifar10 dataset is unsupported now!" +exit 1 +fi + + get_real_path(){ if [ "${1:0:1}" == "/" ]; then echo "$1" @@ -27,12 +46,12 @@ get_real_path(){ echo "$(realpath -m $PWD/$1)" fi } -PATH1=$(get_real_path $1) -echo $PATH1 -if [ $# == 2 ] + +PATH1=$(get_real_path $3) + +if [ $# == 4 ] then - PATH2=$(get_real_path $2) - echo $PATH2 + PATH2=$(get_real_path $4) fi if [ ! -d $PATH1 ] @@ -41,9 +60,9 @@ then exit 1 fi -if [ $# == 2 ] && [ ! -f $PATH2 ] +if [ $# == 4 ] && [ ! -f $PATH2 ] then - echo "error: PRETRAINED_PATH=$PATH2 is not a file" + echo "error: PRETRAINED_CKPT_PATH=$PATH2 is not a file" exit 1 fi @@ -64,13 +83,13 @@ cp -r ../src ./train cd ./train || exit echo "start training for device $DEVICE_ID" env > env.log -if [ $# == 1 ] +if [ $# == 3 ] then - python train.py --do_train=True --dataset_path=$PATH1 &> log & + python train.py --net=$1 --dataset=$2 --dataset_path=$PATH1 &> log & fi -if [ $# == 2 ] +if [ $# == 4 ] then - python train.py --do_train=True --dataset_path=$PATH1 --pre_trained=$PATH2 &> log & + python train.py --net=$1 --dataset=$2 --dataset_path=$PATH1 --pre_trained=$PATH2 &> log & fi cd .. diff --git a/example/resnet50_imagenet2012/config.py b/model_zoo/resnet/src/config.py similarity index 54% rename from example/resnet50_imagenet2012/config.py rename to model_zoo/resnet/src/config.py index cf5093d245b..7b1759fde07 100755 --- a/example/resnet50_imagenet2012/config.py +++ b/model_zoo/resnet/src/config.py @@ -17,17 +17,34 @@ network config setting, will be used in train.py and eval.py """ from easydict import EasyDict as ed -config = ed({ +# config for resent50, cifar10 +config1 = ed({ + "class_num": 10, + "batch_size": 32, + "loss_scale": 1024, + "momentum": 0.9, + "weight_decay": 1e-4, + "epoch_size": 90, + "save_checkpoint": True, + "save_checkpoint_epochs": 5, + "keep_checkpoint_max": 10, + "save_checkpoint_path": "./", + "warmup_epochs": 5, + "lr_decay_mode": "poly", + "lr_init": 0.01, + "lr_end": 0.00001, + "lr_max": 0.1 +}) + +# config for resnet50, imagenet2012 +config2 = ed({ "class_num": 1001, "batch_size": 32, "loss_scale": 1024, "momentum": 0.9, "weight_decay": 1e-4, "epoch_size": 90, - "pretrained_epoch_size": 1, - "buffer_size": 1000, - "image_height": 224, - "image_width": 224, + "pretrain_epoch_size": 1, "save_checkpoint": True, "save_checkpoint_epochs": 5, "keep_checkpoint_max": 10, @@ -40,3 +57,23 @@ config = ed({ "lr_max": 0.1 }) + +# config for resent101, imagenet2012 +config3 = ed({ + "class_num": 1001, + "batch_size": 32, + "loss_scale": 1024, + "momentum": 0.9, + "weight_decay": 1e-4, + "epoch_size": 120, + "pretrain_epoch_size": 0, + "save_checkpoint": True, + "save_checkpoint_epochs": 5, + "keep_checkpoint_max": 10, + "save_checkpoint_path": "./", + "warmup_epochs": 0, + "lr_decay_mode": "cosine", + "use_label_smooth": True, + "label_smooth_factor": 0.1, + "lr": 0.1 +}) diff --git a/model_zoo/resnet101/src/crossentropy.py b/model_zoo/resnet/src/crossentropy.py similarity index 98% rename from model_zoo/resnet101/src/crossentropy.py rename to model_zoo/resnet/src/crossentropy.py index 1145a41804b..5118cb51612 100755 --- a/model_zoo/resnet101/src/crossentropy.py +++ b/model_zoo/resnet/src/crossentropy.py @@ -20,15 +20,18 @@ from mindspore import Tensor from mindspore.common import dtype as mstype import mindspore.nn as nn + class CrossEntropy(_Loss): """the redefined loss function with SoftmaxCrossEntropyWithLogits""" + def __init__(self, smooth_factor=0., num_classes=1001): super(CrossEntropy, self).__init__() self.onehot = P.OneHot() self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) - self.off_value = Tensor(1.0 * smooth_factor / (num_classes -1), mstype.float32) + self.off_value = Tensor(1.0 * smooth_factor / (num_classes - 1), mstype.float32) self.ce = nn.SoftmaxCrossEntropyWithLogits() self.mean = P.ReduceMean(False) + def construct(self, logit, label): one_hot_label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) loss = self.ce(logit, one_hot_label) diff --git a/model_zoo/resnet/src/dataset.py b/model_zoo/resnet/src/dataset.py new file mode 100755 index 00000000000..ac0adc4bc97 --- /dev/null +++ b/model_zoo/resnet/src/dataset.py @@ -0,0 +1,205 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +create train or eval dataset. +""" +import os +import mindspore.common.dtype as mstype +import mindspore.dataset.engine as de +import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.transforms.c_transforms as C2 +from mindspore.communication.management import init, get_rank, get_group_size + + +def create_dataset1(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): + """ + create a train or evaluate cifar10 dataset for resnet50 + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + target(str): the device target. Default: Ascend + + Returns: + dataset + """ + if target == "Ascend": + device_num = int(os.getenv("DEVICE_NUM")) + rank_id = int(os.getenv("RANK_ID")) + else: + init("nccl") + rank_id = get_rank() + device_num = get_group_size() + + if device_num == 1: + ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + + # define map operations + trans = [] + if do_train: + trans += [ + C.RandomCrop((32, 32), (4, 4, 4, 4)), + C.RandomHorizontalFlip(prob=0.5) + ] + + trans += [ + C.Resize((224, 224)), + C.Rescale(1.0 / 255.0, 0.0), + C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), + C.HWC2CHW() + ] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) + ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds + + +def create_dataset2(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): + """ + create a train or eval imagenet2012 dataset for resnet50 + + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + target(str): the device target. Default: Ascend + + Returns: + dataset + """ + if target == "Ascend": + device_num = int(os.getenv("DEVICE_NUM")) + rank_id = int(os.getenv("RANK_ID")) + else: + init("nccl") + rank_id = get_rank() + device_num = get_group_size() + + if device_num == 1: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + + image_size = 224 + mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] + std = [0.229 * 255, 0.224 * 255, 0.225 * 255] + + # define map operations + if do_train: + trans = [ + C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), + C.RandomHorizontalFlip(prob=0.5), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + else: + trans = [ + C.Decode(), + C.Resize((256, 256)), + C.CenterCrop(image_size), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) + ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds + + +def create_dataset3(dataset_path, do_train, repeat_num=1, batch_size=32): + """ + create a train or eval imagenet2012 dataset for resnet101 + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + + Returns: + dataset + """ + device_num = int(os.getenv("RANK_SIZE")) + rank_id = int(os.getenv("RANK_ID")) + + if device_num == 1: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + resize_height = 224 + rescale = 1.0 / 255.0 + shift = 0.0 + + # define map operations + decode_op = C.Decode() + + random_resize_crop_op = C.RandomResizedCrop(resize_height, (0.08, 1.0), (0.75, 1.33), max_attempts=100) + horizontal_flip_op = C.RandomHorizontalFlip(rank_id / (rank_id + 1)) + resize_op_256 = C.Resize((256, 256)) + center_crop = C.CenterCrop(224) + rescale_op = C.Rescale(rescale, shift) + normalize_op = C.Normalize((0.475, 0.451, 0.392), (0.275, 0.267, 0.278)) + changeswap_op = C.HWC2CHW() + + if do_train: + trans = [decode_op, + random_resize_crop_op, + horizontal_flip_op, + rescale_op, + normalize_op, + changeswap_op] + + else: + trans = [decode_op, + resize_op_256, + center_crop, + rescale_op, + normalize_op, + changeswap_op] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=8) + ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds diff --git a/example/resnet50_imagenet2012/lr_generator.py b/model_zoo/resnet/src/lr_generator.py similarity index 64% rename from example/resnet50_imagenet2012/lr_generator.py rename to model_zoo/resnet/src/lr_generator.py index 4a57be2f01e..2af89717150 100755 --- a/example/resnet50_imagenet2012/lr_generator.py +++ b/model_zoo/resnet/src/lr_generator.py @@ -28,7 +28,7 @@ def get_lr(lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch warmup_epochs(int): number of warmup epochs total_epochs(int): total epoch of training steps_per_epoch(int): steps of one epoch - lr_decay_mode(string): learning rate decay mode, including steps, poly, cosine or default + lr_decay_mode(string): learning rate decay mode, including steps, poly or default Returns: np.array, learning rate array @@ -62,18 +62,6 @@ def get_lr(lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch if lr < 0.0: lr = 0.0 lr_each_step.append(lr) - elif lr_decay_mode == 'cosine': - decay_steps = total_steps - warmup_steps - for i in range(total_steps): - if i < warmup_steps: - lr_inc = (float(lr_max) - float(lr_init)) / float(warmup_steps) - lr = float(lr_init) + lr_inc * (i + 1) - else: - linear_decay = (total_steps - i) / decay_steps - cosine_decay = 0.5 * (1 + math.cos(math.pi * 2 * 0.47 * i / decay_steps)) - decayed = linear_decay * cosine_decay + 0.00001 - lr = lr_max * decayed - lr_each_step.append(lr) else: for i in range(total_steps): if i < warmup_steps: @@ -82,6 +70,47 @@ def get_lr(lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch lr = lr_max - (lr_max - lr_end) * (i - warmup_steps) / (total_steps - warmup_steps) lr_each_step.append(lr) - learning_rate = np.array(lr_each_step).astype(np.float32) + lr_each_step = np.array(lr_each_step).astype(np.float32) + return lr_each_step + + +def linear_warmup_lr(current_step, warmup_steps, base_lr, init_lr): + lr_inc = (float(base_lr) - float(init_lr)) / float(warmup_steps) + lr = float(init_lr) + lr_inc * current_step + return lr + + +def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch=120, global_step=0): + """ + generate learning rate array with cosine + + Args: + lr(float): base learning rate + steps_per_epoch(int): steps size of one epoch + warmup_epochs(int): number of warmup epochs + max_epoch(int): total epochs of training + global_step(int): the current start index of lr array + Returns: + np.array, learning rate array + """ + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + decay_steps = total_steps - warmup_steps + + lr_each_step = [] + for i in range(total_steps): + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + linear_decay = (total_steps - i) / decay_steps + cosine_decay = 0.5 * (1 + math.cos(math.pi * 2 * 0.47 * i / decay_steps)) + decayed = linear_decay * cosine_decay + 0.00001 + lr = base_lr * decayed + lr_each_step.append(lr) + + lr_each_step = np.array(lr_each_step).astype(np.float32) + learning_rate = lr_each_step[global_step:] return learning_rate diff --git a/model_zoo/resnet101/src/resnet101.py b/model_zoo/resnet/src/resnet.py similarity index 94% rename from model_zoo/resnet101/src/resnet101.py rename to model_zoo/resnet/src/resnet.py index 33f10fd6cb5..0e21222d21d 100755 --- a/model_zoo/resnet101/src/resnet101.py +++ b/model_zoo/resnet/src/resnet.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -"""ResNet101.""" +"""ResNet.""" import numpy as np import mindspore.nn as nn from mindspore.ops import operations as P @@ -240,6 +240,28 @@ class ResNet(nn.Cell): return out + +def resnet50(class_num=10): + """ + Get ResNet50 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet50 neural network. + + Examples: + >>> net = resnet50(10) + """ + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num) + + def resnet101(class_num=1001): """ Get ResNet101 neural network. diff --git a/model_zoo/resnet/train.py b/model_zoo/resnet/train.py new file mode 100755 index 00000000000..89ce62d7339 --- /dev/null +++ b/model_zoo/resnet/train.py @@ -0,0 +1,162 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""train resnet.""" +import os +import random +import argparse +import numpy as np +from mindspore import context +from mindspore import Tensor +from mindspore import dataset as de +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore.nn.optim.momentum import Momentum +from mindspore.train.model import Model, ParallelMode +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.train.loss_scale_manager import FixedLossScaleManager +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.communication.management import init, get_rank, get_group_size +import mindspore.nn as nn +import mindspore.common.initializer as weight_init +from src.lr_generator import get_lr, warmup_cosine_annealing_lr +from src.crossentropy import CrossEntropy + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--net', type=str, default=None, help='Resnet Model, either resnet50 or resnet101') +parser.add_argument('--dataset', type=str, default=None, help='Dataset, either cifar10 or imagenet2012') +parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') +parser.add_argument('--device_num', type=int, default=1, help='Device num.') + +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') +parser.add_argument('--pre_trained', type=str, default=None, help='Pretrained checkpoint path') +args_opt = parser.parse_args() + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +if args_opt.net == "resnet50": + from src.resnet import resnet50 as resnet + + if args_opt.dataset == "cifar10": + from src.config import config1 as config + from src.dataset import create_dataset1 as create_dataset + else: + from src.config import config2 as config + from src.dataset import create_dataset2 as create_dataset +else: + from src.resnet import resnet101 as resnet + from src.config import config3 as config + from src.dataset import create_dataset3 as create_dataset + +if __name__ == '__main__': + target = args_opt.device_target + ckpt_save_dir = config.save_checkpoint_path + + # init context + context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) + if args_opt.run_distribute: + if target == "Ascend": + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(device_id=device_id, enable_auto_mixed_precision=True) + context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True) + if args_opt.net == "resnet50": + auto_parallel_context().set_all_reduce_fusion_split_indices([107, 160]) + else: + auto_parallel_context().set_all_reduce_fusion_split_indices([180, 313]) + init() + # GPU target + else: + init("nccl") + context.set_auto_parallel_context(device_num=get_group_size(), parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True) + ckpt_save_dir = config.save_checkpoint_path + "ckpt_" + str(get_rank()) + "/" + + # create dataset + if args_opt.net == "resnet50": + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, repeat_num=config.epoch_size, + batch_size=config.batch_size, target=target) + else: + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, repeat_num=config.epoch_size, + batch_size=config.batch_size) + step_size = dataset.get_dataset_size() + + # define net + net = resnet(class_num=config.class_num) + + # init weight + if args_opt.pre_trained: + param_dict = load_checkpoint(args_opt.pre_trained) + load_param_into_net(net, param_dict) + else: + for _, cell in net.cells_and_names(): + if isinstance(cell, nn.Conv2d): + cell.weight.default_input = weight_init.initializer(weight_init.XavierUniform(), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if isinstance(cell, nn.Dense): + cell.weight.default_input = weight_init.initializer(weight_init.TruncatedNormal(), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + + # init lr + if args_opt.net == "resnet50": + if args_opt.dataset == "cifar10": + lr = get_lr(lr_init=config.lr_init, lr_end=config.lr_end, lr_max=config.lr_max, + warmup_epochs=config.warmup_epochs, total_epochs=config.epoch_size, steps_per_epoch=step_size, + lr_decay_mode='poly') + else: + lr = get_lr(lr_init=config.lr_init, lr_end=0.0, lr_max=config.lr_max, warmup_epochs=config.warmup_epochs, + total_epochs=config.epoch_size, steps_per_epoch=step_size, lr_decay_mode='cosine') + else: + lr = warmup_cosine_annealing_lr(config.lr, step_size, config.warmup_epochs, 120, + config.pretrain_epoch_size * step_size) + lr = Tensor(lr) + + # define opt + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, + config.weight_decay, config.loss_scale) + + # define loss, model + if target == "Ascend": + if args_opt.dataset == "imagenet2012": + if not config.use_label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) + else: + loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) + model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}, + amp_level="O2", keep_batchnorm_fp32=False) + else: + # GPU target + loss = SoftmaxCrossEntropyWithLogits(sparse=True, is_grad=False, reduction='mean') + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum) + model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'}) + + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossMonitor() + cb = [time_cb, loss_cb] + if config.save_checkpoint: + config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs * step_size, + keep_checkpoint_max=config.keep_checkpoint_max) + ckpt_cb = ModelCheckpoint(prefix="resnet", directory=ckpt_save_dir, config=config_ck) + cb += [ckpt_cb] + + # train model + model.train(config.epoch_size, dataset, callbacks=cb) diff --git a/model_zoo/resnet101/README.md b/model_zoo/resnet101/README.md deleted file mode 100644 index 86744be3723..00000000000 --- a/model_zoo/resnet101/README.md +++ /dev/null @@ -1,147 +0,0 @@ -# ResNet101 Example - -## Description - -This is an example of training ResNet101 with ImageNet dataset in MindSpore. - -## Requirements - -- Install [MindSpore](https://www.mindspore.cn/install/en). - -- Download the dataset ImageNet2012. - -> Unzip the ImageNet2012 dataset to any path you want, the folder should include train and eval dataset as follows: - -``` -. -└─dataset - ├─ilsvrc - │ - └─validation_preprocess -``` - -## Structure - -```shell -. -└─resnet101 - ├─README.md - ├─scripts - ├─run_standalone_train.sh # launch standalone training(1p) - ├─run_distribute_train.sh # launch distributed training(8p) - └─run_eval.sh # launch evaluating - ├─src - ├─config.py # parameter configuration - ├─crossentropy.py # CrossEntropy loss function - ├─dataset.py # data preprocessin - ├─lr_generator.py # generate learning rate - ├─resnet101.py # resnet101 backbone - ├─eval.py # eval net - └─train.py # train net -``` - -## Parameter configuration - -Parameters for both training and evaluating can be set in config.py. - -``` -"class_num": 1001, # dataset class number -"batch_size": 32, # batch size of input tensor -"loss_scale": 1024, # loss scale -"momentum": 0.9, # momentum optimizer -"weight_decay": 1e-4, # weight decay -"epoch_size": 120, # epoch sizes for training -"pretrain_epoch_size": 0, # epoch size of pretrain checkpoint -"buffer_size": 1000, # number of queue size in data preprocessing -"image_height": 224, # image height -"image_width": 224, # image width -"save_checkpoint": True, # whether save checkpoint or not -"save_checkpoint_epochs": 1, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch -"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint -"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path -"warmup_epochs": 0, # number of warmup epoch -"lr_decay_mode": "cosine" # decay mode for generating learning rate -"label_smooth": 1, # label_smooth -"label_smooth_factor": 0.1, # label_smooth_factor -"lr": 0.1 # base learning rate -``` - -## Running the example - -### Train - -#### Usage - -``` -# distributed training -sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_PATH](optional) - -# standalone training -sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_PATH](optional) -``` - -#### Launch - -```bash -# distributed training example(8p) -sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc - -If you want to load pretrained ckpt file, -sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc ./ckpt/pretrained.ckpt - -# standalone training example（1p） -sh run_standalone_train.sh dataset/ilsvrc - -If you want to load pretrained ckpt file, -sh run_standalone_train.sh dataset/ilsvrc ./ckpt/pretrained.ckpt -``` - -> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). - -#### Result - -Training result will be stored in the scripts path, whose folder name begins with "train" or "train_parallel". You can find checkpoint file together with result like the followings in log. - - -``` -# distribute training result(8p) -epoch: 1 step: 5004, loss is 4.805483 -epoch: 2 step: 5004, loss is 3.2121816 -epoch: 3 step: 5004, loss is 3.429647 -epoch: 4 step: 5004, loss is 3.3667371 -epoch: 5 step: 5004, loss is 3.1718972 -... -epoch: 67 step: 5004, loss is 2.2768745 -epoch: 68 step: 5004, loss is 1.7223864 -epoch: 69 step: 5004, loss is 2.0665488 -epoch: 70 step: 5004, loss is 1.8717369 -... -``` - -### Infer - -#### Usage - -``` -# infer -sh run_eval.sh [VALIDATION_DATASET_PATH] [CHECKPOINT_PATH] -``` - -#### Launch - -```bash -# infer with checkpoint -sh run_eval.sh dataset/validation_preprocess/ train_parallel0/resnet-120_5004.ckpt - -``` - -> checkpoint can be produced in training process. - - -#### Result - -Inference result will be stored in the scripts path, whose folder name is "eval". Under this, you can find result like the followings in log. - -``` -result: {'top_5_accuracy': 0.9429417413572343, 'top_1_accuracy': 0.7853513124199744} ckpt=train_parallel0/resnet-120_5004.ckpt -``` diff --git a/model_zoo/resnet101/eval.py b/model_zoo/resnet101/eval.py deleted file mode 100755 index 73c0289ebd7..00000000000 --- a/model_zoo/resnet101/eval.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -eval. -""" -import os -import argparse -import random -import numpy as np -from mindspore import context -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.train.model import Model, ParallelMode -from mindspore.train.serialization import load_checkpoint, load_param_into_net -import mindspore.dataset.engine as de -from mindspore.communication.management import init -from src.resnet101 import resnet101 -from src.dataset import create_dataset -from src.config import config -from src.crossentropy import CrossEntropy - -random.seed(1) -np.random.seed(1) -de.config.set_seed(1) - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=False, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=True, help='Do eval or not.') -parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -args_opt = parser.parse_args() - -device_id = int(os.getenv('DEVICE_ID')) - -context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id) - -if __name__ == '__main__': - if not args_opt.do_eval and args_opt.run_distribute: - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True, parameter_broadcast=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([180, 313]) - init() - - epoch_size = config.epoch_size - net = resnet101(class_num=config.class_num) - - if not config.label_smooth: - config.label_smooth_factor = 0.0 - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - - if args_opt.do_eval: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size) - step_size = dataset.get_dataset_size() - - if args_opt.checkpoint_path: - param_dict = load_checkpoint(args_opt.checkpoint_path) - load_param_into_net(net, param_dict) - net.set_train(False) - - model = Model(net, loss_fn=loss, metrics={'top_1_accuracy', 'top_5_accuracy'}) - res = model.eval(dataset) - print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/model_zoo/resnet101/src/config.py b/model_zoo/resnet101/src/config.py deleted file mode 100755 index 594b28522ac..00000000000 --- a/model_zoo/resnet101/src/config.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -network config setting, will be used in train.py and eval.py -""" -from easydict import EasyDict as ed - -config = ed({ - "class_num": 1001, - "batch_size": 32, - "loss_scale": 1024, - "momentum": 0.9, - "weight_decay": 1e-4, - "epoch_size": 120, - "pretrain_epoch_size": 0, - "buffer_size": 1000, - "image_height": 224, - "image_width": 224, - "save_checkpoint": True, - "save_checkpoint_epochs": 5, - "keep_checkpoint_max": 10, - "save_checkpoint_path": "./", - "warmup_epochs": 0, - "lr_decay_mode": "cosine", - "label_smooth": 1, - "label_smooth_factor": 0.1, - "lr": 0.1 -}) diff --git a/model_zoo/resnet101/src/dataset.py b/model_zoo/resnet101/src/dataset.py deleted file mode 100755 index b2a074a535a..00000000000 --- a/model_zoo/resnet101/src/dataset.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -create train or eval dataset. -""" -import os -import mindspore.common.dtype as mstype -import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as C -import mindspore.dataset.transforms.c_transforms as C2 -from src.config import config - -def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): - """ - create a train or evaluate dataset - Args: - dataset_path(string): the path of dataset. - do_train(bool): whether dataset is used for train or eval. - repeat_num(int): the repeat times of dataset. Default: 1 - batch_size(int): the batch size of dataset. Default: 32 - - Returns: - dataset - """ - device_num = int(os.getenv("RANK_SIZE")) - rank_id = int(os.getenv("RANK_ID")) - - if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) - else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) - resize_height = 224 - rescale = 1.0 / 255.0 - shift = 0.0 - - # define map operations - decode_op = C.Decode() - - random_resize_crop_op = C.RandomResizedCrop(resize_height, (0.08, 1.0), (0.75, 1.33), max_attempts=100) - horizontal_flip_op = C.RandomHorizontalFlip(rank_id / (rank_id + 1)) - resize_op_256 = C.Resize((256, 256)) - center_crop = C.CenterCrop(224) - rescale_op = C.Rescale(rescale, shift) - normalize_op = C.Normalize((0.475, 0.451, 0.392), (0.275, 0.267, 0.278)) - changeswap_op = C.HWC2CHW() - - trans = [] - if do_train: - trans = [decode_op, - random_resize_crop_op, - horizontal_flip_op, - rescale_op, - normalize_op, - changeswap_op] - - else: - trans = [decode_op, - resize_op_256, - center_crop, - rescale_op, - normalize_op, - changeswap_op] - - type_cast_op = C2.TypeCast(mstype.int32) - - ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=8) - ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) - - # apply shuffle operations - ds = ds.shuffle(buffer_size=config.buffer_size) - # apply batch operations - ds = ds.batch(batch_size, drop_remainder=True) - # apply dataset repeat operation - ds = ds.repeat(repeat_num) - - return ds diff --git a/model_zoo/resnet101/src/lr_generator.py b/model_zoo/resnet101/src/lr_generator.py deleted file mode 100755 index 2392e7a7bf8..00000000000 --- a/model_zoo/resnet101/src/lr_generator.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""learning rate generator""" -import math -import numpy as np - -def linear_warmup_lr(current_step, warmup_steps, base_lr, init_lr): - lr_inc = (float(base_lr) - float(init_lr)) / float(warmup_steps) - lr = float(init_lr) + lr_inc * current_step - return lr - -def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch=120, global_step=0): - """ - generate learning rate array with cosine - - Args: - lr(float): base learning rate - steps_per_epoch(int): steps size of one epoch - warmup_epochs(int): number of warmup epochs - max_epoch(int): total epochs of training - global_step(int): the current start index of lr array - Returns: - np.array, learning rate array - """ - base_lr = lr - warmup_init_lr = 0 - total_steps = int(max_epoch * steps_per_epoch) - warmup_steps = int(warmup_epochs * steps_per_epoch) - decay_steps = total_steps - warmup_steps - - lr_each_step = [] - for i in range(total_steps): - if i < warmup_steps: - lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) - else: - linear_decay = (total_steps - i) / decay_steps - cosine_decay = 0.5 * (1 + math.cos(math.pi * 2 * 0.47 * i / decay_steps)) - decayed = linear_decay * cosine_decay + 0.00001 - lr = base_lr * decayed - lr_each_step.append(lr) - - lr_each_step = np.array(lr_each_step).astype(np.float32) - learning_rate = lr_each_step[global_step:] - return learning_rate diff --git a/model_zoo/resnet101/train.py b/model_zoo/resnet101/train.py deleted file mode 100755 index 1cd3627a119..00000000000 --- a/model_zoo/resnet101/train.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""train_imagenet.""" -import os -import argparse -import random -import numpy as np -from mindspore import context -from mindspore import Tensor -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.nn.optim.momentum import Momentum -from mindspore.train.model import Model, ParallelMode -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor -from mindspore.train.loss_scale_manager import FixedLossScaleManager -from mindspore.train.serialization import load_checkpoint, load_param_into_net -import mindspore.dataset.engine as de -from mindspore.communication.management import init -import mindspore.nn as nn -import mindspore.common.initializer as weight_init -from src.resnet101 import resnet101 -from src.dataset import create_dataset -from src.lr_generator import warmup_cosine_annealing_lr -from src.config import config -from src.crossentropy import CrossEntropy - -random.seed(1) -np.random.seed(1) -de.config.set_seed(1) - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--pre_trained', type=str, default=None, help='Pretrained checkpoint path') -args_opt = parser.parse_args() - -device_id = int(os.getenv('DEVICE_ID')) - -context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id, - enable_auto_mixed_precision=True) - -if __name__ == '__main__': - if not args_opt.do_eval and args_opt.run_distribute: - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True, parameter_broadcast=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([180, 313]) - init() - - epoch_size = config.epoch_size - net = resnet101(class_num=config.class_num) - # weight init - for _, cell in net.cells_and_names(): - if isinstance(cell, nn.Conv2d): - cell.weight.default_input = weight_init.initializer(weight_init.XavierUniform(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() - if isinstance(cell, nn.Dense): - cell.weight.default_input = weight_init.initializer(weight_init.TruncatedNormal(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() - if not config.label_smooth: - config.label_smooth_factor = 0.0 - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - if args_opt.do_train: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, - repeat_num=epoch_size, batch_size=config.batch_size) - step_size = dataset.get_dataset_size() - loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) - if args_opt.pre_trained: - param_dict = load_checkpoint(args_opt.pre_trained) - load_param_into_net(net, param_dict) - - # learning rate strategy with cosine - lr = Tensor(warmup_cosine_annealing_lr(config.lr, step_size, config.warmup_epochs, 120, - config.pretrain_epoch_size*step_size)) - opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, - config.weight_decay, config.loss_scale) - model = Model(net, loss_fn=loss, optimizer=opt, amp_level='O2', keep_batchnorm_fp32=False, - loss_scale_manager=loss_scale, metrics={'acc'}) - time_cb = TimeMonitor(data_size=step_size) - loss_cb = LossMonitor() - cb = [time_cb, loss_cb] - if config.save_checkpoint: - config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs*step_size, - keep_checkpoint_max=config.keep_checkpoint_max) - ckpt_cb = ModelCheckpoint(prefix="resnet", directory=config.save_checkpoint_path, config=config_ck) - cb += [ckpt_cb] - model.train(epoch_size, dataset, callbacks=cb)