From fa6e4dd4f913217b93be9e4e8e30e5570e4eb005 Mon Sep 17 00:00:00 2001 From: huchunmei Date: Wed, 16 Jun 2021 18:39:32 +0800 Subject: [PATCH] clould --- model_zoo/official/cv/maskrcnn/eval.py | 10 +++++----- .../cv/maskrcnn/scripts/run_distribute_train.sh | 2 +- model_zoo/official/cv/maskrcnn/scripts/run_eval.sh | 2 +- .../cv/maskrcnn/scripts/run_standalone_train.sh | 2 +- model_zoo/official/cv/maskrcnn/src/dataset.py | 6 ------ model_zoo/official/cv/maskrcnn/train.py | 11 ++++++----- model_zoo/official/cv/mobilenetv1/default_config.yaml | 7 +++---- .../cv/mobilenetv1/default_config_imagenet.yaml | 5 ++--- model_zoo/official/cv/mobilenetv1/train.py | 4 ++-- model_zoo/research/audio/fcn-4/default_config.yaml | 3 +-- model_zoo/research/audio/fcn-4/train.py | 2 +- 11 files changed, 23 insertions(+), 31 deletions(-) diff --git a/model_zoo/official/cv/maskrcnn/eval.py b/model_zoo/official/cv/maskrcnn/eval.py index d40c6192fb5..d4dc263c6e7 100644 --- a/model_zoo/official/cv/maskrcnn/eval.py +++ b/model_zoo/official/cv/maskrcnn/eval.py @@ -162,16 +162,16 @@ def modelarts_process(): print("#" * 200, os.listdir(save_dir_1)) print("#" * 200, os.listdir(os.path.join(config.data_path, config.modelarts_dataset_unzip_name))) - config.dataset_path = os.path.join(config.data_path, config.modelarts_dataset_unzip_name) - config.coco_root = config.dataset_path - config.checkpoint_path = os.path.join(config.dataset_path, config.ckpt_path) - config.ann_file = os.path.join(config.dataset_path, config.ann_file) - config.mindrecord_dir = os.path.join(config.dataset_path, config.mindrecord_dir) + config.coco_root = os.path.join(config.data_path, config.modelarts_dataset_unzip_name) + config.checkpoint_path = os.path.join(config.output_path, config.ckpt_path) + config.ann_file = os.path.join(config.coco_root, config.ann_file) context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=get_device_id()) @moxing_wrapper(pre_process=modelarts_process) def eval_(): + config.mindrecord_dir = os.path.join(config.coco_root, config.mindrecord_dir) + print('\neval.py config:\n', config) prefix = "MaskRcnn_eval.mindrecord" mindrecord_dir = config.mindrecord_dir mindrecord_file = os.path.join(mindrecord_dir, prefix) diff --git a/model_zoo/official/cv/maskrcnn/scripts/run_distribute_train.sh b/model_zoo/official/cv/maskrcnn/scripts/run_distribute_train.sh index 3f02bec5962..972f625e1a1 100644 --- a/model_zoo/official/cv/maskrcnn/scripts/run_distribute_train.sh +++ b/model_zoo/official/cv/maskrcnn/scripts/run_distribute_train.sh @@ -77,6 +77,6 @@ do echo "start training for rank $RANK_ID, device $DEVICE_ID" env > env.log taskset -c $cmdopt python train.py --do_train=True --device_id=$i --rank_id=$i --run_distribute=True --device_num=$DEVICE_NUM \ - --pre_trained=$PATH2 --data_path=$PATH3 &> log & + --pre_trained=$PATH2 --coco_root=$PATH3 &> log & cd .. done diff --git a/model_zoo/official/cv/maskrcnn/scripts/run_eval.sh b/model_zoo/official/cv/maskrcnn/scripts/run_eval.sh index 1c20b07c7e9..154cbb1e610 100755 --- a/model_zoo/official/cv/maskrcnn/scripts/run_eval.sh +++ b/model_zoo/official/cv/maskrcnn/scripts/run_eval.sh @@ -65,5 +65,5 @@ cd ./eval || exit env > env.log_eval echo "start eval for device $DEVICE_ID" python ./eval.py --device_id=$DEVICE_ID --ann_file=$PATH1 --checkpoint_path=$PATH2 \ ---data_path=$PATH3 &> log_eval.txt & +--coco_root=$PATH3 &> log_eval.txt & cd .. diff --git a/model_zoo/official/cv/maskrcnn/scripts/run_standalone_train.sh b/model_zoo/official/cv/maskrcnn/scripts/run_standalone_train.sh index 10472dee775..8a322ceb7b7 100755 --- a/model_zoo/official/cv/maskrcnn/scripts/run_standalone_train.sh +++ b/model_zoo/official/cv/maskrcnn/scripts/run_standalone_train.sh @@ -56,5 +56,5 @@ cp -r ../src ./train cd ./train || exit echo "start training for device $DEVICE_ID" env > env.log -python ./train.py --do_train=True --device_id=$DEVICE_ID --pre_trained=$PATH1 --data_path=$PATH2 &> log.txt & +python ./train.py --do_train=True --device_id=$DEVICE_ID --pre_trained=$PATH1 --coco_root=$PATH2 &> log.txt & cd .. diff --git a/model_zoo/official/cv/maskrcnn/src/dataset.py b/model_zoo/official/cv/maskrcnn/src/dataset.py index d0be58381ff..3a48eb2bb89 100644 --- a/model_zoo/official/cv/maskrcnn/src/dataset.py +++ b/model_zoo/official/cv/maskrcnn/src/dataset.py @@ -31,12 +31,6 @@ from .model_utils.config import config config.mask_shape = (28, 28) -if config.enable_modelarts and config.need_modelarts_dataset_unzip: - config.coco_root = os.path.join(config.data_path, config.modelarts_dataset_unzip_name) -else: - config.coco_root = config.data_path -config.mindrecord_dir = os.path.join(config.coco_root, config.mindrecord_dir) - def bbox_overlaps(bboxes1, bboxes2, mode='iou'): """Calculate the ious between each bbox of bboxes1 and bboxes2. diff --git a/model_zoo/official/cv/maskrcnn/train.py b/model_zoo/official/cv/maskrcnn/train.py index 27099774f30..c75523f4829 100644 --- a/model_zoo/official/cv/maskrcnn/train.py +++ b/model_zoo/official/cv/maskrcnn/train.py @@ -98,16 +98,17 @@ def modelarts_pre_process(): print("#" * 200, os.listdir(save_dir_1)) print("#" * 200, os.listdir(os.path.join(config.data_path, config.modelarts_dataset_unzip_name))) - config.dataset_path = os.path.join(config.data_path, config.modelarts_dataset_unzip_name) - config.pre_trained = os.path.join(config.dataset_path, config.ckpt_path) + config.coco_root = os.path.join(config.data_path, config.modelarts_dataset_unzip_name) + config.pre_trained = os.path.join(config.coco_root, config.pre_trained) config.save_checkpoint_path = config.output_path - config.mindrecord_dir = os.path.join(config.dataset_path, config.mindrecord_dir) context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=get_device_id()) @moxing_wrapper(pre_process=modelarts_pre_process) def train_maskrcnn(): + config.mindrecord_dir = os.path.join(config.coco_root, config.mindrecord_dir) + print('\ntrain.py config:\n', config) print("Start train for maskrcnn!") if not config.do_eval and config.run_distribute: init() @@ -130,12 +131,12 @@ def train_maskrcnn(): if not os.path.isdir(mindrecord_dir): os.makedirs(mindrecord_dir) if config.dataset == "coco": - if os.path.isdir(config.data_path): + if os.path.isdir(config.coco_root): print("Create Mindrecord.") data_to_mindrecord_byte_image("coco", True, prefix) print("Create Mindrecord Done, at {}".format(mindrecord_dir)) else: - raise Exception("data_path not exits.") + raise Exception("coco_root not exits.") else: if os.path.isdir(config.IMAGE_DIR) and os.path.exists(config.ANNO_PATH): print("Create Mindrecord.") diff --git a/model_zoo/official/cv/mobilenetv1/default_config.yaml b/model_zoo/official/cv/mobilenetv1/default_config.yaml index 88977fdcfc1..2bc7011cbcf 100644 --- a/model_zoo/official/cv/mobilenetv1/default_config.yaml +++ b/model_zoo/official/cv/mobilenetv1/default_config.yaml @@ -11,7 +11,7 @@ device_target: Ascend enable_profiling: False # ============================================================================== -modelarts_dataset_unzip_name: 'ImageNet_Original' +modelarts_dataset_unzip_name: 'cifar10' need_modelarts_dataset_unzip: True # config for mobilenet, cifar10 @@ -34,11 +34,10 @@ lr_max: 0.1 # Image classification - train dataset: 'cifar10' -run_distribute: True +run_distribute: False device_num: 1 dataset_path: "/cache/data" -device_target: 'Ascend' -pre_trained: "" +pre_trained: '' parameter_server: False # Image classification - eval diff --git a/model_zoo/official/cv/mobilenetv1/default_config_imagenet.yaml b/model_zoo/official/cv/mobilenetv1/default_config_imagenet.yaml index 1c9187fd5e9..59d11aa4db2 100644 --- a/model_zoo/official/cv/mobilenetv1/default_config_imagenet.yaml +++ b/model_zoo/official/cv/mobilenetv1/default_config_imagenet.yaml @@ -36,11 +36,10 @@ lr_end: 0.0 # Image classification - train dataset: 'imagenet2012' -run_distribute: True +run_distribute: False device_num: 1 dataset_path: "/cache/data" -device_target: 'Ascend' -pre_trained: "" +pre_trained: '' parameter_server: False # Image classification - eval diff --git a/model_zoo/official/cv/mobilenetv1/train.py b/model_zoo/official/cv/mobilenetv1/train.py index 562109a9787..51fd6e72559 100755 --- a/model_zoo/official/cv/mobilenetv1/train.py +++ b/model_zoo/official/cv/mobilenetv1/train.py @@ -97,8 +97,8 @@ def modelarts_pre_process(): print("#" * 200, os.listdir(os.path.join(config.data_path, config.modelarts_dataset_unzip_name))) config.dataset_path = os.path.join(config.data_path, config.modelarts_dataset_unzip_name) - config.ckpt_path = config.output_path - config.pre_trained = os.path.join(config.dataset_path, config.pre_trained) + config.save_checkpoint_path = config.output_path + # config.pre_trained = os.path.join(config.dataset_path, config.pre_trained) @moxing_wrapper(pre_process=modelarts_pre_process) diff --git a/model_zoo/research/audio/fcn-4/default_config.yaml b/model_zoo/research/audio/fcn-4/default_config.yaml index cea7fe27c0c..3e0a2fa7602 100644 --- a/model_zoo/research/audio/fcn-4/default_config.yaml +++ b/model_zoo/research/audio/fcn-4/default_config.yaml @@ -19,7 +19,6 @@ audio_path: "/cache/data" npy_path: "/cache/data" info_path: "/cache/data" info_name: 'annotations_final.csv' -device_target: 'Ascend' device_id: 0 mr_path: "/cache/data" mr_name: ['train', 'val'] @@ -41,7 +40,7 @@ prefix: 'MusicTagger' model_name: 'MusicTagger-10_543.ckpt' # export -file_name: "/cache/data/musicTagger/fcn-4.air" +file_name: "fcn-4" file_format: "MINDIR" # 310 infer diff --git a/model_zoo/research/audio/fcn-4/train.py b/model_zoo/research/audio/fcn-4/train.py index 7ee9fb3bf8d..7b79011794f 100644 --- a/model_zoo/research/audio/fcn-4/train.py +++ b/model_zoo/research/audio/fcn-4/train.py @@ -92,5 +92,5 @@ if __name__ == "__main__": save_checkpoint_steps=config.save_step, keep_checkpoint_max=config.keep_checkpoint_max, prefix=config.prefix, - directory=config.checkpoint_path + "_{}".format(get_device_id())) + directory=config.checkpoint_path) # + "_{}".format(get_device_id()) print("train success")