p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!3927 Remove redundent codes in fasterrcnn 

Merge pull request !3927 from zhouyuanshen/master
This commit is contained in:
mindspore-ci-bot 2020-08-04 17:37:41 +08:00 committed by Gitee
parent 8d6f5efaee 26327fa638
commit 8a0b3e234d
4 changed files with 56 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
model_zoo/official/cv/faster_rcnn/eval.py
View File

@ -108,26 +108,26 @@ if __name__ == '__main__':
prefix = "FasterRcnn_eval.mindrecord" prefix = "FasterRcnn_eval.mindrecord"
mindrecord_dir = config.mindrecord_dir mindrecord_dir = config.mindrecord_dir
mindrecord_file = os.path.join(mindrecord_dir, prefix) mindrecord_file = os.path.join(mindrecord_dir, prefix)
if args_opt.rank_id == 0 and not os.path.exists(mindrecord_file): print("CHECKING MINDRECORD FILES ...")
if not os.path.exists(mindrecord_file):
if not os.path.isdir(mindrecord_dir): if not os.path.isdir(mindrecord_dir):
os.makedirs(mindrecord_dir) os.makedirs(mindrecord_dir)
if args_opt.dataset == "coco": if args_opt.dataset == "coco":
if os.path.isdir(config.coco_root): if os.path.isdir(config.coco_root):
print("Create Mindrecord.") print("Create Mindrecord. It may take some time.")
data_to_mindrecord_byte_image("coco", False, prefix, file_num=1) data_to_mindrecord_byte_image("coco", False, prefix, file_num=1)
print("Create Mindrecord Done, at {}".format(mindrecord_dir)) print("Create Mindrecord Done, at {}".format(mindrecord_dir))
else: else:
print("coco_root not exits.") print("coco_root not exits.")
else: else:
if os.path.isdir(config.IMAGE_DIR) and os.path.exists(config.ANNO_PATH): if os.path.isdir(config.IMAGE_DIR) and os.path.exists(config.ANNO_PATH):
print("Create Mindrecord.") print("Create Mindrecord. It may take some time.")
data_to_mindrecord_byte_image("other", False, prefix, file_num=1) data_to_mindrecord_byte_image("other", False, prefix, file_num=1)
print("Create Mindrecord Done, at {}".format(mindrecord_dir)) print("Create Mindrecord Done, at {}".format(mindrecord_dir))
else: else:
print("IMAGE_DIR or ANNO_PATH not exits.") print("IMAGE_DIR or ANNO_PATH not exits.")
while not os.path.exists(mindrecord_file + ".db"): print("CHECKING MINDRECORD FILES DONE!")
time.sleep(5)
print("Start Eval!") print("Start Eval!")
FasterRcnn_eval(mindrecord_file, args_opt.checkpoint_path, args_opt.ann_file) FasterRcnn_eval(mindrecord_file, args_opt.checkpoint_path, args_opt.ann_file)

2
model_zoo/official/cv/faster_rcnn/scripts/run_distribute_train.sh
View File

@ -62,6 +62,6 @@ do
cd ./train_parallel$i || exit cd ./train_parallel$i || exit
echo "start training for rank $RANK_ID, device $DEVICE_ID" echo "start training for rank $RANK_ID, device $DEVICE_ID"
env > env.log env > env.log
python train.py --do_train=True --device_id=$i --rank_id=$i --run_distribute=True --device_num=$DEVICE_NUM --pre_trained=$PATH2 &> log & python train.py --device_id=$i --rank_id=$i --run_distribute=True --device_num=$DEVICE_NUM --pre_trained=$PATH2 &> log &
cd .. cd ..
done done

2
model_zoo/official/cv/faster_rcnn/scripts/run_standalone_train.sh
View File

@ -54,5 +54,5 @@ cp -r ../src ./train
cd ./train || exit cd ./train || exit
echo "start training for device $DEVICE_ID" echo "start training for device $DEVICE_ID"
env > env.log env > env.log
python train.py --do_train=True --device_id=$DEVICE_ID --pre_trained=$PATH1 &> log & python train.py --device_id=$DEVICE_ID --pre_trained=$PATH1 &> log &
cd .. cd ..

97
model_zoo/official/cv/faster_rcnn/train.py
View File

@ -41,22 +41,18 @@ np.random.seed(1)
de.config.set_seed(1) de.config.set_seed(1)
parser = argparse.ArgumentParser(description="FasterRcnn training") parser = argparse.ArgumentParser(description="FasterRcnn training")
parser.add_argument("--only_create_dataset", type=bool, default=False, help="If set it true, only create " parser.add_argument("--run_distribute", type=bool, default=False, help="Run distribute, default: false.")
"Mindrecord, default is false.") parser.add_argument("--dataset", type=str, default="coco", help="Dataset name, default: coco.")
parser.add_argument("--run_distribute", type=bool, default=False, help="Run distribute, default is false.") parser.add_argument("--pre_trained", type=str, default="", help="Pretrained file path.")
parser.add_argument("--do_train", type=bool, default=True, help="Do train or not, default is true.") parser.add_argument("--device_id", type=int, default=0, help="Device id, default: 0.")
parser.add_argument("--do_eval", type=bool, default=False, help="Do eval or not, default is false.") parser.add_argument("--device_num", type=int, default=1, help="Use device nums, default: 1.")
parser.add_argument("--dataset", type=str, default="coco", help="Dataset, default is coco.") parser.add_argument("--rank_id", type=int, default=0, help="Rank id, default: 0.")
parser.add_argument("--pre_trained", type=str, default="", help="Pretrain file path.")
parser.add_argument("--device_id", type=int, default=0, help="Device id, default is 0.")
parser.add_argument("--device_num", type=int, default=1, help="Use device nums, default is 1.")
parser.add_argument("--rank_id", type=int, default=0, help="Rank id, default is 0.")
args_opt = parser.parse_args() args_opt = parser.parse_args()
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=args_opt.device_id) context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=args_opt.device_id)
if __name__ == '__main__': if __name__ == '__main__':
if not args_opt.do_eval and args_opt.run_distribute: if args_opt.run_distribute:
rank = args_opt.rank_id rank = args_opt.rank_id
device_num = args_opt.device_num device_num = args_opt.device_num
context.set_auto_parallel_context(device_num=device_num, parallel_mode=ParallelMode.DATA_PARALLEL, context.set_auto_parallel_context(device_num=device_num, parallel_mode=ParallelMode.DATA_PARALLEL,
@ -73,19 +69,21 @@ if __name__ == '__main__':
prefix = "FasterRcnn.mindrecord" prefix = "FasterRcnn.mindrecord"
mindrecord_dir = config.mindrecord_dir mindrecord_dir = config.mindrecord_dir
mindrecord_file = os.path.join(mindrecord_dir, prefix + "0") mindrecord_file = os.path.join(mindrecord_dir, prefix + "0")
print("CHECKING MINDRECORD FILES ...")
if rank == 0 and not os.path.exists(mindrecord_file): if rank == 0 and not os.path.exists(mindrecord_file):
if not os.path.isdir(mindrecord_dir): if not os.path.isdir(mindrecord_dir):
os.makedirs(mindrecord_dir) os.makedirs(mindrecord_dir)
if args_opt.dataset == "coco": if args_opt.dataset == "coco":
if os.path.isdir(config.coco_root): if os.path.isdir(config.coco_root):
print("Create Mindrecord.") print("Create Mindrecord. It may take some time.")
data_to_mindrecord_byte_image("coco", True, prefix) data_to_mindrecord_byte_image("coco", True, prefix)
print("Create Mindrecord Done, at {}".format(mindrecord_dir)) print("Create Mindrecord Done, at {}".format(mindrecord_dir))
else: else:
print("coco_root not exits.") print("coco_root not exits.")
else: else:
if os.path.isdir(config.IMAGE_DIR) and os.path.exists(config.ANNO_PATH): if os.path.isdir(config.IMAGE_DIR) and os.path.exists(config.ANNO_PATH):
print("Create Mindrecord.") print("Create Mindrecord. It may take some time.")
data_to_mindrecord_byte_image("other", True, prefix) data_to_mindrecord_byte_image("other", True, prefix)
print("Create Mindrecord Done, at {}".format(mindrecord_dir)) print("Create Mindrecord Done, at {}".format(mindrecord_dir))
else: else:
@ -94,47 +92,48 @@ if __name__ == '__main__':
while not os.path.exists(mindrecord_file + ".db"): while not os.path.exists(mindrecord_file + ".db"):
time.sleep(5) time.sleep(5)
if not args_opt.only_create_dataset: print("CHECKING MINDRECORD FILES DONE!")
loss_scale = float(config.loss_scale)
# When create MindDataset, using the fitst mindrecord file, such as FasterRcnn.mindrecord0. loss_scale = float(config.loss_scale)
dataset = create_fasterrcnn_dataset(mindrecord_file, repeat_num=1,
batch_size=config.batch_size, device_num=device_num, rank_id=rank)
dataset_size = dataset.get_dataset_size() # When create MindDataset, using the fitst mindrecord file, such as FasterRcnn.mindrecord0.
print("Create dataset done!") dataset = create_fasterrcnn_dataset(mindrecord_file, repeat_num=1,
batch_size=config.batch_size, device_num=device_num, rank_id=rank)
net = Faster_Rcnn_Resnet50(config=config) dataset_size = dataset.get_dataset_size()
net = net.set_train() print("Create dataset done!")
load_path = args_opt.pre_trained net = Faster_Rcnn_Resnet50(config=config)
if load_path != "": net = net.set_train()
param_dict = load_checkpoint(load_path)
for item in list(param_dict.keys()):
if not item.startswith('backbone'):
param_dict.pop(item)
load_param_into_net(net, param_dict)
loss = LossNet() load_path = args_opt.pre_trained
lr = Tensor(dynamic_lr(config, rank_size=device_num), mstype.float32) if load_path != "":
param_dict = load_checkpoint(load_path)
for item in list(param_dict.keys()):
if not item.startswith('backbone'):
param_dict.pop(item)
load_param_into_net(net, param_dict)
opt = SGD(params=net.trainable_params(), learning_rate=lr, momentum=config.momentum, loss = LossNet()
weight_decay=config.weight_decay, loss_scale=config.loss_scale) lr = Tensor(dynamic_lr(config, rank_size=device_num), mstype.float32)
net_with_loss = WithLossCell(net, loss)
if args_opt.run_distribute:
net = TrainOneStepCell(net_with_loss, net, opt, sens=config.loss_scale, reduce_flag=True,
mean=True, degree=device_num)
else:
net = TrainOneStepCell(net_with_loss, net, opt, sens=config.loss_scale)
time_cb = TimeMonitor(data_size=dataset_size) opt = SGD(params=net.trainable_params(), learning_rate=lr, momentum=config.momentum,
loss_cb = LossCallBack() weight_decay=config.weight_decay, loss_scale=config.loss_scale)
cb = [time_cb, loss_cb] net_with_loss = WithLossCell(net, loss)
if config.save_checkpoint: if args_opt.run_distribute:
ckptconfig = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs * dataset_size, net = TrainOneStepCell(net_with_loss, net, opt, sens=config.loss_scale, reduce_flag=True,
keep_checkpoint_max=config.keep_checkpoint_max) mean=True, degree=device_num)
ckpoint_cb = ModelCheckpoint(prefix='faster_rcnn', directory=config.save_checkpoint_path, config=ckptconfig) else:
cb += [ckpoint_cb] net = TrainOneStepCell(net_with_loss, net, opt, sens=config.loss_scale)
model = Model(net) time_cb = TimeMonitor(data_size=dataset_size)
model.train(config.epoch_size, dataset, callbacks=cb) loss_cb = LossCallBack()
cb = [time_cb, loss_cb]
if config.save_checkpoint:
ckptconfig = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs * dataset_size,
keep_checkpoint_max=config.keep_checkpoint_max)
ckpoint_cb = ModelCheckpoint(prefix='faster_rcnn', directory=config.save_checkpoint_path, config=ckptconfig)
cb += [ckpoint_cb]
model = Model(net)
model.train(config.epoch_size, dataset, callbacks=cb)