sbjkx
/
mindspore
forked from OSSInnovation/mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

adjust order of split allreduce

This commit is contained in:
wangmin 2020-09-18 16:34:15 +08:00
parent faa0a6ad45
commit 8d525fc051
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
model_zoo/official/nlp/bert_thor/run_pretrain.py
View File

@ -153,8 +153,8 @@ def run_pretrain():
device_num = D.get_group_size() device_num = D.get_group_size()
rank = D.get_rank() rank = D.get_rank()
ckpt_save_dir = args_opt.save_checkpoint_path + 'ckpt_' + str(rank) + '/' ckpt_save_dir = args_opt.save_checkpoint_path + 'ckpt_' + str(rank) + '/'
_set_bert_all_reduce_split()
context.reset_auto_parallel_context() context.reset_auto_parallel_context()
_set_bert_all_reduce_split()
context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL, gradients_mean=True, context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL, gradients_mean=True,
device_num=device_num) device_num=device_num)