From 8d525fc05103bd0570ae60418bf0259fb85574be Mon Sep 17 00:00:00 2001 From: wangmin Date: Fri, 18 Sep 2020 16:34:15 +0800 Subject: [PATCH] adjust order of split allreduce --- model_zoo/official/nlp/bert_thor/run_pretrain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model_zoo/official/nlp/bert_thor/run_pretrain.py b/model_zoo/official/nlp/bert_thor/run_pretrain.py index 767538a471..e00eb2de79 100644 --- a/model_zoo/official/nlp/bert_thor/run_pretrain.py +++ b/model_zoo/official/nlp/bert_thor/run_pretrain.py @@ -153,8 +153,8 @@ def run_pretrain(): device_num = D.get_group_size() rank = D.get_rank() ckpt_save_dir = args_opt.save_checkpoint_path + 'ckpt_' + str(rank) + '/' - _set_bert_all_reduce_split() context.reset_auto_parallel_context() + _set_bert_all_reduce_split() context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL, gradients_mean=True, device_num=device_num)