Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!46989 通信子图复用fix 

Merge pull request !46989 from lyqlola/hcom-test
This commit is contained in:
i-robot 2022-12-20 11:51:31 +00:00 committed by Gitee
parent 51d3553d2a 6a3b1dd728
commit c41d48d8e8
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
2 changed files with 19 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
mindspore/ccsrc/plugin/device/ascend/hal/device/ge_runtime/runtime_model.cc
View File

@ -125,7 +125,10 @@ void RuntimeModel::InitLabel(const std::shared_ptr<DavinciModel> &davinci_model)
rtLabel_t rt_label = nullptr;
rtError_t rt_ret = rtLabelCreateExV2(&rt_label, rt_model_handle_, stream_list_[label_set_task_info->stream_id()]);
if (rt_ret != RT_ERROR_NONE) {
MS_LOG(EXCEPTION) << "Call rt api rtLabelCreate failed, ret: " << rt_ret;
MS_LOG(EXCEPTION) << "Call rt api rtLabelCreate failed, ret: " << rt_ret
<< "\nIf you have set MS_COMM_COMPILER_OPT, notice that it will increase labels used and "
<< "may exceed the maximum label number: 1024. For more details, please refer to"
<< " 'MS_COMM_COMPILER_OPT' at https://www.mindspore.cn .";
}
label_list_[label_set_task_info->label_id()] = rt_label;
}

17
mindspore/ccsrc/plugin/device/ascend/hal/hardware/ascend_graph_optimization.cc
View File

@ -44,6 +44,7 @@ using AscendAutoMonad = mindspore::session::AscendAutoMonad;
namespace {
const std::unordered_set<std::string> kDefaultFormatAclOps = {kAddNOpName};
const size_t DEFAULT_MAX_COMM_OP_REUSE_NUM = 1000;
void RemoveUnusedValueNode(const KernelGraphPtr &graph) {
auto m = graph->manager();
@ -249,9 +250,21 @@ void AscendGraphOptimization::CommOpReuse(const KernelGraphPtr &graph) const {
if (!graph->is_graph_run_mode() || max_comm_op_reuse_num_env.empty()) {
return;
}
MS_LOG(INFO) << "Status record: start comm op reuse. graph id: " << graph->graph_id();
const size_t max_comm_op_reuse_num = LongToSize(std::stol(max_comm_op_reuse_num_env));
int64_t max_comm_op_reuse_num_l = -1;
TRY_AND_CATCH_WITH_EXCEPTION((max_comm_op_reuse_num_l = std::stol(max_comm_op_reuse_num_env)),
"Invalid MS_COMM_COMPILER_OPT value! It should be -1 or a positive integer.");
size_t max_comm_op_reuse_num;
if (max_comm_op_reuse_num_l == -1) {
max_comm_op_reuse_num = DEFAULT_MAX_COMM_OP_REUSE_NUM;
} else if (max_comm_op_reuse_num_l > 0) {
max_comm_op_reuse_num = LongToSize(max_comm_op_reuse_num_l);
} else {
MS_LOG(WARNING) << "MS_COMM_COMPILER_OPT should be -1 or a positive integer but set to " << max_comm_op_reuse_num_l
<< ". Comm subgraph reuse is disabled.";
return;
}
MS_LOG(INFO) << "MAX_COMM_OP_REUSE_NUM: " << max_comm_op_reuse_num;
MS_LOG(INFO) << "Status record: start comm op reuse. graph id: " << graph->graph_id();
opt::AscendCommOpReuse comm_io_reuse(graph, max_comm_op_reuse_num);
comm_io_reuse.Run();
MS_LOG(INFO) << "Status record: end comm op reuse. graph id: " << graph->graph_id();