Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!8903 add trace info when mindspore error 

From: @jjfeing
Reviewed-by: @chujinjin,@zh_qh,@kisnwang
Signed-off-by: @zh_qh
This commit is contained in:
mindspore-ci-bot 2020-11-25 10:04:30 +08:00 committed by Gitee
parent ecd9c4f255 27257b9901
commit 42cbdfcafc
34 changed files with 303 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
mindspore/ccsrc/backend/optimizer/ascend/ascend_helper.cc
View File

@ -27,7 +27,7 @@
#include "backend/session/anf_runtime_algorithm.h"
#include "backend/session/kernel_graph.h"
#include "utils/ms_context.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder;
@ -133,7 +133,7 @@ AnfNodePtr InsertTransOpForSingleOutput(const FuncGraphPtr &func_graph, const An
std::vector<size_t> origin_shape = AnfAlgo::GetOutputInferShape(node, 0);
if (output_format == kOpFormat_NC1KHKWHWC0) {
MS_LOG(EXCEPTION) << "got the hw format " << output_format << "when insert the transdata node "
<< node->DebugString();
<< node->DebugString() << " trace: " << trace::DumpSourceLines(node);
}
if (kCommonFormatSet.find(output_format) == kCommonFormatSet.end() && origin_shape.size() > 1) {
MS_LOG(DEBUG) << "Inserted Transdata " << output_format << " To default , index :0";
@ -166,7 +166,7 @@ AnfNodePtr InsertTransOpForMultipleOutput(const FuncGraphPtr &func_graph, const
std::string output_format = AnfAlgo::GetOutputFormat(node, output_idx);
if (output_format == kOpFormat_NC1KHKWHWC0) {
MS_LOG(EXCEPTION) << "Got the special format" << output_format << " when insert the transdata node "
<< node->DebugString();
<< node->DebugString() << " trace: " << trace::DumpSourceLines(node);
}
auto tuple_getitem = CreatTupleGetItemNode(func_graph, node, output_idx);
std::vector<size_t> origin_shape = AnfAlgo::GetOutputInferShape(node, output_idx);

7
mindspore/ccsrc/backend/optimizer/ascend/enhancer/insert_memcpy_async_for_cascade.cc
View File

@ -19,6 +19,7 @@
#include "backend/session/anf_runtime_algorithm.h"
#include "frontend/optimizer/opt.h"
#include "backend/optimizer/ascend/ascend_helper.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -45,7 +46,8 @@ bool IsPartOutputsOfHcclOp(const AnfNodePtr &node, const CNodePtr &cur_hccl, con
auto &node_users = manager->node_users();
auto iter = node_users.find(prev_hccl_op);
if (iter == node_users.end()) {
MS_LOG(EXCEPTION) << "node has no output in manager";
MS_LOG(EXCEPTION) << "node has no output in manager"
<< " trace: " << trace::DumpSourceLines(cur_hccl);
}
for (const auto &node_index : iter->second) {
AnfNodePtr output = node_index.first;
@ -79,7 +81,8 @@ AnfNodePtr InsertMemcpyAsyncForCascade::InsertMemcpyAsync(const FuncGraphPtr &gr
if (IsPartOutputsOfHcclOp(input, hccl_node, graph)) {
auto memcpy_async = CreateMemcpyAsyncOp(graph, input);
if (memcpy_async == nullptr) {
MS_LOG(EXCEPTION) << "Create memcpy_async op failed.";
MS_LOG(EXCEPTION) << "Create memcpy_async op failed."
<< " trace: " << trace::DumpSourceLines(hccl_node);
}
if (AnfAlgo::IsNodeDynamicShape(input)) {
AnfAlgo::SetNodeAttr(kAttrIsDynamicShape, MakeValue(true), memcpy_async);

11
mindspore/ccsrc/backend/optimizer/ascend/enhancer/insert_memcpy_async_for_hccl_op.cc
View File

@ -21,7 +21,7 @@
#include "backend/session/anf_runtime_algorithm.h"
#include "frontend/optimizer/opt.h"
#include "backend/optimizer/ascend/ascend_helper.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
namespace {
@ -61,7 +61,8 @@ void DealControlForGetitem(const CNodePtr &tuple_getitem, const FuncGraphPtr &gr
auto &node_users = manager->node_users();
auto iter = node_users.find(tuple_getitem);
if (iter == node_users.end()) {
MS_LOG(EXCEPTION) << "node has no output in manager";
MS_LOG(EXCEPTION) << "node has no output in manager"
<< " trace: " << trace::DumpSourceLines(hccl_node);
}
for (const auto &node_index : iter->second) {
AnfNodePtr output = node_index.first;
@ -81,7 +82,8 @@ void TransferControl(const CNodePtr &hccl_node, const std::vector<AnfNodePtr> &m
auto &node_users = manager->node_users();
auto iter = node_users.find(hccl_node);
if (iter == node_users.end()) {
MS_LOG(EXCEPTION) << "node has no output in manager";
MS_LOG(EXCEPTION) << "node has no output in manager"
<< " trace: " << trace::DumpSourceLines(hccl_node);
}
// find hccl_node's output which is a control depend
for (const auto &node_index : iter->second) {
@ -140,7 +142,8 @@ bool InsertMemcpyAsyncForHcclOp::NeedInsertMemcpy(const FuncGraphPtr &graph, con
// when input is used by others
auto iter = node_users.find(input);
if (iter == node_users.end()) {
MS_LOG(EXCEPTION) << "node has no output in manager";
MS_LOG(EXCEPTION) << "node has no output in manager"
<< " trace: " << trace::DumpSourceLines(input);
}
if (IsNodeOutPutUsedByOtherRealKernel(iter->second)) {
return true;

3
mindspore/ccsrc/backend/optimizer/ascend/format_type/split_unsupported_transdata.cc
View File

@ -18,6 +18,7 @@
#include <vector>
#include <memory>
#include "backend/session/anf_runtime_algorithm.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -39,7 +40,7 @@ const AnfNodePtr SplitUnsupportedTransData::Process(const FuncGraphPtr &func_gra
MS_EXCEPTION_IF_NULL(kernel_info);
if (kernel_info->GetInputNum() != 1 || kernel_info->GetOutputNum() != 1) {
MS_LOG(EXCEPTION) << "Transdata node's kernel info's input and output format size is not 1"
<< ori_trans_data->DebugString();
<< ori_trans_data->DebugString() << trace::DumpSourceLines(node);
}
return SplitTransData(func_graph, ori_trans_data);
}

13
mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_bert_fission.cc
View File

@ -19,6 +19,7 @@
#include <algorithm>
#include "backend/session/anf_runtime_algorithm.h"
#include "backend/optimizer/common/helper.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -65,7 +66,7 @@ AnfNodePtr CreateBNTrainingReduce(const FuncGraphPtr &func_graph, const AnfNodeP
MS_EXCEPTION_IF_NULL(bn_cnode);
if (bn_cnode->inputs().size() < kBatchNormRealInputNum + 1) {
MS_LOG(EXCEPTION) << "The input size of node " + bn_cnode->DebugString() + " is less than "
<< kBatchNormRealInputNum + 1;
<< kBatchNormRealInputNum + 1 << " trace: " << trace::DumpSourceLines(bn);
}
std::vector<AnfNodePtr> bn_training_reduce_inputs = {
NewValueNode(std::make_shared<Primitive>(kBNTrainingReduceOpName)), bn_cnode->input(1)};
@ -91,11 +92,12 @@ AnfNodePtr CreateBNTrainingUpdateV2(const FuncGraphPtr &func_graph, const AnfNod
MS_EXCEPTION_IF_NULL(bn_cnode);
if (bn_cnode->inputs().size() < kBatchNormRealInputNum + 1) {
MS_LOG(EXCEPTION) << "The input size of node " + bn_cnode->DebugString() + " is less than "
<< kBatchNormRealInputNum + 1;
<< kBatchNormRealInputNum + 1 << " trace: " << trace::DumpSourceLines(bn);
}
if (bn_training_reduce_outputs.size() != kBNTrainingReduceOutputNum) {
MS_LOG(EXCEPTION) << "The output size of node bn_training_reduce must be " << kBNTrainingReduceOutputNum
<< ", but it is " << bn_training_reduce_outputs.size();
<< ", but it is " << bn_training_reduce_outputs.size()
<< " trace: " << trace::DumpSourceLines(bn);
}
std::vector<AnfNodePtr> bn_training_update_v2_inputs = {
NewValueNode(std::make_shared<Primitive>(kBNTrainingUpdateV2OpName)),
@ -111,7 +113,7 @@ AnfNodePtr CreateBNTrainingUpdateV2(const FuncGraphPtr &func_graph, const AnfNod
MS_EXCEPTION_IF_NULL(bn_abstract_tuple);
if (bn_abstract_tuple->elements().size() != kBatchNormOutputNum) {
MS_LOG(EXCEPTION) << "The abstract size of node bn must be " << kBatchNormOutputNum << ", but it is "
<< bn_abstract_tuple->elements().size();
<< bn_abstract_tuple->elements().size() << " trace: " << trace::DumpSourceLines(bn);
}
std::vector<AbstractBasePtr> abstract_list{bn_abstract_tuple->elements()[0], bn_abstract_tuple->elements()[3],
bn_abstract_tuple->elements()[4]};
@ -155,7 +157,8 @@ const AnfNodePtr BatchNormBertFission::Process(const FuncGraphPtr &func_graph, c
&bn_training_update_v2_outputs);
if (bn_training_update_v2_outputs.size() != kBNTrainingUpdateV2OutputNum) {
MS_LOG(EXCEPTION) << "The output size of node bn_training_reduce must be " << kBNTrainingUpdateV2OutputNum
<< ", but it is " << bn_training_update_v2_outputs.size();
<< ", but it is " << bn_training_update_v2_outputs.size()
<< " trace: " << trace::DumpSourceLines(node);
}
auto manager = func_graph->manager();
MS_EXCEPTION_IF_NULL(manager);

31
mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_grad_infer_fission.cc
View File

@ -17,6 +17,7 @@
#include <vector>
#include "backend/optimizer/common/helper.h"
#include "backend/session/anf_runtime_algorithm.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -61,15 +62,18 @@ AnfNodePtr BatchNormGradInferFission::CreateBNInferGrad(const FuncGraphPtr &func
// Set inputs
auto iter_input0 = (*equiv).find(input0_var_);
if (iter_input0 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input0 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input0 var after matched."
<< " trace: " << trace::DumpSourceLines(bn_grad);
}
auto iter_input2 = (*equiv).find(input2_var_);
if (iter_input2 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input2 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input2 var after matched."
<< " trace: " << trace::DumpSourceLines(bn_grad);
}
auto iter_input4 = (*equiv).find(input4_var_);
if (iter_input4 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input4 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input4 var after matched."
<< " trace: " << trace::DumpSourceLines(bn_grad);
}
std::vector<AnfNodePtr> bn_infer_grad_inputs = {
NewValueNode(std::make_shared<Primitive>(kBNInferGradOpName)), utils::cast<AnfNodePtr>(iter_input0->second),
@ -80,7 +84,8 @@ AnfNodePtr BatchNormGradInferFission::CreateBNInferGrad(const FuncGraphPtr &func
auto bn_grad_abstract_tuple = dyn_cast<abstract::AbstractTuple>(bn_grad->abstract());
MS_EXCEPTION_IF_NULL(bn_grad_abstract_tuple);
if (bn_grad_abstract_tuple->elements().empty()) {
MS_LOG(EXCEPTION) << "The abstract tuple of node " << bn_grad->DebugString() << "should not be empty";
MS_LOG(EXCEPTION) << "The abstract tuple of node " << bn_grad->DebugString() << "should not be empty"
<< " trace: " << trace::DumpSourceLines(bn_grad);
}
bn_infer_grad->set_abstract(bn_grad_abstract_tuple->elements()[0]);
AnfAlgo::CopyNodeAttr(kAttrEpsilon, bn_grad, bn_infer_grad);
@ -97,19 +102,23 @@ AnfNodePtr BatchNormGradInferFission::CreateBNTrainingUpdateGrad(const FuncGraph
// Set inputs
auto iter_input0 = (*equiv).find(input0_var_);
if (iter_input0 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input0 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input0 var after matched."
<< " trace: " << trace::DumpSourceLines(bn_grad);
}
auto iter_input1 = (*equiv).find(input1_var_);
if (iter_input1 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input1 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input1 var after matched."
<< " trace: " << trace::DumpSourceLines(bn_grad);
}
auto iter_input3 = (*equiv).find(input3_var_);
if (iter_input3 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input3 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input3 var after matched."
<< " trace: " << trace::DumpSourceLines(bn_grad);
}
auto iter_input4 = (*equiv).find(input4_var_);
if (iter_input4 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input4 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input4 var after matched."
<< " trace: " << trace::DumpSourceLines(bn_grad);
}
std::vector<AnfNodePtr> bn_training_update_grad_inputs = {
NewValueNode(std::make_shared<Primitive>(kBNTrainingUpdateGradOpName)),
@ -121,7 +130,8 @@ AnfNodePtr BatchNormGradInferFission::CreateBNTrainingUpdateGrad(const FuncGraph
auto bn_grad_abstract_tuple = dyn_cast<abstract::AbstractTuple>(bn_grad->abstract());
MS_EXCEPTION_IF_NULL(bn_grad_abstract_tuple);
if (bn_grad_abstract_tuple->elements().size() < kBatchNormGradInferOutputNum) {
MS_LOG(EXCEPTION) << "The abstract tuple of node " << bn_grad->DebugString() << "should not be less than 3";
MS_LOG(EXCEPTION) << "The abstract tuple of node " << bn_grad->DebugString() << "should not be less than 3"
<< trace::DumpSourceLines(bn_grad);
}
std::vector<AbstractBasePtr> abstract_list{bn_grad_abstract_tuple->elements()[1],
bn_grad_abstract_tuple->elements()[2]};
@ -160,7 +170,8 @@ const AnfNodePtr BatchNormGradInferFission::Process(const FuncGraphPtr &func_gra
&bn_training_update_grad_outputs);
if (bn_training_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) {
MS_LOG(EXCEPTION) << "The output size of " << bn_training_update_grad << " should be "
<< kBNTrainingUpdateGradOutputNum << ", but it is " << bn_training_update_grad_outputs.size();
<< kBNTrainingUpdateGradOutputNum << ", but it is " << bn_training_update_grad_outputs.size()
<< trace::DumpSourceLines(node);
}
std::vector<AnfNodePtr> make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), bn_infer_grad,
bn_training_update_grad_outputs[0], bn_training_update_grad_outputs[1]};

19
mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_grad_split.cc
View File

@ -23,6 +23,7 @@
#include "backend/optimizer/common/helper.h"
#include "runtime/device/kernel_info.h"
#include "backend/session/anf_runtime_algorithm.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -33,7 +34,8 @@ void CreateOutputsOfUpdateGrad(const FuncGraphPtr &graph, const CNodePtr &bn_gra
MS_EXCEPTION_IF_NULL(bn_grad_node);
const auto &bn_grad_inputs = bn_grad_node->inputs();
if (bn_grad_inputs.size() < kBNGradInputNum) {
MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size";
MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size."
<< " trace: " << trace::DumpSourceLines(bn_grad_node);
}
std::vector<AnfNodePtr> bn_update_grad_inputs = {
NewValueNode(std::make_shared<Primitive>(kBNTrainingUpdateGradOpName)), bn_grad_inputs[1], bn_grad_inputs[2],
@ -59,10 +61,12 @@ void CreateOutputsOfReduceGrad(const FuncGraphPtr &graph, const CNodePtr &bn_gra
MS_EXCEPTION_IF_NULL(bn_reduce_grad_outputs);
const auto &bn_grad_inputs = bn_grad_node->inputs();
if (bn_grad_inputs.size() < kBNGradInputNum) {
MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size";
MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size"
<< " trace: " << trace::DumpSourceLines(bn_grad_node);
}
if (bn_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) {
MS_LOG(EXCEPTION) << "BNTrainingReduceGrad_outputs has wrong size";
MS_LOG(EXCEPTION) << "BNTrainingReduceGrad_outputs has wrong size"
<< " trace: " << trace::DumpSourceLines(bn_grad_node);
}
std::vector<AnfNodePtr> bn_reduce_grad_inputs = {
NewValueNode(std::make_shared<Primitive>(kBNTrainingReduceGradOpName)),
@ -101,7 +105,8 @@ const AnfNodePtr BatchNormGradSplit::Process(const FuncGraphPtr &func_graph, con
auto primitive = AnfAlgo::GetCNodePrimitive(cnode);
MS_EXCEPTION_IF_NULL(primitive);
if (!primitive->HasAttr(kAttrIsTraining)) {
MS_LOG(INFO) << "Op BatchNormGrad must have attrs of is_training";
MS_LOG(INFO) << "Op BatchNormGrad must have attrs of is_training"
<< " trace: " << trace::DumpSourceLines(node);
return nullptr;
}
if (!AnfAlgo::GetNodeAttr<bool>(cnode, kAttrIsTraining)) {
@ -112,13 +117,15 @@ const AnfNodePtr BatchNormGradSplit::Process(const FuncGraphPtr &func_graph, con
std::vector<AnfNodePtr> bn_update_grad_outputs;
CreateOutputsOfUpdateGrad(func_graph, cnode, &bn_update_grad_outputs);
if (bn_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) {
MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size";
MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size"
<< " trace: " << trace::DumpSourceLines(node);
}
std::vector<AnfNodePtr> bn_reduce_grad_outputs;
CreateOutputsOfReduceGrad(func_graph, cnode, bn_update_grad_outputs, &bn_reduce_grad_outputs);
if (bn_reduce_grad_outputs.size() != kSingleOutputNum) {
MS_LOG(EXCEPTION) << "bn_reduce_grad_outputs has wrong size";
MS_LOG(EXCEPTION) << "bn_reduce_grad_outputs has wrong size"
<< " trace: " << trace::DumpSourceLines(node);
}
std::vector<AnfNodePtr> make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), bn_reduce_grad_outputs[0],

13
mindspore/ccsrc/backend/optimizer/ascend/ir_fission/bn_grad_split.cc
View File

@ -23,6 +23,7 @@
#include "backend/optimizer/common/helper.h"
#include "runtime/device/kernel_info.h"
#include "backend/session/anf_runtime_algorithm.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -33,7 +34,8 @@ void CreateOutputsOfUpdateGrad(const FuncGraphPtr &graph, const CNodePtr &bn_gra
MS_EXCEPTION_IF_NULL(bn_grad_node);
auto bn_grad_inputs = bn_grad_node->inputs();
if (bn_grad_inputs.size() != kBNGradInputNum) {
MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size";
MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size"
<< " trace: " << trace::DumpSourceLines(bn_grad_node);
}
std::vector<AnfNodePtr> bn_update_grad_inputs = {
NewValueNode(std::make_shared<Primitive>(kBNTrainingUpdateGradOpName)), bn_grad_inputs[1], bn_grad_inputs[2],
@ -58,7 +60,8 @@ void CreateOutputsOfReduceGrad(const FuncGraphPtr &graph, const CNodePtr &bn_gra
MS_EXCEPTION_IF_NULL(bn_grad_node);
auto bn_grad_inputs = bn_grad_node->inputs();
if (bn_grad_inputs.size() != kBNGradInputNum) {
MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size";
MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size"
<< " trace: " << trace::DumpSourceLines(bn_grad_node);
}
if (bn_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) {
MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size";
@ -90,13 +93,15 @@ CNodePtr BNGradSplitForTBE(const FuncGraphPtr &func_graph, const CNodePtr &cnode
std::vector<AnfNodePtr> bn_update_grad_outputs;
CreateOutputsOfUpdateGrad(func_graph, cnode, &bn_update_grad_outputs);
if (bn_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) {
MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size";
MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size"
<< " trace: " << trace::DumpSourceLines(cnode);
}
std::vector<AnfNodePtr> bn_reduce_grad_outputs;
CreateOutputsOfReduceGrad(func_graph, cnode, bn_update_grad_outputs, &bn_reduce_grad_outputs);
if (bn_reduce_grad_outputs.size() != 1) {
MS_LOG(EXCEPTION) << "bn_reduce_grad_outputs has wrong size";
MS_LOG(EXCEPTION) << "bn_reduce_grad_outputs has wrong size"
<< " trace: " << trace::DumpSourceLines(cnode);
}
std::vector<AnfNodePtr> make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), bn_reduce_grad_outputs[0],

10
mindspore/ccsrc/backend/optimizer/ascend/ir_fission/bn_split.cc
View File

@ -23,6 +23,7 @@
#include "backend/optimizer/common/helper.h"
#include "runtime/device/kernel_info.h"
#include "backend/session/anf_runtime_algorithm.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -64,10 +65,12 @@ AnfNodePtr CreateOutputsOfBNTrainingUpdate(const FuncGraphPtr &graph, const CNod
MS_EXCEPTION_IF_NULL(graph);
MS_EXCEPTION_IF_NULL(bn_cnode);
if (bn_cnode->inputs().size() != kBnInputNum) {
MS_LOG(EXCEPTION) << "BN node has wrong input size";
MS_LOG(EXCEPTION) << "BN node has wrong input size"
<< " trace: " << trace::DumpSourceLines(bn_cnode);
}
if (bn_training_reduce_outputs.size() != kBNTrainingReduceOutputNum) {
MS_LOG(EXCEPTION) << "BN1 outputs has wrong input size";
MS_LOG(EXCEPTION) << "BN1 outputs has wrong input size"
<< " trace: " << trace::DumpSourceLines(bn_cnode);
}
// the inputs of BNTrainingUpdate are from the outputs of BNTrainingReduce and the inputs of BN
std::vector<AnfNodePtr> bn_training_update_inputs = {
@ -110,7 +113,8 @@ AnfNodePtr SplitFusedBatchNormForTBE(const FuncGraphPtr &func_graph, const AnfNo
return nullptr;
}
if (bn_training_reduce_outputs.size() != kBN1OutputNum) {
MS_LOG(EXCEPTION) << "make outputs of op BNTrainingReduce fail";
MS_LOG(EXCEPTION) << "make outputs of op BNTrainingReduce fail"
<< " trace: " << trace::DumpSourceLines(node);
}
// Create BNTrainingUpdate node

4
mindspore/ccsrc/backend/optimizer/ascend/ir_fission/concat_fission.cc
View File

@ -17,6 +17,7 @@
#include <memory>
#include <vector>
#include "backend/session/anf_runtime_algorithm.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -47,7 +48,8 @@ AnfNodePtr CreateNewConcat(const FuncGraphPtr &func_graph, const CNodePtr &origi
}
auto output_shape = AnfAlgo::GetOutputInferShape(origin_concat_cnode, 0);
if (axis < 0 || axis >= SizeToLong(output_shape.size()) || axis >= SizeToLong(input_shape.size())) {
MS_LOG(EXCEPTION) << "The concat_dim value " << axis << "is out of range";
MS_LOG(EXCEPTION) << "The concat_dim value " << axis << "is out of range"
<< " trace: " << trace::DumpSourceLines(origin_concat_cnode);
}
output_shape[axis] = input_shape[axis] * offset;
AnfAlgo::SetOutputInferTypeAndShape({AnfAlgo::GetOutputInferDataType(origin_concat_cnode, 0)}, {output_shape},

4
mindspore/ccsrc/backend/optimizer/ascend/ir_fission/dynamic_rnn_grad_fission_v2.cc
View File

@ -17,6 +17,7 @@
#include <vector>
#include <memory>
#include "backend/session/anf_runtime_algorithm.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -290,7 +291,8 @@ AnfNodePtr CreateHConcat(const FuncGraphPtr &func_graph, const CNodePtr &dynamic
std::vector<AnfNodePtr> splitv_outputs;
CreateMultipleOutputsOfAnfNode(func_graph, splitv, kSplitVOutputNum, &splitv_outputs);
if (splitv_outputs.size() != kSplitVOutputNum) {
MS_LOG(EXCEPTION) << "Create outputs of node " << splitv->DebugString() << " failed";
MS_LOG(EXCEPTION) << "Create outputs of node " << splitv->DebugString() << " failed"
<< " trace: " << trace::DumpSourceLines(dynamic_rnn_grad_cnode);
}
auto origin_input4 = dynamic_rnn_grad_cnode->input(5);
auto origin_input4_shape = AnfAlgo::GetOutputInferShape(origin_input4, 0);

10
mindspore/ccsrc/backend/optimizer/ascend/ir_fission/lars_v2_fission.cc
View File

@ -19,6 +19,7 @@
#include "backend/session/anf_runtime_algorithm.h"
#include "backend/optimizer/common/helper.h"
#include "utils/utils.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -28,7 +29,8 @@ void CreateOutputsOfSquareSumAll(const FuncGraphPtr &graph, const CNodePtr &lars
MS_EXCEPTION_IF_NULL(graph);
MS_EXCEPTION_IF_NULL(lars_v2);
if (lars_v2->size() != kLarsV2InputNum) {
MS_LOG(EXCEPTION) << "Op lars_v2's input not equal " << kLarsV2InputNum;
MS_LOG(EXCEPTION) << "Op lars_v2's input not equal " << kLarsV2InputNum
<< " trace: " << trace::DumpSourceLines(lars_v2);
}
std::vector<AnfNodePtr> inputs = {NewValueNode(std::make_shared<Primitive>(kSquareSumAllOpName)), lars_v2->input(1),
@ -50,10 +52,12 @@ CNodePtr CreateLarsV2Update(const FuncGraphPtr &graph, const CNodePtr &lars_v2,
MS_EXCEPTION_IF_NULL(graph);
MS_EXCEPTION_IF_NULL(lars_v2);
if (square_sum_all_outputs.size() != 2) {
MS_LOG(EXCEPTION) << "square_sum_all_outputs' size not equal 2";
MS_LOG(EXCEPTION) << "square_sum_all_outputs' size not equal 2"
<< " trace: " << trace::DumpSourceLines(lars_v2);
}
if (lars_v2->size() != kLarsV2InputNum) {
MS_LOG(EXCEPTION) << "Op lars_v2's input not equal " << kLarsV2InputNum;
MS_LOG(EXCEPTION) << "Op lars_v2's input not equal " << kLarsV2InputNum
<< " trace: " << trace::DumpSourceLines(lars_v2);
}
std::vector<AnfNodePtr> inputs = {NewValueNode(std::make_shared<Primitive>(kLarsV2UpdateOpName)),
lars_v2->input(1),

7
mindspore/ccsrc/backend/optimizer/ascend/ir_fission/layer_norm_grad_split.cc
View File

@ -22,6 +22,7 @@
#include "runtime/device/kernel_info.h"
#include "ir/primitive.h"
#include "utils/utils.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -98,14 +99,16 @@ const AnfNodePtr LayerNormGradSplit::Process(const FuncGraphPtr &graph, const An
std::vector<AnfNodePtr> layer_norm_x_backprop_outputs;
CreateOutputsOfLayerNormXBackprop(graph, cnode, &layer_norm_x_backprop_outputs);
if (layer_norm_x_backprop_outputs.size() != kSingleOutputNum) {
MS_LOG(EXCEPTION) << "layer_norm_grad_outputs has wrong size";
MS_LOG(EXCEPTION) << "layer_norm_grad_outputs has wrong size"
<< " trace: " << trace::DumpSourceLines(node);
}
// create layer_norm_beta_gamma_backprop
std::vector<AnfNodePtr> layer_norm_beta_gamma_backprop_outputs;
CreateOutputsOfLayerNormBetaGammaBackprop(graph, cnode, &layer_norm_beta_gamma_backprop_outputs);
if (layer_norm_beta_gamma_backprop_outputs.size() != kLayerNormBetaGammaBackpropOutputNum) {
MS_LOG(EXCEPTION) << "layer_norm_beta_gamma_outputs has wrong size";
MS_LOG(EXCEPTION) << "layer_norm_beta_gamma_outputs has wrong size"
<< " trace: " << trace::DumpSourceLines(node);
}
std::vector<AnfNodePtr> make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), layer_norm_x_backprop_outputs[0],

4
mindspore/ccsrc/backend/optimizer/ascend/ir_fission/pack_fission.cc
View File

@ -17,6 +17,7 @@
#include <memory>
#include <vector>
#include "backend/session/anf_runtime_algorithm.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -45,7 +46,8 @@ AnfNodePtr CreateNewPack(const FuncGraphPtr &func_graph, const CNodePtr &origin_
axis += output_shape.size();
}
if (axis < 0) {
MS_LOG(EXCEPTION) << "The concat_dim value " << axis << "is out of range";
MS_LOG(EXCEPTION) << "The concat_dim value " << axis << "is out of range"
<< " trace: " << trace::DumpSourceLines(origin_pack_cnode);
}
std::vector<size_t> new_shape;
for (size_t i = 0; i < output_shape.size() + 1; ++i) {

10
mindspore/ccsrc/backend/optimizer/ascend/ir_fission/single_batch_norm_fission.cc
View File

@ -18,6 +18,7 @@
#include <memory>
#include "backend/session/anf_runtime_algorithm.h"
#include "backend/optimizer/common/helper.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -31,7 +32,7 @@ AnfNodePtr CreateBNTrainingReduce(const FuncGraphPtr &func_graph, const AnfNodeP
MS_EXCEPTION_IF_NULL(bn_cnode);
if (bn_cnode->inputs().size() < kBatchNormRealInputNum + 1) {
MS_LOG(EXCEPTION) << "The input size of node " + bn_cnode->DebugString() + " is less than "
<< kBatchNormRealInputNum + 1;
<< kBatchNormRealInputNum + 1 << " trace: " << trace::DumpSourceLines(bn);
}
std::vector<AnfNodePtr> bn_training_reduce_inputs = {
NewValueNode(std::make_shared<Primitive>(kBNTrainingReduceOpName)), bn_cnode->input(1)};
@ -56,11 +57,12 @@ AnfNodePtr CreateBNTrainingUpdateV3(const FuncGraphPtr &func_graph, const AnfNod
MS_EXCEPTION_IF_NULL(bn_cnode);
if (bn_cnode->inputs().size() < kBatchNormRealInputNum + 1) {
MS_LOG(EXCEPTION) << "The input size of node " + bn_cnode->DebugString() + " is less than "
<< kBatchNormRealInputNum + 1;
<< kBatchNormRealInputNum + 1 << " trace: " << trace::DumpSourceLines(bn);
}
if (bn_training_reduce_outputs.size() != kBNTrainingReduceOutputNum) {
MS_LOG(EXCEPTION) << "The output size of node bn_training_reduce must be " << kBNTrainingReduceOutputNum
<< ", but it is " << bn_training_reduce_outputs.size();
<< ", but it is " << bn_training_reduce_outputs.size()
<< " trace: " << trace::DumpSourceLines(bn);
}
std::vector<AnfNodePtr> bn_training_update_v3_inputs = {
NewValueNode(std::make_shared<Primitive>(kBNTrainingUpdateV3OpName)),
@ -76,7 +78,7 @@ AnfNodePtr CreateBNTrainingUpdateV3(const FuncGraphPtr &func_graph, const AnfNod
MS_EXCEPTION_IF_NULL(bn_abstract_tuple);
if (bn_abstract_tuple->elements().size() != kBatchNormOutputNum) {
MS_LOG(EXCEPTION) << "The abstract size of node bn must be " << kBatchNormOutputNum << ", but it is "
<< bn_abstract_tuple->elements().size();
<< bn_abstract_tuple->elements().size() << " trace: " << trace::DumpSourceLines(bn);
}
bn_training_update_v3->set_abstract(bn->abstract());
bn_training_update_v3->set_scope(bn->scope());

3
mindspore/ccsrc/backend/optimizer/ascend/ir_fission/split_fission.cc
View File

@ -17,6 +17,7 @@
#include <memory>
#include <vector>
#include "backend/session/anf_runtime_algorithm.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -35,7 +36,7 @@ CNodePtr CreateBaseSplitVNode(const FuncGraphPtr &func_graph, const CNodePtr &or
MS_EXCEPTION_IF_NULL(origin_cnode);
if (origin_cnode->inputs().size() < kSplitInputNum) {
MS_LOG(EXCEPTION) << "The input number of split: " << origin_cnode->DebugString() << " should be "
<< kSplitInputNum - 1;
<< kSplitInputNum - 1 << " trace: " << trace::DumpSourceLines(origin_cnode);
}
return CreateSplitVNode(func_graph, origin_cnode->input(1));
}

4
mindspore/ccsrc/backend/optimizer/ascend/ir_fission/transdata_split.cc
View File

@ -18,6 +18,7 @@
#include "backend/optimizer/ascend/ascend_helper.h"
#include "backend/session/anf_runtime_algorithm.h"
#include "debug/anf_ir_dump.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -99,7 +100,8 @@ bool TransDataSplit::DoSplit(const FuncGraphPtr &func_graph, const AnfNodePtr &n
manager->AddFuncGraph(func_graph);
if (!manager->Replace(node, new_replace_node)) {
MS_LOG(EXCEPTION) << "Manager replace node failed";
MS_LOG(EXCEPTION) << "Manager replace node failed"
<< " trace: " << trace::DumpSourceLines(node);
}
MS_LOG(INFO) << "Transdata node:" << cnode->DebugString() << "split success.";
return true;

12
mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/adam_apply_one_fusion.cc
View File

@ -16,6 +16,7 @@
#include "backend/optimizer/ascend/ir_fusion/adam_apply_one_fusion.h"
#include "backend/optimizer/common/helper.h"
#include "backend/session/anf_runtime_algorithm.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
const BaseRef AdamApplyOneFusion::DefinePattern() const {
@ -218,7 +219,8 @@ const AnfNodePtr AdamApplyOneFusion::Process(const FuncGraphPtr &func_graph, con
if (AnfAlgo::CheckPrimitiveType(node, prim::kPrimDepend)) {
auto iter_sub0 = (*equiv).find(sub0_var_);
if (iter_sub0 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the sub0 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the sub0 var after matched."
<< " trace: " << trace::DumpSourceLines(node);
}
sub0 = utils::cast<AnfNodePtr>(iter_sub0->second);
}
@ -233,11 +235,13 @@ const AnfNodePtr AdamApplyOneFusion::Process(const FuncGraphPtr &func_graph, con
AbstractBasePtrList new_node_abstract_list;
auto iter_add0 = (*equiv).find(add0_var_);
if (iter_add0 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add0 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add0 var after matched."
<< " trace: " << trace::DumpSourceLines(node);
}
auto iter_add1 = (*equiv).find(add1_var_);
if (iter_add1 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched."
<< " trace: " << trace::DumpSourceLines(node);
}
auto add0 = utils::cast<AnfNodePtr>(iter_add0->second);
MS_EXCEPTION_IF_NULL(add0);
@ -253,7 +257,7 @@ const AnfNodePtr AdamApplyOneFusion::Process(const FuncGraphPtr &func_graph, con
CreateMultipleOutputsOfAnfNode(func_graph, new_node, kAdamApplyOneOutputNum, &new_node_outputs);
if (new_node_outputs.size() != kAdamApplyOneOutputNum) {
MS_LOG(EXCEPTION) << "The output size of node " << new_node->DebugString() << " should be "
<< kAdamApplyOneOutputNum;
<< kAdamApplyOneOutputNum << " trace: " << trace::DumpSourceLines(node);
}
auto manager = func_graph->manager();
MS_EXCEPTION_IF_NULL(manager);

10
mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/adam_apply_one_with_decay_rule.cc
View File

@ -21,6 +21,7 @@
#include "backend/session/anf_runtime_algorithm.h"
#include "ir/primitive.h"
#include "backend/optimizer/common/helper.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -281,7 +282,8 @@ const AnfNodePtr AdamApplyOneWithDecayRule::Process(const FuncGraphPtr &graph, c
if (AnfAlgo::CheckPrimitiveType(node, prim::kPrimDepend)) {
auto iter_sub0 = (*equiv).find(sub0_var_);
if (iter_sub0 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the sub0 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the sub0 var after matched."
<< " trace: " << trace::DumpSourceLines(node);
}
sub0 = utils::cast<AnfNodePtr>(iter_sub0->second);
}
@ -296,11 +298,13 @@ const AnfNodePtr AdamApplyOneWithDecayRule::Process(const FuncGraphPtr &graph, c
auto iter_add0 = (*equiv).find(add0_var_);
if (iter_add0 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add0 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add0 var after matched."
<< " trace: " << trace::DumpSourceLines(node);
}
auto iter_add1 = (*equiv).find(add1_var_);
if (iter_add1 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched."
<< " trace: " << trace::DumpSourceLines(node);
}
auto add0 = utils::cast<AnfNodePtr>(iter_add0->second);
MS_EXCEPTION_IF_NULL(add0);

4
mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/add_input_to_output.cc
View File

@ -19,6 +19,7 @@
#include "backend/optimizer/ascend/ir_fusion/input_to_output_registry.h"
#include "backend/session/anf_runtime_algorithm.h"
#include "backend/kernel_compiler/oplib/oplib.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -107,7 +108,8 @@ const AnfNodePtr AddInputToOutput::Process(const FuncGraphPtr &func_graph, const
MS_EXCEPTION_IF_NULL(new_abstract_tuple);
CreateMultipleOutputsOfAnfNode(func_graph, cnode, new_abstract_tuple->size(), &new_outputs);
if (new_outputs.size() != new_abstract_tuple->size()) {
MS_LOG(EXCEPTION) << "Failed to create outputs of " << cnode->DebugString();
MS_LOG(EXCEPTION) << "Failed to create outputs of " << cnode->DebugString()
<< " trace: " << trace::DumpSourceLines(node);
}
return new_outputs[0];
}

5
mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/confusion_mul_grad_fusion.cc
View File

@ -24,7 +24,7 @@
#include "utils/utils.h"
#include "abstract/abstract_value.h"
#include "backend/optimizer/common/helper.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
namespace {
@ -58,7 +58,8 @@ AnfNodePtr GetMul0(const FuncGraphPtr &graph, const AnfNodePtr &input2, const An
auto manager = graph->manager();
MS_EXCEPTION_IF_NULL(manager);
if (manager->node_users().find(input2) == manager->node_users().end()) {
MS_LOG(EXCEPTION) << "node has no output in manager";
MS_LOG(EXCEPTION) << "node has no output in manager"
<< " trace: " << trace::DumpSourceLines(input2);
}
AnfNodePtr mul0 = nullptr;

29
mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/fused_batch_norm_fusion.cc
View File

@ -19,6 +19,7 @@
#include "backend/optimizer/common/helper.h"
#include "backend/session/anf_runtime_algorithm.h"
#include "utils/utils.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
@ -41,7 +42,8 @@ void GetBNOutput(const FuncGraphPtr &func_graph, const AnfNodePtr &bn, std::vect
auto manager = func_graph->manager();
MS_EXCEPTION_IF_NULL(manager);
if (manager->node_users().find(bn) == manager->node_users().end()) {
MS_LOG(EXCEPTION) << "The bn node " << bn->DebugString() << " should has some outputs";
MS_LOG(EXCEPTION) << "The bn node " << bn->DebugString() << " should has some outputs"
<< " trace: " << trace::DumpSourceLines(bn);
}
for (const auto &node_index : manager->node_users()[bn]) {
const AnfNodePtr &output = node_index.first;
@ -113,7 +115,8 @@ AnfNodePtr FusedBatchNormFusion::CreateBNTrainingReduce(const FuncGraphPtr &func
// Set input to create node
auto iter_data_input0 = (*equiv).find(data_input0_var_);
if (iter_data_input0 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input0 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input0 var after matched."
<< " trace: " << trace::DumpSourceLines(node);
}
std::vector<AnfNodePtr> bn_training_reduce_inputs = {
NewValueNode(std::make_shared<Primitive>(kBNTrainingReduceOpName)),
@ -124,13 +127,15 @@ AnfNodePtr FusedBatchNormFusion::CreateBNTrainingReduce(const FuncGraphPtr &func
// Set abstract
auto iter_data_input1 = (*equiv).find(data_input1_var_);
if (iter_data_input1 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input1 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input1 var after matched."
<< " trace: " << trace::DumpSourceLines(node);
}
auto data_input1 = utils::cast<AnfNodePtr>(iter_data_input1->second);
MS_EXCEPTION_IF_NULL(data_input1);
auto iter_data_input2 = (*equiv).find(data_input2_var_);
if (iter_data_input2 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input2 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input2 var after matched."
<< " trace: " << trace::DumpSourceLines(node);
}
auto data_input2 = utils::cast<AnfNodePtr>(iter_data_input2->second);
MS_EXCEPTION_IF_NULL(data_input2);
@ -190,17 +195,19 @@ void FusedBatchNormFusion::GetBNTrainingUpdateAbstractList(const EquivPtr &equiv
MS_EXCEPTION_IF_NULL(bn_abstract_tuple);
if (bn_abstract_tuple->elements().size() < kBnOutputNum) {
MS_LOG(EXCEPTION) << "The abstract size of node bn must not be less than " << kBnOutputNum << ", but it is "
<< bn_abstract_tuple->elements().size();
<< bn_abstract_tuple->elements().size() << " trace: " << trace::DumpSourceLines(bn);
}
auto iter_variable_input0 = (*equiv).find(variable_input0_var_);
if (iter_variable_input0 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the variable_input0 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the variable_input0 var after matched."
<< " trace: " << trace::DumpSourceLines(bn);
}
auto variable_input0 = utils::cast<AnfNodePtr>(iter_variable_input0->second);
MS_EXCEPTION_IF_NULL(variable_input0);
auto iter_variable_input1 = (*equiv).find(variable_input1_var_);
if (iter_variable_input1 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the variable_input1 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the variable_input1 var after matched."
<< " trace: " << trace::DumpSourceLines(bn);
}
auto variable_input1 = utils::cast<AnfNodePtr>(iter_variable_input1->second);
MS_EXCEPTION_IF_NULL(variable_input1);
@ -222,7 +229,8 @@ AnfNodePtr FusedBatchNormFusion::CreateBNTrainingUpdate(
// Set abstract
auto iter_batch_norm = (*equiv).find(batch_norm_var_);
if (iter_batch_norm == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the batch_norm var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the batch_norm var after matched."
<< " trace: " << trace::DumpSourceLines(node);
}
AnfNodePtr bn = utils::cast<AnfNodePtr>(iter_batch_norm->second);
MS_EXCEPTION_IF_NULL(bn);
@ -260,12 +268,13 @@ const AnfNodePtr FusedBatchNormFusion::Process(const FuncGraphPtr &func_graph, c
&bn_training_update_outputs);
if (bn_training_update_outputs.size() < kBNTrainingUpdateOutputNum) {
MS_LOG(EXCEPTION) << "The output size of node bn must be " << kBNTrainingUpdateOutputNum << ", but it is "
<< bn_training_update_outputs.size();
<< bn_training_update_outputs.size() << " trace: " << trace::DumpSourceLines(node);
}
// Replace old bn outputs with new outputs
auto iter_batch_norm = (*equiv).find(batch_norm_var_);
if (iter_batch_norm == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the batch_norm var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the batch_norm var after matched."
<< " trace: " << trace::DumpSourceLines(node);
}
AnfNodePtr bn = utils::cast<AnfNodePtr>(iter_batch_norm->second);
std::vector<AnfNodePtr> bn_outputs;

5
mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_mv_with_decay_rule.cc
View File

@ -17,7 +17,7 @@
#include <utility>
#include "backend/session/anf_runtime_algorithm.h"
#include "frontend/optimizer/opt.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
AnfNodePtr LambNextMVWithDecayRule::GetLambNextMVWithDecayOutput(const FuncGraphPtr &func_graph,
@ -97,7 +97,8 @@ const AnfNodePtr LambNextMVWithDecayRule::Process(const FuncGraphPtr &func_graph
auto manager = func_graph->manager();
MS_EXCEPTION_IF_NULL(manager);
if (manager->node_users().find(mul4) == manager->node_users().end()) {
MS_LOG(EXCEPTION) << "The Mul4 should be used by at least another node input";
MS_LOG(EXCEPTION) << "The Mul4 should be used by at least another node input"
<< " trace: " << trace::DumpSourceLines(node);
}
AnfNodeIndexSet mul4_outputs = manager->node_users()[mul4];
auto iter = std::find_if(mul4_outputs.begin(), mul4_outputs.end(),

26
mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.cc
View File

@ -22,7 +22,7 @@
#include "backend/session/anf_runtime_algorithm.h"
#include "frontend/optimizer/opt.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
namespace {
@ -31,27 +31,31 @@ std::tuple<AnfNodePtr, AnfNodePtr, AnfNodePtr, AnfNodePtr> GetSharedNodes(const
auto add3 = node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(add3);
if (add3->inputs().size() < kAddInputNum) {
MS_LOG(EXCEPTION) << "The input size of Add3 is less than " << kAddInputNum;
MS_LOG(EXCEPTION) << "The input size of Add3 is less than " << kAddInputNum
<< " trace: " << trace::DumpSourceLines(node);
}
auto real_div2_anf = add3->input(1);
MS_EXCEPTION_IF_NULL(real_div2_anf);
auto real_div2 = real_div2_anf->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(real_div2);
if (real_div2->inputs().size() < kRealDivInputNum) {
MS_LOG(EXCEPTION) << "The input size of RealDiv2 is less than " << kRealDivInputNum;
MS_LOG(EXCEPTION) << "The input size of RealDiv2 is less than " << kRealDivInputNum
<< " trace: " << trace::DumpSourceLines(node);
}
auto sqrt0_anf = real_div2->input(2);
MS_EXCEPTION_IF_NULL(sqrt0_anf);
auto sqrt0 = sqrt0_anf->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(sqrt0);
if (sqrt0->inputs().size() < kRsqrtInputNum) {
MS_LOG(EXCEPTION) << "The input size of Sqrt0 is less than " << kSqrtInputNum;
MS_LOG(EXCEPTION) << "The input size of Sqrt0 is less than " << kSqrtInputNum
<< " trace: " << trace::DumpSourceLines(node);
}
auto add2_anf = sqrt0->input(1);
MS_EXCEPTION_IF_NULL(add2_anf);
auto add2 = add2_anf->cast<CNodePtr>();
if (add2->inputs().size() < kAddInputNum) {
MS_LOG(EXCEPTION) << "The input size of Add2 is less than " << kAddInputNum;
MS_LOG(EXCEPTION) << "The input size of Add2 is less than " << kAddInputNum
<< " trace: " << trace::DumpSourceLines(node);
}
return std::make_tuple(add3->input(2), real_div2->input(1), add2->input(1), add2->input(2));
}
@ -101,10 +105,12 @@ std::tuple<AnfNodePtr, AnfNodePtr> GetAdd0Add1Nodes(const AnfNodePtr &real_div0_
MS_EXCEPTION_IF_NULL(real_div0);
MS_EXCEPTION_IF_NULL(real_div1);
if (real_div0->inputs().size() != kRealDivInputNum) {
MS_LOG(EXCEPTION) << "RealDiv0 has wrong input size";
MS_LOG(EXCEPTION) << "RealDiv0 has wrong input size"
<< " trace: " << trace::DumpSourceLines(real_div0_anf);
}
if (real_div1->inputs().size() != kRealDivInputNum) {
MS_LOG(EXCEPTION) << "RealDiv1 has wrong input size";
MS_LOG(EXCEPTION) << "RealDiv1 has wrong input size"
<< " trace: " << trace::DumpSourceLines(real_div1_anf);
}
return std::make_tuple(real_div0->input(1), real_div1->input(1));
}
@ -165,7 +171,8 @@ const AnfNodePtr LambNextMVWithDecayV1Rule::Process(const FuncGraphPtr &func_gra
auto manager = func_graph->manager();
MS_EXCEPTION_IF_NULL(manager);
if (manager->node_users().find(mul4) == manager->node_users().end()) {
MS_LOG(EXCEPTION) << "The Mul4 should be used by at least another node input";
MS_LOG(EXCEPTION) << "The Mul4 should be used by at least another node input"
<< " trace: " << trace::DumpSourceLines(node);
}
AnfNodeIndexSet mul4_output_node_index_set = manager->node_users()[mul4];
auto iter = std::find_if(
@ -195,7 +202,8 @@ const AnfNodePtr LambNextMVWithDecayV1Rule::Process(const FuncGraphPtr &func_gra
std::vector<AnfNodePtr> fusion_node_outputs;
CreateMultipleOutputsOfAnfNode(func_graph, fusion_node, kLambNextMVWithDecayV1OutputNum, &fusion_node_outputs);
if (fusion_node_outputs.size() != kLambNextMVWithDecayV1OutputNum) {
MS_LOG(EXCEPTION) << "create multiple outputs for fusion node fail!";
MS_LOG(EXCEPTION) << "create multiple outputs for fusion node fail!"
<< " trace: " << trace::DumpSourceLines(node);
}
(void)manager->Replace(add0, fusion_node_outputs[1]);

5
mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_right_rule.cc
View File

@ -16,7 +16,7 @@
#include "backend/optimizer/ascend/ir_fusion/lamb_next_right_rule.h"
#include <vector>
#include "backend/optimizer/common/helper.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
AnfNodePtr LambNextRightRule::CreateLambNextRightNode(const FuncGraphPtr &func_graph, const EquivPtr &equiv) const {
@ -69,7 +69,8 @@ const AnfNodePtr LambNextRightRule::Process(const FuncGraphPtr &func_graph, cons
// Set abstract of new node
auto iter_add1 = (*equiv).find(add1_var_);
if (iter_add1 == (*equiv).end()) {
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched.";
MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched."
<< " trace: " << trace::DumpSourceLines(node);
}
auto add1 = utils::cast<AnfNodePtr>(iter_add1->second);
MS_EXCEPTION_IF_NULL(add1);

10
mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.cc
View File

@ -17,7 +17,7 @@
#include <memory>
#include <vector>
#include "backend/session/anf_runtime_algorithm.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
using common::SafeCStr;
@ -36,7 +36,7 @@ void GetOutputCastNodes(const FuncGraphPtr &func_graph, const AnfNodePtr &node,
MS_EXCEPTION_IF_NULL(output_cnode);
if (AnfAlgo::GetCNodeName(output_cnode) != prim::kPrimTupleGetItem->name()) {
MS_LOG(EXCEPTION) << "The output of node " << node->DebugString() << " should be "
<< prim::kPrimTupleGetItem->name();
<< prim::kPrimTupleGetItem->name() << " trace: " << trace::DumpSourceLines(node);
}
if (manager->node_users().find(output) == manager->node_users().end() ||
manager->node_users()[output].size() != 1) {
@ -149,11 +149,13 @@ const AnfNodePtr LayerNormBetaGammaBackpropFusion::Process(const FuncGraphPtr &f
MS_EXCEPTION_IF_NULL(cast_nodes[0]);
MS_EXCEPTION_IF_NULL(cast_nodes[1]);
if (cast_nodes[0]->inputs().size() != kCastInputNum) {
MS_LOG(EXCEPTION) << "The cast0 " << cast_nodes[0]->DebugString() << " input size should be " << kCastInputNum;
MS_LOG(EXCEPTION) << "The cast0 " << cast_nodes[0]->DebugString() << " input size should be " << kCastInputNum
<< " trace: " << trace::DumpSourceLines(node);
}
(void)manager->Replace(cast_nodes[0], cast_nodes[0]->input(1));
if (cast_nodes[1]->inputs().size() != kCastInputNum) {
MS_LOG(EXCEPTION) << "The cast1 " << cast_nodes[1]->DebugString() << " input size should be " << kCastInputNum;
MS_LOG(EXCEPTION) << "The cast1 " << cast_nodes[1]->DebugString() << " input size should be " << kCastInputNum
<< " trace: " << trace::DumpSourceLines(node);
}
(void)manager->Replace(cast_nodes[1], cast_nodes[1]->input(1));
return nullptr;

5
mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/matmul_biasadd_fusion.cc
View File

@ -18,7 +18,7 @@
#include "backend/optimizer/common/helper.h"
#include "backend/session/anf_runtime_algorithm.h"
#include "utils/utils.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
const BaseRef MatmulBiasaddFusion::DefinePattern() const {
@ -43,7 +43,8 @@ const AnfNodePtr MatmulBiasaddFusion::Process(const FuncGraphPtr &graph, const A
auto matmul = GetAnfNodeByVar(equiv, matmul_var_);
if (matmul == nullptr || !matmul->isa<CNode>()) {
MS_LOG(EXCEPTION) << "Get CNode MatMul failed!";
MS_LOG(EXCEPTION) << "Get CNode MatMul failed!"
<< " trace: " << trace::DumpSourceLines(node);
}
AnfAlgo::CopyNodeAttrs(matmul, new_node);
return new_node;

17
mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/square_sum_fusion.cc
View File

@ -25,7 +25,7 @@
#include "base/core_ops.h"
#include "backend/optimizer/common/helper.h"
#include "runtime/device/kernel_info.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace opt {
namespace {
@ -34,7 +34,8 @@ CNodePtr GenerateSquareSumV1(const FuncGraphPtr &graph, const CNodePtr &square,
MS_EXCEPTION_IF_NULL(square);
MS_EXCEPTION_IF_NULL(sum);
if (square->inputs().size() != kSquareNodeInputNum) {
MS_LOG(EXCEPTION) << "Square node has wrong input size";
MS_LOG(EXCEPTION) << "Square node has wrong input size"
<< " trace: " << trace::DumpSourceLines(square);
}
auto prim = std::make_shared<Primitive>(kSquareSumV1OpName);
MS_EXCEPTION_IF_NULL(prim);
@ -60,7 +61,8 @@ CNodePtr GenerateSquareSumV2(const FuncGraphPtr &graph, const CNodePtr &square,
MS_EXCEPTION_IF_NULL(square);
MS_EXCEPTION_IF_NULL(sum);
if (square->inputs().size() != kSquareNodeInputNum) {
MS_LOG(EXCEPTION) << "Square node has wrong input size";
MS_LOG(EXCEPTION) << "Square node has wrong input size"
<< " trace: " << trace::DumpSourceLines(square);
}
auto prim = std::make_shared<Primitive>(kSquareSumV2OpName);
MS_EXCEPTION_IF_NULL(prim);
@ -83,7 +85,8 @@ std::tuple<CNodePtr, AnfNodePtr, CNodePtr> GetPrevNodes(const AnfNodePtr &node)
auto sum = node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(sum);
if (sum->inputs().size() != kSumNodeInputNum) {
MS_LOG(EXCEPTION) << "ReduceSumD node has wrong input size";
MS_LOG(EXCEPTION) << "ReduceSumD node has wrong input size"
<< " trace: " << trace::DumpSourceLines(sum);
}
auto square_anf = sum->input(1);
MS_EXCEPTION_IF_NULL(square_anf);
@ -111,7 +114,8 @@ const AnfNodePtr SquareSumFusion::Process(const FuncGraphPtr &graph, const AnfNo
auto manager = graph->manager();
MS_EXCEPTION_IF_NULL(manager);
if (manager->node_users().find(square_anf) == manager->node_users().end()) {
MS_LOG(EXCEPTION) << "Square node has no output in NodeUsersMap";
MS_LOG(EXCEPTION) << "Square node has no output in NodeUsersMap"
<< " trace: " << trace::DumpSourceLines(node);
}
AnfNodePtr ret_node = nullptr;
if (manager->node_users()[square_anf].size() == 1) {
@ -122,7 +126,8 @@ const AnfNodePtr SquareSumFusion::Process(const FuncGraphPtr &graph, const AnfNo
std::vector<AnfNodePtr> square_sumv2_outputs;
CreateMultipleOutputsOfAnfNode(graph, square_sumv2, kSquareSumv2OutputNum, &square_sumv2_outputs);
if (square_sumv2_outputs.size() != kSquareSumv2OutputNum) {
MS_LOG(EXCEPTION) << "make SquareSumV2 outputs fail";
MS_LOG(EXCEPTION) << "make SquareSumV2 outputs fail"
<< " trace: " << trace::DumpSourceLines(square_sumv2);
}
(void)manager->Replace(square, square_sumv2_outputs[1]);
ret_node = square_sumv2_outputs[0];

138
mindspore/ccsrc/backend/session/anf_runtime_algorithm.cc
View File

@ -31,6 +31,7 @@
#include "common/trans.h"
#include "abstract/param_validator.h"
#include "pipeline/jit/static_analysis/static_analysis.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace session {
@ -164,7 +165,7 @@ KernelWithIndex AnfRuntimeAlgorithm::VisitKernelWithReturnType(const AnfNodePtr
}
if (opt::IsNopNode(cnode) && visit_nop_node) {
if (cnode->size() != kNopNodeInputSize) {
MS_LOG(EXCEPTION) << "Invalid nop node " << cnode->DebugString();
MS_LOG(EXCEPTION) << "Invalid nop node " << cnode->DebugString() << " trace: " << trace::DumpSourceLines(cnode);
}
return VisitKernelWithReturnType(cnode->input(kNopNodeRealInputIndex), 0, visit_nop_node, return_types);
}
@ -250,7 +251,7 @@ std::string AnfRuntimeAlgorithm::GetCNodeName(const AnfNodePtr &node) {
}
return func_graph->ToString();
}
MS_LOG(EXCEPTION) << "Unknown anf node type " << node->DebugString();
MS_LOG(EXCEPTION) << "Unknown anf node type " << node->DebugString() << " trace: " << trace::DumpSourceLines(node);
}
std::string AnfRuntimeAlgorithm::GetNodeDebugString(const AnfNodePtr &node) {
@ -261,7 +262,8 @@ std::string AnfRuntimeAlgorithm::GetNodeDebugString(const AnfNodePtr &node) {
void AnfRuntimeAlgorithm::SetNodeAttr(const std::string &key, const ValuePtr &value, const AnfNodePtr &node) {
MS_EXCEPTION_IF_NULL(node);
if (!node->isa<CNode>()) {
MS_LOG(EXCEPTION) << "Only cnode has attr, but this anf is " << node->DebugString();
MS_LOG(EXCEPTION) << "Only cnode has attr, but this anf is " << node->DebugString()
<< " trace: " << trace::DumpSourceLines(node);
}
// single op cnode.
auto primitive = AnfAlgo::GetCNodePrimitive(node);
@ -285,7 +287,7 @@ void AnfRuntimeAlgorithm::CopyNodeAttr(const std::string &old_key, const std::st
MS_EXCEPTION_IF_NULL(to);
if (!from->isa<CNode>() || !to->isa<CNode>()) {
MS_LOG(EXCEPTION) << "Only cnode has attr, but this from_anf is " << from->DebugString() << " ,to_node is "
<< to->DebugString();
<< to->DebugString() << " trace: " << trace::DumpSourceLines(from);
}
auto from_primitive = AnfAlgo::GetCNodePrimitive(from);
MS_EXCEPTION_IF_NULL(from_primitive);
@ -299,7 +301,7 @@ void AnfRuntimeAlgorithm::CopyNodeAttrs(const AnfNodePtr &from, const AnfNodePtr
MS_EXCEPTION_IF_NULL(to);
if (!from->isa<CNode>() || !to->isa<CNode>()) {
MS_LOG(EXCEPTION) << "Only cnode has attr, but this from_anf is " << from->DebugString() << ",to_node is "
<< from->DebugString();
<< from->DebugString() << " trace: " << trace::DumpSourceLines(from);
}
auto from_primitive = AnfAlgo::GetCNodePrimitive(from);
MS_EXCEPTION_IF_NULL(from_primitive);
@ -311,7 +313,8 @@ void AnfRuntimeAlgorithm::CopyNodeAttrs(const AnfNodePtr &from, const AnfNodePtr
void AnfRuntimeAlgorithm::EraseNodeAttr(const std::string &key, const AnfNodePtr node) {
MS_EXCEPTION_IF_NULL(node);
if (!node->isa<CNode>()) {
MS_LOG(EXCEPTION) << "Only cnode has attr, but this anf is " << node->DebugString();
MS_LOG(EXCEPTION) << "Only cnode has attr, but this anf is " << node->DebugString()
<< " trace: " << trace::DumpSourceLines(node);
}
// single op cnode.
auto primitive = AnfAlgo::GetCNodePrimitive(node);
@ -345,13 +348,15 @@ bool AnfRuntimeAlgorithm::HasNodeAttr(const std::string &key, const CNodePtr &no
size_t AnfRuntimeAlgorithm::GetInputTensorNum(const AnfNodePtr &node) {
MS_EXCEPTION_IF_NULL(node);
if (!node->isa<CNode>()) {
MS_LOG(EXCEPTION) << "Only cnode has real input, but this anf is " << node->DebugString();
MS_LOG(EXCEPTION) << "Only cnode has real input, but this anf is " << node->DebugString()
<< " trace: " << trace::DumpSourceLines(node);
}
auto cnode = node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(cnode);
size_t input_num = cnode->inputs().size();
if (input_num == 0) {
MS_LOG(EXCEPTION) << "Cnode inputs size can't be zero";
MS_LOG(EXCEPTION) << "Cnode inputs size can't be zero"
<< " trace: " << trace::DumpSourceLines(node);
}
// exclude intputs[0],which is value_node storing attr,inputs left are real input
return input_num - 1;
@ -380,7 +385,8 @@ std::vector<std::string> AnfRuntimeAlgorithm::GetAllOutputFormats(const AnfNodeP
MS_EXCEPTION_IF_NULL(node);
if (!AnfAlgo::IsRealKernel(node)) {
MS_LOG(EXCEPTION) << "Not real kernel:"
<< "#node [" << node->DebugString() << "]";
<< "#node [" << node->DebugString() << "]"
<< " trace: " << trace::DumpSourceLines(node);
}
auto kernel_info = static_cast<device::KernelInfo *>(node->kernel_info());
MS_EXCEPTION_IF_NULL(kernel_info);
@ -394,7 +400,8 @@ std::vector<std::string> AnfRuntimeAlgorithm::GetAllInputFormats(const AnfNodePt
MS_EXCEPTION_IF_NULL(node);
if (!AnfAlgo::IsRealKernel(node)) {
MS_LOG(EXCEPTION) << "Not real kernel:"
<< "#node [" << node->DebugString() << "]";
<< "#node [" << node->DebugString() << "]"
<< " trace: " << trace::DumpSourceLines(node);
}
auto kernel_info = static_cast<device::KernelInfo *>(node->kernel_info());
MS_EXCEPTION_IF_NULL(kernel_info);
@ -408,7 +415,8 @@ std::string AnfRuntimeAlgorithm::GetOriginDataFormat(const AnfNodePtr &node) {
MS_EXCEPTION_IF_NULL(node);
if (!AnfAlgo::IsRealKernel(node)) {
MS_LOG(EXCEPTION) << "Not real kernel:"
<< "#node [" << node->DebugString() << "]";
<< "#node [" << node->DebugString() << "]"
<< " trace: " << trace::DumpSourceLines(node);
}
auto kernel_info = static_cast<device::KernelInfo *>(node->kernel_info());
MS_EXCEPTION_IF_NULL(kernel_info);
@ -423,7 +431,8 @@ std::string AnfRuntimeAlgorithm::GetOutputFormat(const AnfNodePtr &node, size_t
if (output_idx > GetOutputTensorNum(node)) {
MS_LOG(EXCEPTION) << "Output index:" << output_idx
<< " is out of the node output range :" << GetOutputTensorNum(node) << " #node ["
<< node->DebugString() << "]";
<< node->DebugString() << "]"
<< " trace: " << trace::DumpSourceLines(node);
}
if (!AnfAlgo::IsRealKernel(node)) {
return AnfAlgo::GetPrevNodeOutputFormat(node, output_idx);
@ -435,7 +444,8 @@ std::string AnfRuntimeAlgorithm::GetOutputFormat(const AnfNodePtr &node, size_t
auto format = build_info->GetOutputFormat(output_idx);
if (format == kernel::KernelBuildInfo::kInvalidFormat) {
MS_LOG(EXCEPTION) << "Node [" << node->DebugString() << "]"
<< " has a invalid output format";
<< " has a invalid output format"
<< " trace: " << trace::DumpSourceLines(node);
}
return format;
}
@ -445,7 +455,8 @@ std::string AnfRuntimeAlgorithm::GetInputFormat(const AnfNodePtr &node, size_t i
if (input_idx > GetInputTensorNum(node)) {
MS_LOG(EXCEPTION) << "Input index :" << input_idx
<< " is out of the number node Input range :" << GetInputTensorNum(node) << "#node ["
<< node->DebugString() << "]";
<< node->DebugString() << "]"
<< " trace: " << trace::DumpSourceLines(node);
}
if (!IsRealKernel(node)) {
return GetPrevNodeOutputFormat(node, input_idx);
@ -457,7 +468,8 @@ std::string AnfRuntimeAlgorithm::GetInputFormat(const AnfNodePtr &node, size_t i
auto format = build_info->GetInputFormat(input_idx);
if (format == kernel::KernelBuildInfo::kInvalidFormat) {
MS_LOG(EXCEPTION) << "Node [" << node->DebugString() << "]"
<< " has a invalid input format";
<< " has a invalid input format"
<< " trace: " << trace::DumpSourceLines(node);
}
return format;
}
@ -465,7 +477,8 @@ std::string AnfRuntimeAlgorithm::GetInputFormat(const AnfNodePtr &node, size_t i
KernelWithIndex AnfRuntimeAlgorithm::GetPrevNodeOutput(const AnfNodePtr &anf_node, size_t input_idx) {
MS_EXCEPTION_IF_NULL(anf_node);
if (!anf_node->isa<CNode>()) {
MS_LOG(EXCEPTION) << anf_node->DebugString() << "anf_node is not CNode.";
MS_LOG(EXCEPTION) << anf_node->DebugString() << "anf_node is not CNode."
<< " trace: " << trace::DumpSourceLines(anf_node);
}
auto input_node = AnfAlgo::GetInputNode(anf_node->cast<CNodePtr>(), input_idx);
MS_EXCEPTION_IF_NULL(input_node);
@ -491,13 +504,15 @@ std::vector<size_t> AnfRuntimeAlgorithm::GetOutputInferShape(const AnfNodePtr &n
return TransShapeToSizet(base_shape->cast<abstract::ShapePtr>());
}
MS_LOG(EXCEPTION) << "The node " << node->DebugString() << "is a single output node but got index [" << output_idx
<< ".";
<< "."
<< " trace: " << trace::DumpSourceLines(node);
} else if (base_shape->isa<abstract::TupleShape>()) {
auto tuple_shape = base_shape->cast<abstract::TupleShapePtr>();
MS_EXCEPTION_IF_NULL(tuple_shape);
if (output_idx >= tuple_shape->size()) {
MS_LOG(EXCEPTION) << "Output index " << output_idx << "is larger than output number " << tuple_shape->size()
<< " node:" << node->DebugString() << ".";
<< " node:" << node->DebugString() << "."
<< " trace: " << trace::DumpSourceLines(node);
}
auto b_shp = (*tuple_shape)[output_idx];
if (b_shp->isa<abstract::Shape>()) {
@ -507,13 +522,15 @@ std::vector<size_t> AnfRuntimeAlgorithm::GetOutputInferShape(const AnfNodePtr &n
} else {
MS_LOG(EXCEPTION) << "The output type of ApplyKernel index:" << output_idx
<< " should be a NoShape , ArrayShape or a TupleShape, but it is " << base_shape->ToString()
<< "node :" << node->DebugString() << ".";
<< "node :" << node->DebugString() << "."
<< " trace: " << trace::DumpSourceLines(node);
}
} else if (base_shape->isa<abstract::NoShape>()) {
return std::vector<size_t>();
}
MS_LOG(EXCEPTION) << "The output type of ApplyKernel should be a NoShape , ArrayShape or a TupleShape, but it is "
<< base_shape->ToString() << " node : " << node->DebugString();
<< base_shape->ToString() << " node : " << node->DebugString()
<< " trace: " << trace::DumpSourceLines(node);
}
std::vector<size_t> AnfRuntimeAlgorithm::GetPrevNodeOutputInferShape(const AnfNodePtr &node, size_t input_idx) {
@ -556,7 +573,8 @@ std::vector<Axis> AnfRuntimeAlgorithm::GetInputReshapeType(const AnfNodePtr &nod
if (input_idx > GetInputTensorNum(node)) {
MS_LOG(EXCEPTION) << "The index:" << input_idx
<< " is out of range of the node's input size : " << GetInputTensorNum(node) << "#node["
<< node->DebugString() << "]";
<< node->DebugString() << "]"
<< " trace: " << trace::DumpSourceLines(node);
}
if (!IsRealKernel(node)) {
return GetPrevNodeOutputReshapeType(node, input_idx);
@ -575,7 +593,8 @@ std::vector<Axis> AnfRuntimeAlgorithm::GetOutputReshapeType(const AnfNodePtr &no
MS_EXCEPTION_IF_NULL(node);
if (output_idx > GetOutputTensorNum(node)) {
MS_LOG(EXCEPTION) << "The index [" << output_idx << "] is out of range of the node's output size [ "
<< GetOutputTensorNum(node) << "#node[ " << node->DebugString() << "]";
<< GetOutputTensorNum(node) << "#node[ " << node->DebugString() << "]"
<< " trace: " << trace::DumpSourceLines(node);
}
if (!IsRealKernel(node)) {
return GetPrevNodeOutputReshapeType(node, output_idx);
@ -631,7 +650,8 @@ TypeId AnfRuntimeAlgorithm::GetOutputDeviceDataType(const AnfNodePtr &node, size
MS_EXCEPTION_IF_NULL(node);
if (output_idx > GetOutputTensorNum(node)) {
MS_LOG(EXCEPTION) << "The index [" << output_idx << "] is out of range of the node's output size [ "
<< GetOutputTensorNum(node) << "#node [ " << node->DebugString() << "]";
<< GetOutputTensorNum(node) << "#node [ " << node->DebugString() << "]"
<< " trace: " << trace::DumpSourceLines(node);
}
if (!IsRealKernel(node)) {
return GetPrevNodeOutputDeviceDataType(node, output_idx);
@ -643,7 +663,8 @@ TypeId AnfRuntimeAlgorithm::GetOutputDeviceDataType(const AnfNodePtr &node, size
auto dtype = build_info->GetOutputDeviceType(output_idx);
if (dtype == TypeId::kNumberTypeEnd) {
MS_LOG(EXCEPTION) << "Node [" << node->DebugString() << "]"
<< " has a invalid dtype";
<< " has a invalid dtype"
<< " trace: " << trace::DumpSourceLines(node);
}
return dtype;
}
@ -652,7 +673,8 @@ TypeId AnfRuntimeAlgorithm::GetInputDeviceDataType(const AnfNodePtr &node, size_
MS_EXCEPTION_IF_NULL(node);
if (input_idx > GetInputTensorNum(node)) {
MS_LOG(EXCEPTION) << "The index [" << input_idx << "] is out of range of the node's input size [ "
<< GetInputTensorNum(node) << "#node [ " << node->DebugString() << "]";
<< GetInputTensorNum(node) << "#node [ " << node->DebugString() << "]"
<< " trace: " << trace::DumpSourceLines(node);
}
if (!IsRealKernel(node)) {
return GetPrevNodeOutputDeviceDataType(node, 0);
@ -664,7 +686,8 @@ TypeId AnfRuntimeAlgorithm::GetInputDeviceDataType(const AnfNodePtr &node, size_
auto dtype = build_info->GetInputDeviceType(input_idx);
if (dtype == TypeId::kNumberTypeEnd) {
MS_LOG(EXCEPTION) << "Node [" << node->DebugString() << "]"
<< " has a invalid dtype";
<< " has a invalid dtype"
<< " trace: " << trace::DumpSourceLines(node);
}
return dtype;
}
@ -684,7 +707,8 @@ const DeviceAddress *AnfRuntimeAlgorithm::GetOutputAddr(const AnfNodePtr &node,
if (cnode->size() == kNopNodeInputSize) {
return AnfRuntimeAlgorithm::GetPrevNodeOutputAddr(cnode, 0);
} else {
MS_LOG(EXCEPTION) << node->DebugString() << "Invalid nop node";
MS_LOG(EXCEPTION) << node->DebugString() << "Invalid nop node"
<< " trace: " << trace::DumpSourceLines(node);
}
}
auto kernel_info = static_cast<device::KernelInfo *>(node->kernel_info());
@ -692,7 +716,8 @@ const DeviceAddress *AnfRuntimeAlgorithm::GetOutputAddr(const AnfNodePtr &node,
auto addr = kernel_info->GetOutputAddr(output_idx);
if (addr == nullptr) {
MS_LOG(EXCEPTION) << "Output_idx " << output_idx << " of node " << node->DebugString()
<< " output addr is not exist";
<< " output addr is not exist"
<< " trace: " << trace::DumpSourceLines(node);
}
return addr;
}
@ -706,7 +731,8 @@ DeviceAddressPtr AnfRuntimeAlgorithm::GetMutableOutputAddr(const AnfNodePtr &nod
if (cnode->inputs().size() == kNopNodeInputSize) {
return AnfRuntimeAlgorithm::GetPrevNodeMutableOutputAddr(cnode, 0);
} else {
MS_LOG(EXCEPTION) << node->DebugString() << "Invalid nop node.";
MS_LOG(EXCEPTION) << node->DebugString() << "Invalid nop node."
<< " trace: " << trace::DumpSourceLines(node);
}
}
// Critical path performance optimization: `KernelInfo` is unique subclass of `KernelInfoDevice`
@ -714,8 +740,8 @@ DeviceAddressPtr AnfRuntimeAlgorithm::GetMutableOutputAddr(const AnfNodePtr &nod
MS_EXCEPTION_IF_NULL(kernel_info);
auto addr = kernel_info->GetMutableOutputAddr(output_idx);
if (addr == nullptr) {
MS_LOG(EXCEPTION) << "Output_idx" << output_idx << " of node " << node->DebugString()
<< " output addr is not exist";
MS_LOG(EXCEPTION) << "Output_idx" << output_idx << " of node " << node->DebugString() << " output addr is not exist"
<< " trace: " << trace::DumpSourceLines(node);
}
return addr;
}
@ -747,7 +773,8 @@ void AnfRuntimeAlgorithm::SetOutputAddr(const DeviceAddressPtr &addr, size_t out
auto kernel_info = static_cast<device::KernelInfo *>(node->kernel_info());
MS_EXCEPTION_IF_NULL(kernel_info);
if (!kernel_info->SetOutputAddr(addr, output_idx)) {
MS_LOG(EXCEPTION) << "Node " << node->DebugString() << "set adr" << output_idx << " fail";
MS_LOG(EXCEPTION) << "Node " << node->DebugString() << "set adr" << output_idx << " fail."
<< " trace: " << trace::DumpSourceLines(node);
}
}
@ -757,7 +784,8 @@ void AnfRuntimeAlgorithm::SetWorkspaceAddr(const DeviceAddressPtr &addr, size_t
auto kernel_info = static_cast<device::KernelInfo *>(node->kernel_info());
MS_EXCEPTION_IF_NULL(kernel_info);
if (!kernel_info->SetWorkspaceAddr(addr, output_idx)) {
MS_LOG(EXCEPTION) << "Node " << node->DebugString() << "set adr" << output_idx << " fail";
MS_LOG(EXCEPTION) << "Node " << node->DebugString() << "set adr" << output_idx << " fail。"
<< " trace: " << trace::DumpSourceLines(node);
}
}
@ -769,7 +797,8 @@ DeviceAddress *AnfRuntimeAlgorithm::GetWorkspaceAddr(const AnfNodePtr &node, siz
auto addr = kernel_info->GetWorkspaceAddr(output_idx);
if (addr == nullptr) {
MS_LOG(EXCEPTION) << "Output_idx " << output_idx << " of node " << node->DebugString()
<< "] workspace addr is not exist";
<< "] workspace addr is not exist"
<< " trace: " << trace::DumpSourceLines(node);
}
return addr;
}
@ -781,7 +810,8 @@ DeviceAddressPtr AnfRuntimeAlgorithm::GetMutableWorkspaceAddr(const AnfNodePtr &
MS_EXCEPTION_IF_NULL(kernel_info);
auto addr = kernel_info->GetMutableWorkspaceAddr(index);
if (addr == nullptr) {
MS_LOG(EXCEPTION) << "Index " << index << " of node " << node->DebugString() << "] workspace addr is not exist";
MS_LOG(EXCEPTION) << "Index " << index << " of node " << node->DebugString() << "] workspace addr is not exist"
<< " trace: " << trace::DumpSourceLines(node);
}
return addr;
}
@ -791,7 +821,8 @@ void AnfRuntimeAlgorithm::SetOutputInferTypeAndShape(const std::vector<TypeId> &
const std::vector<std::vector<size_t>> &shapes, AnfNode *node) {
MS_EXCEPTION_IF_NULL(node);
if (types.size() != shapes.size()) {
MS_LOG(EXCEPTION) << "Types size " << types.size() << "should be same with shapes size " << shapes.size();
MS_LOG(EXCEPTION) << "Types size " << types.size() << "should be same with shapes size " << shapes.size()
<< " trace: " << trace::DumpSourceLines(node);
}
if (shapes.empty()) {
node->set_abstract(std::make_shared<abstract::AbstractNone>());
@ -898,7 +929,8 @@ bool AnfRuntimeAlgorithm::IsRealKernel(const AnfNodePtr &node) {
auto cnode = node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(cnode);
if (cnode->inputs().empty()) {
MS_LOG(EXCEPTION) << "Illegal null input of cnode(%s)" << node->DebugString();
MS_LOG(EXCEPTION) << "Illegal null input of cnode(%s)" << node->DebugString()
<< " trace: " << trace::DumpSourceLines(node);
}
auto input = cnode->inputs()[0];
bool is_virtual_node = IsPrimitive(input, prim::kPrimImageSummary) || IsPrimitive(input, prim::kPrimScalarSummary) ||
@ -1007,7 +1039,7 @@ AnfNodePtr AnfRuntimeAlgorithm::GetInputNode(const CNodePtr &node, size_t index)
auto get_input_index = index + 1;
if (index + 1 >= node->inputs().size()) {
MS_LOG(EXCEPTION) << "Input index size " << get_input_index << "but the node input size just"
<< node->inputs().size();
<< node->inputs().size() << " trace: " << trace::DumpSourceLines(node);
}
// input 0 is primitive node
return node->input(get_input_index);
@ -1025,7 +1057,8 @@ bool AnfRuntimeAlgorithm::IsFeatureMapOutput(const AnfNodePtr &node) {
bool AnfRuntimeAlgorithm::IsFeatureMapInput(const AnfNodePtr &node, size_t input_index) {
if (!node->isa<CNode>()) {
MS_LOG(EXCEPTION) << "Cannot input a parameter or a valuenode to charge it's input if is a feature map";
MS_LOG(EXCEPTION) << "Cannot input a parameter or a valuenode to charge it's input if is a feature map"
<< " trace: " << trace::DumpSourceLines(node);
}
auto cnode = node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(cnode);
@ -1116,7 +1149,8 @@ FuncGraphPtr AnfRuntimeAlgorithm::GetValueNodeFuncGraph(const AnfNodePtr &node)
std::vector<KernelGraphPtr> AnfRuntimeAlgorithm::GetCallSwitchKernelGraph(const CNodePtr &cnode) {
MS_EXCEPTION_IF_NULL(cnode);
if (!(AnfAlgo::CheckPrimitiveType(cnode, prim::kPrimCall) || AnfAlgo::CheckPrimitiveType(cnode, prim::kPrimSwitch))) {
MS_LOG(EXCEPTION) << "Node: " << cnode->DebugString() << "is not a call or switch node.";
MS_LOG(EXCEPTION) << "Node: " << cnode->DebugString() << "is not a call or switch node."
<< " trace: " << trace::DumpSourceLines(cnode);
}
if (AnfAlgo::CheckPrimitiveType(cnode, prim::kPrimCall)) {
auto input1 = cnode->input(kCallKernelGraphIndex);
@ -1153,7 +1187,8 @@ std::vector<KernelGraphPtr> AnfRuntimeAlgorithm::GetCallSwitchKernelGraph(const
bool AnfRuntimeAlgorithm::IsSwitchCall(const CNodePtr &call_node) {
MS_EXCEPTION_IF_NULL(call_node);
if (!CheckPrimitiveType(call_node, prim::kPrimCall)) {
MS_LOG(EXCEPTION) << "Call node should be a 'call', but is a " << call_node->DebugString();
MS_LOG(EXCEPTION) << "Call node should be a 'call', but is a " << call_node->DebugString()
<< " trace: " << trace::DumpSourceLines(call_node);
}
auto input1 = call_node->input(1);
if (input1->isa<ValueNode>()) {
@ -1161,7 +1196,8 @@ bool AnfRuntimeAlgorithm::IsSwitchCall(const CNodePtr &call_node) {
} else if (input1->isa<CNode>() && AnfAlgo::CheckPrimitiveType(input1, prim::kPrimSwitch)) {
return true;
}
MS_LOG(EXCEPTION) << "Unexpected input1 of call node,input1:" << input1->DebugString();
MS_LOG(EXCEPTION) << "Unexpected input1 of call node,input1:" << input1->DebugString()
<< " trace: " << trace::DumpSourceLines(call_node);
}
bool AnfRuntimeAlgorithm::IsScalarInput(const CNodePtr &cnode, size_t index) {
@ -1212,7 +1248,8 @@ TypeId AnfRuntimeAlgorithm::GetCNodeOutputPrecision(const AnfNodePtr &node) {
} else if (output_type_str == "float32") {
except_type = kNumberTypeFloat32;
} else {
MS_LOG(EXCEPTION) << "The fix precision must be float16 or float32, but got " << output_type_str;
MS_LOG(EXCEPTION) << "The fix precision must be float16 or float32, but got " << output_type_str
<< " trace: " << trace::DumpSourceLines(node);
}
}
@ -1221,12 +1258,14 @@ TypeId AnfRuntimeAlgorithm::GetCNodeOutputPrecision(const AnfNodePtr &node) {
TypeId AnfRuntimeAlgorithm::GetPrevNodeOutputPrecision(const AnfNodePtr &node, size_t input_idx) {
if (!node->isa<CNode>()) {
MS_LOG(EXCEPTION) << node->DebugString() << ", input node is not CNode.";
MS_LOG(EXCEPTION) << node->DebugString() << ", input node is not CNode."
<< " trace: " << trace::DumpSourceLines(node);
}
auto cnode = node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(cnode);
if (input_idx + 1 >= cnode->inputs().size()) {
MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " << GetInputTensorNum(cnode);
MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " << GetInputTensorNum(cnode)
<< " trace: " << trace::DumpSourceLines(node);
}
auto input_node = cnode->input(input_idx + 1);
MS_EXCEPTION_IF_NULL(input_node);
@ -1240,7 +1279,8 @@ TypeId AnfRuntimeAlgorithm::GetPrevNodeOutputPrecision(const AnfNodePtr &node, s
bool AnfRuntimeAlgorithm::IsCondControlKernel(const CNodePtr &node) {
MS_EXCEPTION_IF_NULL(node);
if (node->inputs().empty()) {
MS_LOG(EXCEPTION) << "Illegal null input of cnode.";
MS_LOG(EXCEPTION) << "Illegal null input of cnode."
<< " trace: " << trace::DumpSourceLines(node);
}
auto input = node->input(kAnfPrimitiveIndex);
return IsPrimitive(input, prim::kPrimLabelGoto) || IsPrimitive(input, prim::kPrimLabelSwitch);
@ -1346,7 +1386,8 @@ std::vector<int64_t> AnfRuntimeAlgorithm::GetOutputMaxShape(const AnfNodePtr &an
} else if (shape->isa<abstract::NoShape>()) {
return {};
} else {
MS_LOG(EXCEPTION) << "Invalid Shape Type";
MS_LOG(EXCEPTION) << "Invalid Shape Type"
<< " trace: " << trace::DumpSourceLines(anf_node);
}
}
@ -1363,7 +1404,8 @@ std::vector<int64_t> AnfRuntimeAlgorithm::GetOutputMinShape(const AnfNodePtr &an
} else if (shape->isa<abstract::NoShape>()) {
return {};
} else {
MS_LOG(EXCEPTION) << "Invalid Shape Type";
MS_LOG(EXCEPTION) << "Invalid Shape Type"
<< " trace: " << trace::DumpSourceLines(anf_node);
}
}

20
mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc
View File

@ -44,6 +44,7 @@
#include "debug/data_dump/dump_json_parser.h"
#include "toolchain/adx_datadump_server.h"
#include "utils/shape_utils.h"
#include "utils/trace_base.h"
#ifdef MEM_REUSE_DEBUG
#include "backend/optimizer/mem_reuse/mem_reuse_checker.h"
#endif
@ -111,7 +112,7 @@ std::string GetRankId() {
}
} // namespace
std::vector<rtExceptionInfo> AscendKernelRuntime::exception_infos_;
std::vector<rtExceptionInfo> AscendKernelRuntime::exception_infoes_;
AscendKernelRuntime::~AscendKernelRuntime() { graph_model_map_.clear(); }
void AscendKernelRuntime::SetContext() {
@ -543,7 +544,7 @@ void AscendKernelRuntime::LaunchDataDump(GraphId graph_id) {
void AscendKernelRuntime::ExceptionCallback(rtExceptionInfo *exception_info) {
static std::mutex exception_mutex;
std::lock_guard<std::mutex> lock(exception_mutex);
exception_infos_.push_back(*exception_info);
exception_infoes_.push_back(*exception_info);
}
void AscendKernelRuntime::DumpTaskExceptionInfo(const session::KernelGraph *graph) {
@ -551,11 +552,12 @@ void AscendKernelRuntime::DumpTaskExceptionInfo(const session::KernelGraph *grap
std::vector<std::string> full_scope_name{};
// Find node name(full scope name)
auto runtime_info_map = ModelRunner::Instance().GetRuntimeInfoMap(graph->graph_id());
MS_LOG(ERROR) << "Exception_infos_ size: " << exception_infos_.size() << ". first example: "
<< ", task_id: " << exception_infos_.at(0).taskid << ", stream_id: " << exception_infos_.at(0).streamid
<< ", tid: " << exception_infos_.at(0).tid << ", device_id: " << exception_infos_.at(0).deviceid;
MS_LOG(ERROR) << "Exception_infos_ size: " << exception_infoes_.size() << ". first example: "
<< ", task_id: " << exception_infoes_.at(0).taskid
<< ", stream_id: " << exception_infoes_.at(0).streamid << ", tid: " << exception_infoes_.at(0).tid
<< ", device_id: " << exception_infoes_.at(0).deviceid;
for (const auto &exception_info : exception_infos_) {
for (const auto &exception_info : exception_infoes_) {
for (const auto &iter : runtime_info_map) {
auto task_id = std::get<kTupleTaskId>(*iter.second);
auto stream_id = std::get<kTupleStreamId>(*iter.second);
@ -566,11 +568,12 @@ void AscendKernelRuntime::DumpTaskExceptionInfo(const session::KernelGraph *grap
}
}
// Dump error data in local path
const std::string local_path = std::string("./task_error_dump/") + std::to_string(exception_infos_.at(0).deviceid);
const std::string local_path = std::string("./task_error_dump/") + std::to_string(exception_infoes_.at(0).deviceid);
for (const auto &node : graph->execution_order()) {
for (auto &name : full_scope_name) {
if (node->fullname_with_scope() == name) {
MS_LOG(ERROR) << "Begin to dump node (" << name << ") task error input/output data in local path.";
MS_LOG(ERROR) << "Begin to dump node (" << name << ") task error input/output data in local path."
<< " trace: " << trace::DumpSourceLines(node);
E2eDumpUtil::DumpInputImpl(node, false, local_path, &name, nullptr);
E2eDumpUtil::DumpOutputImpl(node, false, local_path, &name, nullptr);
}
@ -686,6 +689,7 @@ bool AscendKernelRuntime::RunTask(const session::KernelGraph *graph) {
DumpTaskExceptionInfo(graph);
return false;
}
exception_infoes_.clear();
return true;
}

2
mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.h
View File

@ -88,8 +88,8 @@ class AscendKernelRuntime : public KernelRuntime {
unordered_map<GraphId, vector<std::shared_ptr<TaskInfo>>> task_map_;
unordered_map<GraphId, std::shared_ptr<ge::model_runner::DavinciModel>> graph_model_map_;
unordered_map<GraphId, std::shared_ptr<DataDumper>> graph_data_dumper_;
static std::vector<rtExceptionInfo> exception_infos_;
std::map<std::pair<uint32_t, uint32_t>, std::string> stream_id_task_id_op_name_map_;
static std::vector<rtExceptionInfo> exception_infoes_;
};
MS_REG_KERNEL_RUNTIME(kAscendDevice, AscendKernelRuntime);

10
mindspore/ccsrc/runtime/device/ascend/kernel_select_ascend.cc
View File

@ -33,7 +33,7 @@
#include "frontend/operator/ops.h"
#include "utils/ms_context.h"
#include "utils/ms_utils.h"
#include "utils/trace_base.h"
namespace mindspore {
namespace device {
namespace ascend {
@ -321,7 +321,8 @@ void SetCastAndWeightFormat(const CNodePtr &kernel_node) {
if (!AnfAlgo::HasNodeAttr(kAttrPynativeNextIndex, kernel_node) ||
!AnfAlgo::HasNodeAttr(kAttrPynativeNextOpName, kernel_node)) {
MS_LOG(EXCEPTION) << "The node [" << kernel_node->DebugString() << "] attr of " << kAttrPynativeNextIndex << " or "
<< kAttrPynativeNextOpName << " has been not setted yet!";
<< kAttrPynativeNextOpName << " has been not setted yet!"
<< " trace: " << trace::DumpSourceLines(kernel_node);
}
auto next_index = AnfAlgo::GetNodeAttr<size_t>(kernel_node, kAttrPynativeNextIndex);
auto next_op_name = AnfAlgo::GetNodeAttr<std::string>(kernel_node, kAttrPynativeNextOpName);
@ -332,7 +333,7 @@ void SetCastAndWeightFormat(const CNodePtr &kernel_node) {
}
if (iter->second.size() < next_index) {
MS_LOG(EXCEPTION) << "Next input index " << next_index << "is out of range in the next op map max size is "
<< iter->second.size();
<< iter->second.size() << " trace: " << trace::DumpSourceLines(kernel_node);
}
if (AnfAlgo::GetCNodeName(kernel_node) != prim::kPrimCast->name()) {
MS_LOG(INFO) << "Only supported to change the node Cast's build info!!!";
@ -469,7 +470,8 @@ KernelSelectStatus SelectKernelInfo(const CNodePtr &kernel_node, KernelType kern
MS_LOG(WARNING) << " <<<";
MS_EXCEPTION(TypeError) << "The node [" << kernel_node->DebugString()
<< "] cannot find valid kernel info, not supported the type:" << buffer.str()
<< ", please refer to the supported dtypes in candidates kernel info list";
<< ", please refer to the supported dtypes in candidates kernel info list."
<< " trace: " << trace::DumpSourceLines(kernel_node);
}
}
return select_status;

4
mindspore/core/utils/trace_base.cc
View File

@ -111,7 +111,9 @@ std::string GetDebugInfo(const DebugInfoPtr &info, const std::string &prefix, So
return oss.str();
}
std::string DumpSourceLines(const AnfNodePtr node) {
std::string DumpSourceLines(const AnfNodePtr node) { return DumpSourceLines(node.get()); }
std::string DumpSourceLines(AnfNode *node) {
if (node == nullptr) {
MS_LOG(WARNING) << "Node is null";
return "";

1
mindspore/core/utils/trace_base.h
View File

@ -35,6 +35,7 @@ std::string GetDebugInfo(const DebugInfoPtr &info, const std::string &prefix,
SourceLineTip tip = kSourceLineTipNextLine);
// Generate the call stack of python source code to a string
std::string DumpSourceLines(const AnfNodePtr node);
std::string DumpSourceLines(AnfNode *node);
// Generate the call stack of python source code to a vector
std::vector<std::string> GetSourceLineList(const AnfNodePtr node);
// Get the locations of the call stack of python source code