p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!22247 error log rectification 

Merge pull request !22247 from hwjiaorui/error-log
This commit is contained in:
i-robot 2021-08-26 03:13:08 +00:00 committed by Gitee
parent f8375afce9 12b4940e8a
commit 49d84b3a87
10 changed files with 33 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_kernel_select/tbe_kernel_broadcast_selecter.cc
View File

@ -36,8 +36,12 @@ bool TbeKernelBroadCastSelecter::GetShapeInfo(SupportFormat *support_format) {
if (AnfAlgo::HasNodeAttr(kAttrDynInputSizes, cnode_ptr_)) {
auto dynamic_size_vec = AnfAlgo::GetNodeAttr<std::vector<int64_t>>(cnode_ptr_, kAttrDynInputSizes);
constexpr int64_t DYNAMIC_INPUT_NUM = 2;
if (dynamic_size_vec.empty() || dynamic_size_vec[0] < DYNAMIC_INPUT_NUM) {
MS_LOG(EXCEPTION) << "dynamic attr set error, please check.";
if (dynamic_size_vec.empty()) {
MS_LOG(EXCEPTION) << "Node [" << AnfAlgo::GetCNodeName(cnode_ptr_) << "]'s attr [dyn_input_sizes] is empty.";
}
if (dynamic_size_vec[0] < DYNAMIC_INPUT_NUM) {
MS_LOG(EXCEPTION) << "Node [" << AnfAlgo::GetCNodeName(cnode_ptr_)
<< "]'s attr [dyn_input_sizes] value less than " << DYNAMIC_INPUT_NUM;
}
auto dynamic_input_shape0_ = AnfAlgo::GetPrevNodeOutputInferShape(cnode_ptr_, kInputIndex_0);
PadScalarShape(&dynamic_input_shape0_);

2
mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_kernel_select/tbe_kernel_select.cc
View File

@ -96,7 +96,7 @@ void TbeKernelSelect::GetCommonPatternKernelInfo(const OpInfo &op_info) {
size_t real_output_tensor_num = AnfAlgo::GetOutputTensorNum(cnode_ptr_);
const auto outputs_info = op_info.outputs_ptr();
if (inputs_info.empty() && outputs_info.empty()) {
MS_LOG(EXCEPTION) << "op info input & output is null, please check.";
MS_LOG(EXCEPTION) << AnfAlgo::GetCNodeName(cnode_ptr_) << "'s op info input & output is null, please check.";
}
// create kernel build info from opinfo
size_t kernel_build_info_num =

83
mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.cc
View File

@ -1159,63 +1159,6 @@ bool AscendStreamAssign::CheckStreamSwitch(const CNodePtr &switch_ptr) {
return true;
}
void AscendStreamAssign::UpdateStreamSwitch(const NotNull<KernelGraphPtr> &graph_ptr, const CNodePtr &switch_ptr,
vector<CNodePtr> *orders) {
if (!CheckStreamSwitch(switch_ptr)) {
orders->emplace_back(switch_ptr);
return;
}
auto kind = AnfAlgo::GetNodeAttr<uint32_t>(switch_ptr, kAttrStreamSwitchKind);
if (kind == kIndependentStreamSwitch) {
bool independent_empty = independent_stream_map_.empty();
// if independent empty: delete independent streamswitch
if (!independent_empty) {
for (const auto &item : independent_stream_map_) {
// first independent stream id is minimum and order by std map;
auto first_independent_stream = item.first;
AnfAlgo::SetNodeAttr(kAttrTrueBranchStream, MakeValue<uint32_t>(first_independent_stream), switch_ptr);
orders->emplace_back(switch_ptr);
break;
}
} else {
MS_LOG(ERROR) << "Independent stream switch exit, but independent stream is empty";
}
// update processed stream
independent_stream_activated_ = true;
for (const auto &item : independent_stream_map_) {
processed_streams_.emplace(item.first);
}
} else if (kind == kFpBpStreamSwitch) {
if (hcom_stream_map_.empty()) {
orders->emplace_back(switch_ptr);
return;
}
if (!AnfAlgo::HasNodeAttr(kAttrTrueBranchStream, switch_ptr)) {
// FpBp StreamSwitch has no true branch attr
orders->emplace_back(switch_ptr);
return;
}
auto true_stream_id = AnfAlgo::GetNodeAttr<uint32_t>(switch_ptr, kAttrTrueBranchStream);
MS_LOG(INFO) << "Switch stream id:" << AnfAlgo::GetStreamId(switch_ptr) << "; active stream id:" << true_stream_id;
CNodePtr active_ptr = KernelAdjust::GetInstance().CreateStreamActiveOp(graph_ptr);
AnfAlgo::SetStreamId(true_stream_id, active_ptr.get());
vector<uint32_t> active_ids;
// active hcom stream
for (const auto &item : hcom_stream_map_) {
active_ids.emplace_back(item.first);
}
AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue<std::vector<uint32_t>>(active_ids), active_ptr);
hcom_stream_activated_ = true;
for (const auto &item : hcom_stream_map_) {
processed_streams_.emplace(item.first);
}
orders->emplace_back(switch_ptr);
orders->emplace_back(active_ptr);
}
}
bool AscendStreamAssign::IsProcessedStream(uint32_t stream_id) {
auto it = std::find(processed_streams_.begin(), processed_streams_.end(), stream_id);
if (it != processed_streams_.end()) {
@ -1936,16 +1879,18 @@ void AscendStreamAssign::CheckEventAssign(const NotNull<KernelGraphPtr> &graph_p
}
uint32_t assigned_event_num = resource_manager.get_cur_event_num();
if ((max_event_id != assigned_event_num - 1) || (event_map.size() != assigned_event_num)) {
MS_LOG(EXCEPTION) << "Event should be consecutive";
MS_LOG(EXCEPTION) << "Event should be consecutive, however, assigned event num is: " << assigned_event_num
<< ", max event id:" << max_event_id << ", event map is:" << event_map;
}
for (const auto &item : event_map) {
if (item.second.size() != 2) {
MS_LOG(EXCEPTION) << "Send/recv should be in pair and share one event id";
MS_LOG(EXCEPTION) << "Send/recv should be in pair and share one event id, invalid event id is:" << item.first
<< ", event size is:" << item.second.size();
}
auto first_name = AnfAlgo::GetCNodeName(item.second[0]);
auto second_name = AnfAlgo::GetCNodeName(item.second[1]);
if (!(first_name == kSendOpName && second_name == kRecvOpName)) {
MS_LOG(EXCEPTION) << "Send should be before recv";
MS_LOG(EXCEPTION) << "Send should be before recv, invalid event id is:" << item.first;
}
}
}
@ -2220,20 +2165,20 @@ void AscendStreamAssign::GetStreamActiveStreamRelation(const NotNull<KernelGraph
auto cur_stream_id = AnfAlgo::GetStreamId(cur_cnode);
auto active_list = AnfAlgo::GetNodeAttr<vector<uint32_t>>(cur_cnode, kAttrActiveStreamList);
if (kind == kHead) {
uint32_t active_current_node = GetStreamByActivedStream(cur_stream_id);
if (active_current_node == kInvalidStreamId) {
MS_LOG(EXCEPTION) << "No stream to active streamactive stream";
uint32_t active_current_stream_id = GetStreamByActivedStream(cur_stream_id);
if (active_current_stream_id == kInvalidStreamId) {
MS_LOG(EXCEPTION) << "No stream to active streamactive stream: " << cur_stream_id;
}
for (const auto &item : active_list) {
if (item <= active_current_node) {
if (item <= active_current_stream_id) {
MS_LOG(WARNING) << "Activated stream is less than activing stream";
continue;
}
auto it =
std::find(stream_relations_[active_current_node].begin(), stream_relations_[active_current_node].end(), item);
if (it == stream_relations_[active_current_node].end()) {
stream_relations_[active_current_node].emplace_back(item);
auto it = std::find(stream_relations_[active_current_stream_id].begin(),
stream_relations_[active_current_stream_id].end(), item);
if (it == stream_relations_[active_current_stream_id].end()) {
stream_relations_[active_current_stream_id].emplace_back(item);
}
}
}
@ -2277,7 +2222,7 @@ StreamActiveKind AscendStreamAssign::GetStreamActiveKind(const NotNull<KernelGra
auto cur_cnode = exe_orders[index];
auto cur_stream_id = AnfAlgo::GetStreamId(cur_cnode);
if (AnfAlgo::GetCNodeName(cur_cnode) != kStreamActiveOpName) {
MS_LOG(EXCEPTION) << "Current node name is not StreamActive";
MS_LOG(EXCEPTION) << "Current node name [" << AnfAlgo::GetCNodeName(cur_cnode) << "] is not StreamActive.";
}
if (index == 0) {

2
mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.h
View File

@ -141,8 +141,6 @@ class AscendStreamAssign {
const std::set<uint32_t> &independent_streams);
void ActiveOtherGraphParallel(const NotNull<KernelGraphPtr> &graph_ptr,
std::map<uint32_t, std::set<uint32_t>> other_graph);
void UpdateStreamSwitch(const NotNull<KernelGraphPtr> &graph_ptr, const CNodePtr &switch_ptr,
vector<CNodePtr> *orders);
bool CheckStreamSwitch(const CNodePtr &switch_ptr);
void InsertEventForIndependentParallel(const NotNull<KernelGraphPtr> &graph_ptr);
void InsertCtrlForIndependentParallel(const NotNull<KernelGraphPtr> &graph_ptr);

6
mindspore/ccsrc/runtime/device/kernel_adjust.cc
View File

@ -425,7 +425,7 @@ void KernelAdjust::InsertSwitchLoop(const std::shared_ptr<session::KernelGraph>
const std::vector<CNodePtr> &orders = kernel_graph_ptr->execution_order();
if (orders.empty()) {
MS_LOG(EXCEPTION) << "graph execution order is empty";
MS_LOG(EXCEPTION) << "graph " << kernel_graph_ptr->graph_id() << " execution order is empty";
}
std::vector<CNodePtr> exec_order;
@ -571,7 +571,7 @@ CNodePtr KernelAdjust::CreateStreamSwitchOp(const std::shared_ptr<session::Kerne
} else if (kind == kGetNextStreamSwitch || kind == kIndependentStreamSwitch) {
inputs.push_back(switch_loop_input.at(kNextLoopCountParamName));
} else {
MS_LOG(ERROR) << "unknown stream switch kind";
MS_LOG(ERROR) << "unknown stream switch kind: " << kind;
}
inputs.push_back(switch_loop_input.at(kIterLoopParamName));
@ -826,7 +826,7 @@ void KernelAdjust::InsertProfilingKernel(const ProfilingTraceInfo &profiling_tra
std::vector<CNodePtr> new_cnode_list;
std::vector<CNodePtr> cnode_ptr_list = kernel_graph_ptr->execution_order();
if (cnode_ptr_list.empty()) {
MS_LOG(ERROR) << "No CNode in graph";
MS_LOG(ERROR) << "No CNode in graph " << kernel_graph_ptr->graph_id();
return;
}
for (const auto &cnode_ptr : cnode_ptr_list) {

4
mindspore/ccsrc/runtime/device/kernel_info.cc
View File

@ -24,7 +24,7 @@ kernel::KernelBuildInfoPtr KernelInfo::GetMutableSelectKernelBuildInfo() const {
const DeviceAddress *KernelInfo::GetOutputAddr(size_t index) const {
if (index >= output_address_list_.size()) {
MS_LOG(ERROR) << "Index [" << index << "] out of range";
MS_LOG(ERROR) << "Index [" << index << "] out of range 0~" << output_address_list_.size() - 1;
return nullptr;
}
return output_address_list_[index].get();
@ -101,7 +101,7 @@ bool KernelInfo::SetWorkspaceAddr(const DeviceAddressPtr &output_address, size_t
}
}
if (index >= workspace_address_list_.size()) {
MS_LOG(ERROR) << "Index" << index << " out of range";
MS_LOG(ERROR) << "Index [" << index << "] out of range";
return false;
}
workspace_address_list_[index] = output_address;

2
mindspore/ccsrc/runtime/device/kernel_runtime.cc
View File

@ -582,7 +582,7 @@ DeviceAddressPtr KernelRuntime::PreAssignCNodeMemory(const AnfNodePtr &anf_node,
MS_EXCEPTION_IF_NULL(kernel_mod);
auto output_sizes = kernel_mod->GetOutputSizeList();
if (output_sizes.size() <= index) {
MS_LOG(EXCEPTION) << "Previous node output size < node index";
MS_LOG(EXCEPTION) << "Previous node output size " << output_sizes.size() << " <= node index " << index;
}
std::string output_format = AnfAlgo::GetOutputFormat(anf_node, index);
auto output_type = AnfAlgo::GetOutputDeviceDataType(anf_node, index);

3
mindspore/ccsrc/runtime/device/launch_kernel.cc
View File

@ -44,7 +44,8 @@ std::vector<kernel::AddressPtr> LaunchKernel::ObtainKernelInputs(const std::vect
const std::vector<uint8_t *> &inputs_addr) {
std::vector<kernel::AddressPtr> kernel_inputs;
if (inputs_list.size() != inputs_addr.size()) {
MS_LOG(ERROR) << "input_list size should equal to input_addr_ size";
MS_LOG(ERROR) << "input_list size should equal to input_addr_ size, input_list size: " << inputs_list.size()
<< ", input_addr_ size: " << inputs_addr.size();
}
for (size_t i = 0; i < inputs_list.size(); ++i) {
auto input_size = AlignSizeForLaunchKernel(inputs_list[i]);

3
mindspore/ccsrc/runtime/device/launch_mul.cc
View File

@ -52,7 +52,8 @@ kernel::KernelMod *LaunchMul::ObtainLaunchMulKernelMod() {
}
// obtain kernel_mod
if (mul_graph_->execution_order().size() != 1) {
MS_LOG(ERROR) << "the execution order of the mul graph should have only one node";
MS_LOG(ERROR) << "the execution order of the mul graph should have only one node, however, it has "
<< mul_graph_->execution_order().size() << " nodes.";
}
return AnfAlgo::GetKernelMod(mul_graph_->execution_order()[0]);
}

3
mindspore/ccsrc/runtime/device/memory_manager.cc
View File

@ -177,7 +177,8 @@ bool MemoryManager::MallocContinuousMemFromMemPool(const DeviceAddressPtrList ad
return false;
}
if (addr_list.size() != device_ptr_list.size()) {
MS_LOG(EXCEPTION) << "The size of device list is not equal to the size of address list.";
MS_LOG(EXCEPTION) << "The size of device list " << addr_list.size() << " is not equal to the size of address list "
<< device_ptr_list.size();
}
for (size_t i = 0; i < addr_list.size(); i++) {
MS_EXCEPTION_IF_NULL(device_ptr_list[i]);