p83571920
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!12087 optimize the memory alloc error info 

From: @limingqi107
Reviewed-by: @kisnwang,@cristoval
Signed-off-by: @cristoval
This commit is contained in:
mindspore-ci-bot 2021-02-04 20:23:55 +08:00 committed by Gitee
parent 7f709d9528 366f3e668d
commit 025fec757c
3 changed files with 12 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
mindspore/ccsrc/runtime/device/gpu/gpu_kernel_runtime.cc
View File

@ -302,7 +302,7 @@ void GPUKernelRuntime::AllocInplaceNodeMemory(const session::KernelGraph *graph)
auto output_size = kernel_mod->GetOutputSizeList();
auto ret = mem_manager_->MallocMemFromMemPool(device_address, output_size[output_index]);
if (!ret) {
MS_LOG(EXCEPTION) << "Cannot alloc address, tensor size is: " << output_size[output_index];
MS_LOG(EXCEPTION) << "Device memory isn't enough and alloc failed, alloc size:" << output_size[output_index];
}
for (auto &node : item) {
@ -662,8 +662,9 @@ bool GPUKernelRuntime::LaunchKernelDynamic(const session::KernelGraph *graph, bo
if (profiler_inst->GetEnableFlag()) {
profiler_inst->OpDataProducerBegin(kernel->fullname_with_scope(), stream_);
}
CHECK_OP_RET_WITH_EXCEPT(kernel_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, stream_),
"Launch kernel failed.");
if (!kernel_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, stream_)) {
MS_LOG(EXCEPTION) << "Launch kernel failed: " << kernel->fullname_with_scope();
}
if (profiler_inst->GetEnableFlag()) {
profiler_inst->OpDataProducerEnd();
if (profiler_inst->GetSyncEnableFlag()) {

4
mindspore/ccsrc/runtime/device/gpu/gpu_memory_manager.cc
View File

@ -106,7 +106,9 @@ uint8_t *GPUMemoryManager::MallocStaticMem(size_t size, bool, uint32_t) {
MS_EXCEPTION_IF_NULL(context_ptr);
if (context_ptr->get_param<bool>(MS_CTX_ENABLE_DYNAMIC_MEM_POOL)) {
auto device_ptr = MallocMemFromMemPool(size);
MS_EXCEPTION_IF_NULL(device_ptr);
if (device_ptr == nullptr) {
MS_LOG(EXCEPTION) << "Device memory isn't enough and alloc failed, alloc size:" << size;
}
return AddressOffset(device_ptr, 0);
}

10
mindspore/ccsrc/runtime/device/kernel_runtime.cc
View File

@ -187,7 +187,7 @@ void KernelRuntime::RunOpAssignInputMemory(const std::vector<tensor::TensorPtr>
MS_EXCEPTION_IF_NULL(mem_manager_);
auto ret = mem_manager_->MallocMemFromMemPool(device_address, tensor_size);
if (!ret) {
MS_LOG(EXCEPTION) << "Malloc device memory failed.";
MS_LOG(EXCEPTION) << "Device memory isn't enough and alloc failed, alloc size:" << tensor_size;
}
AnfAlgo::SetOutputAddr(device_address, index, item.get());
}
@ -220,7 +220,7 @@ void KernelRuntime::RunOpAssignOutputMemory(const AnfNodePtr &kernel) {
MS_EXCEPTION_IF_NULL(device_address);
auto ret = mem_manager_->MallocMemFromMemPool(device_address, output_sizes[i]);
if (!ret) {
MS_LOG(EXCEPTION) << "Malloc device memory failed.";
MS_LOG(EXCEPTION) << "Device memory isn't enough and alloc failed, alloc size:" << output_sizes[i];
}
AnfAlgo::SetOutputAddr(device_address, i, kernel.get());
}
@ -238,7 +238,7 @@ void KernelRuntime::RunOpAssignWorkSpaceMemory(const AnfNodePtr &kernel) {
MS_EXCEPTION_IF_NULL(device_address);
auto ret = mem_manager_->MallocMemFromMemPool(device_address, workspace_lists[i]);
if (!ret) {
MS_LOG(EXCEPTION) << "Malloc device memory failed.";
MS_LOG(EXCEPTION) << "Device memory isn't enough and alloc failed, alloc size:" << workspace_lists[i];
}
AnfAlgo::SetWorkspaceAddr(device_address, i, kernel.get());
}
@ -651,7 +651,7 @@ void KernelRuntime::AssignValueNodeTensor(const ValueNodePtr &value_node, const
MS_EXCEPTION_IF_NULL(address);
if (ms_context->get_param<bool>(MS_CTX_ENABLE_PYNATIVE_INFER) &&
!mem_manager_->MallocMemFromMemPool(address, node_size)) {
MS_LOG(EXCEPTION) << "Cannot alloc address from memory pool when tensor size is: " << node_size;
MS_LOG(EXCEPTION) << "Device memory isn't enough and alloc failed, alloc size:" << node_size;
} else if (mem_manager_->MallocMem(kStaticMem, node_size, address, graph_id) == nullptr) {
MS_LOG(EXCEPTION) << "Cannot alloc address when flag is: " << kStaticMem << ", tensor size is: " << node_size;
}
@ -692,7 +692,7 @@ void KernelRuntime::AssignStaticMemoryValueNode(session::KernelGraph *graph) {
MS_EXCEPTION_IF_NULL(address);
if (ms_context->get_param<bool>(MS_CTX_ENABLE_PYNATIVE_INFER) &&
!mem_manager_->MallocMemFromMemPool(address, tensor_size)) {
MS_LOG(EXCEPTION) << "Cannot alloc address from memory pool when tensor size is: " << tensor_size;
MS_LOG(EXCEPTION) << "Device memory isn't enough and alloc failed, alloc size:" << tensor_size;
} else if (mem_manager_->MallocMem(kStaticMem, tensor_size, address, graph->graph_id()) == nullptr) {
MS_LOG(EXCEPTION) << "Cannot alloc address when flag is: " << kStaticMem << ", tensor size is: " << tensor_size;
}