p60975123
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!23989 add more info when out of memory 

Merge pull request !23989 from laiyongqiang/OoO
This commit is contained in:
i-robot 2021-09-23 13:52:01 +00:00 committed by Gitee
parent 75a7cf3bee 2c787279c0
commit 1e15b00a31
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mindspore/ccsrc/runtime/device/ascend/ascend_memory_manager.cc
View File

@ -180,11 +180,11 @@ uint8_t *AscendMemoryManager::MallocDynamicMem(size_t size, bool communication_m
auto offset = dynamic_mem_offset_;
auto new_offset = dynamic_mem_offset_ + align_size;
if (new_offset >= device_mem_pool_offset) {
MS_LOG(EXCEPTION) << "Out of memory!!! total[" << device_mem_size_ << "] (dynamic[" << total_dynamic_size_
MS_LOG(EXCEPTION) << "Out of Memory!!! total[" << device_mem_size_ << "] (dynamic[" << total_dynamic_size_
<< "] memory pool[" << device_mem_size_ - device_mem_pool_offset << "])"
<< " malloc [" << align_size
<< "] failed! Please try to reduce 'batch_size' or check whether exists extra large shape. More "
"details can be found in mindspore's FAQ";
"details can be found in MindSpore's FAQ with keyword 'Out of Memory'.";
}
total_dynamic_size_ += align_size;
dynamic_mem_offset_ = new_offset;

6
mindspore/ccsrc/runtime/device/ascend/ascend_memory_pool.cc
View File

@ -86,9 +86,11 @@ size_t AscendMemoryPool::AllocDeviceMem(size_t size, DeviceMemPtr *addr) {
}
if (device_mem_pool_offset_ - size < graph_dynamic_mem_offset_) {
MS_LOG(EXCEPTION) << "Failed to alloc memory pool memory, the current device_mem_pool_offset_ ["
MS_LOG(EXCEPTION) << "Out of Memory!!! Failed to alloc memory pool memory, the current device_mem_pool_offset_ ["
<< device_mem_pool_offset_ << "], current graph_dynamic_mem_offset_ " << graph_dynamic_mem_offset_
<< "], need memory size [" << size << "]";
<< "], need memory size [" << size
<< "]. Please try to reduce 'batch_size' or check whether exists extra large shape. More details "
"can be found in MindSpore's FAQ with keyword 'Out of Memory'.";
}
device_mem_pool_offset_ -= size;
*addr = device_mem_pool_base_ + device_mem_pool_offset_;