p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add log for memory allocate failed when device is occupied 

Merge pull request  from laiyongqiang/DeviceOccupy
This commit is contained in:
mindspore-ci-bot 2020-09-18 14:34:46 +08:00 committed by Gitee
parent e4d8865a6b 0e2124b8ed
commit 0b88808577
1 changed files with 9 additions and 1 deletions
mindspore/ccsrc/runtime/device/ascend
ascend_memory_manager.cc

10
mindspore/ccsrc/runtime/device/ascend/ascend_memory_manager.cc
View File

@ -30,7 +30,15 @@ void AscendMemoryManager::MallocDeviceMemory() {
device_mem_size_ = context_mem == 0 ? kAscendDeviceMemSize : context_mem; device_mem_size_ = context_mem == 0 ? kAscendDeviceMemSize : context_mem;
auto ret = rtMalloc(reinterpret_cast<void **>(&device_mem_base_), device_mem_size_, RT_MEMORY_HBM); auto ret = rtMalloc(reinterpret_cast<void **>(&device_mem_base_), device_mem_size_, RT_MEMORY_HBM);
if (ret != RT_ERROR_NONE) { if (ret != RT_ERROR_NONE) {
MS_EXCEPTION(DeviceProcessError) << "rtMalloc mem size[" << device_mem_size_ << "] fail, ret[" << ret << "]"; if (ret == RT_ERROR_DRV_ERR) {
auto context_ptr = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context_ptr);
unsigned int device_id = context_ptr->get_param<uint32_t>(MS_CTX_DEVICE_ID);
MS_LOG(EXCEPTION) << "Device " << device_id << " is occupied, malloc device memory failed, size["
<< device_mem_size_ << "], ret[" << ret << "]";
} else {
MS_EXCEPTION(DeviceProcessError) << "rtMalloc mem size[" << device_mem_size_ << "] fail, ret[" << ret << "]";
}
} }
AscendMemoryPool::GetInstance().Init(device_mem_base_, device_mem_size_, dynamic_mem_offset_); AscendMemoryPool::GetInstance().Init(device_mem_base_, device_mem_size_, dynamic_mem_offset_);