p04896132
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!10192 fix the bug of Profiler module do not print meaningful error when training process unexpectedly stopped 

From: @gzhcv
Reviewed-by: @wangyue01,@wuxuejian
Signed-off-by: @wuxuejian
This commit is contained in:
mindspore-ci-bot 2020-12-25 11:51:43 +08:00 committed by Gitee
parent e6f20bfb64 3a40bfc01d
commit f1579bc5d1
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mindspore/ccsrc/profiler/device/gpu/gpu_profiling.cc
View File

@ -70,6 +70,10 @@ uint32_t GetStreamID(const CUcontext context, const void *stream) {
uint32_t stream_id = 0; uint32_t stream_id = 0;
if (stream != nullptr) { if (stream != nullptr) {
CHECK_CUPTI_RET_WITH_ERROR(CuptiGetStreamId(context, (CUstream)stream, &stream_id), "CuptiGetStreamId"); CHECK_CUPTI_RET_WITH_ERROR(CuptiGetStreamId(context, (CUstream)stream, &stream_id), "CuptiGetStreamId");
if (CuptiGetStreamId(context, (CUstream)stream, &stream_id) != CUPTI_SUCCESS) {
MS_LOG(ERROR) << "Training process unexpectedly stopped, profiling data cannot be write to file"
<< "To obtain the profiling data, do not interrupt the training process.";
}
} }
return stream_id; return stream_id;
} }