mindspore-Ecosystem
/
mindspore
21
0
Fork 27
Code Issues Pull Requests 6 Projects Releases Wiki Activity

error manager

This commit is contained in:
jiaorui 2022-02-08 09:40:32 +08:00
parent 358b7124bc
commit 803210d280
2 changed files with 26 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
mindspore/ccsrc/runtime/hardware/ascend/ascend_device_context.cc
View File

@ -354,24 +354,30 @@ void AscendDeviceContext::GenKernelEvents(const NotNull<KernelGraphPtr> &root_gr
void AscendDeviceContext::PreprocessBeforeRunGraph(const KernelGraphPtr &graph) const {
MS_EXCEPTION_IF_NULL(graph);
MS_LOG(INFO) << "Status record: start preprocess before run graph. graph id: " << graph->graph_id();
if (graph->is_executing_sink()) {
device::ascend::InsertAtomicCleanOps(graph->execution_order(), &node_atomics_);
UpdateExecOrder(graph);
device::KernelAdjust::GetInstance().InsertDeviceLoopCtrl(graph);
device::KernelAdjust::GetInstance().ProcessLoopSink(graph);
AscendStreamAssign::GetInstance().AssignStream(NOT_NULL(graph));
SetErrorManagerContext();
try {
if (graph->is_executing_sink()) {
device::ascend::InsertAtomicCleanOps(graph->execution_order(), &node_atomics_);
UpdateExecOrder(graph);
device::KernelAdjust::GetInstance().InsertDeviceLoopCtrl(graph);
device::KernelAdjust::GetInstance().ProcessLoopSink(graph);
AscendStreamAssign::GetInstance().AssignStream(NOT_NULL(graph));
#ifndef ENABLE_SECURITY
// Insert profiling point, this function must be executed after assign stream.
device::KernelAdjust::GetInstance().Profiling(NOT_NULL(graph.get()));
// Insert profiling point, this function must be executed after assign stream.
device::KernelAdjust::GetInstance().Profiling(NOT_NULL(graph.get()));
#endif
CreateKernel(graph->execution_order());
AllocateGraphMemory(NOT_NULL(graph));
LoadModel(NOT_NULL(graph));
AssignOutputNopNodeDeviceAddress(graph);
} else {
PreprocessBeforeRunSingleOpGraph(graph);
AscendStreamAssign::GetInstance().AssignStream(NOT_NULL(graph));
GenKernelEvents(NOT_NULL(graph));
CreateKernel(graph->execution_order());
AllocateGraphMemory(NOT_NULL(graph));
LoadModel(NOT_NULL(graph));
AssignOutputNopNodeDeviceAddress(graph);
} else {
PreprocessBeforeRunSingleOpGraph(graph);
AscendStreamAssign::GetInstance().AssignStream(NOT_NULL(graph));
GenKernelEvents(NOT_NULL(graph));
}
} catch (const std::exception &e) {
ReportErrorMessage();
MS_LOG(EXCEPTION) << "Preprocess failed before run graph " << graph->graph_id();
}
MS_LOG(INFO) << "Status record: end preprocess before run graph. graph id: " << graph->graph_id();
@ -559,6 +565,7 @@ bool AscendDeviceContext::LaunchGraph(const KernelGraphPtr &graph) const {
MS_EXCEPTION_IF_NULL(graph);
MS_EXCEPTION_IF_NULL(runtime_instance_);
runtime_instance_->SetContext();
SetErrorManagerContext();
device::KernelAdjust::GetInstance().LoadDeviceLoopCtrlParameters(graph);
auto ret = ExecuteGraph(graph);
if (!ret) {
@ -578,6 +585,8 @@ void AscendDeviceContext::ReportErrorMessage() const {
}
}
void AscendDeviceContext::SetErrorManagerContext() const { ErrorManager::GetInstance().GenWorkStreamIdDefault(); }
void AscendDeviceContext::ReportWarningMessage() const {
const string &warning_message = ErrorManager::GetInstance().GetWarningMessage();
if (!warning_message.empty()) {

1
mindspore/ccsrc/runtime/hardware/ascend/ascend_device_context.h
View File

@ -145,6 +145,7 @@ class AscendDeviceContext : public DeviceContext {
void ReportErrorMessage() const;
void ReportWarningMessage() const;
void SetErrorManagerContext() const;
// Kernel Runtime --- only for task sink
AscendKernelRuntime *runtime_instance_{nullptr};