forked from mindspore-Ecosystem/mindspore
enhance RDR support on CPU
1. collect backend graph and graph execution order 2. add trigger point when CPUKernelRuntime::Run() raise "Launch kernel failed"
This commit is contained in:
parent
d14f688ea1
commit
0cb9a2d218
|
@ -37,6 +37,10 @@
|
|||
#include "ps/util.h"
|
||||
#include "ps/ps_context.h"
|
||||
#endif
|
||||
#ifdef ENABLE_DUMP_IR
|
||||
#include "debug/rdr/graph_recorder.h"
|
||||
#include "debug/rdr/running_data_recorder.h"
|
||||
#endif
|
||||
|
||||
namespace mindspore {
|
||||
namespace session {
|
||||
|
@ -106,6 +110,17 @@ GraphId CPUSession::CompileGraphImpl(const AnfNodePtrList &lst, const AnfNodePtr
|
|||
auto execution_order = graph->execution_order();
|
||||
Reorder(&execution_order);
|
||||
graph->set_execution_order(execution_order);
|
||||
|
||||
#ifdef ENABLE_DUMP_IR
|
||||
std::string name = "graph_build." + std::to_string(graph->graph_id());
|
||||
DumpGraphParams dump_params = {true, static_cast<int>(kWholeStack)};
|
||||
mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, name, graph, dump_params, ".ir");
|
||||
|
||||
const std::vector<CNodePtr> &exec_order = graph->execution_order();
|
||||
std::string exec_order_name = "graph_exec_order." + std::to_string(graph->graph_id());
|
||||
mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_SESSION, exec_order_name, exec_order);
|
||||
#endif
|
||||
|
||||
// runtime init
|
||||
if (!runtime_.Init()) {
|
||||
MS_LOG(EXCEPTION) << "Kernel runtime init error.";
|
||||
|
|
|
@ -39,6 +39,9 @@
|
|||
#ifdef MEM_REUSE_DEBUG
|
||||
#include "backend/optimizer/mem_reuse/mem_reuse_checker.h"
|
||||
#endif
|
||||
#ifdef ENABLE_DUMP_IR
|
||||
#include "debug/rdr/running_data_recorder.h"
|
||||
#endif
|
||||
|
||||
namespace mindspore {
|
||||
namespace device {
|
||||
|
@ -424,6 +427,9 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph, bool is_task_sink
|
|||
profiler_inst->OpDataProducerEnd();
|
||||
}
|
||||
if (!ret) {
|
||||
#ifdef ENABLE_DUMP_IR
|
||||
mindspore::RDR::TriggerAll();
|
||||
#endif
|
||||
MS_LOG(EXCEPTION) << "Launch kernel failed. Trace:" << trace::DumpSourceLines(kernel);
|
||||
}
|
||||
static_cast<CPUMemoryManager *>(mem_manager_.get())->DecreaseAddressRefCount(kernel);
|
||||
|
|
Loading…
Reference in New Issue