enhance RDR support on CPU

1. collect backend graph and graph execution order
2. add trigger point when CPUKernelRuntime::Run() raise "Launch kernel failed"
This commit is contained in:
luopengting 2021-04-19 09:28:33 +08:00
parent d14f688ea1
commit 0cb9a2d218
2 changed files with 21 additions and 0 deletions

View File

@ -37,6 +37,10 @@
#include "ps/util.h"
#include "ps/ps_context.h"
#endif
#ifdef ENABLE_DUMP_IR
#include "debug/rdr/graph_recorder.h"
#include "debug/rdr/running_data_recorder.h"
#endif
namespace mindspore {
namespace session {
@ -106,6 +110,17 @@ GraphId CPUSession::CompileGraphImpl(const AnfNodePtrList &lst, const AnfNodePtr
auto execution_order = graph->execution_order();
Reorder(&execution_order);
graph->set_execution_order(execution_order);
#ifdef ENABLE_DUMP_IR
std::string name = "graph_build." + std::to_string(graph->graph_id());
DumpGraphParams dump_params = {true, static_cast<int>(kWholeStack)};
mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, name, graph, dump_params, ".ir");
const std::vector<CNodePtr> &exec_order = graph->execution_order();
std::string exec_order_name = "graph_exec_order." + std::to_string(graph->graph_id());
mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_SESSION, exec_order_name, exec_order);
#endif
// runtime init
if (!runtime_.Init()) {
MS_LOG(EXCEPTION) << "Kernel runtime init error.";

View File

@ -39,6 +39,9 @@
#ifdef MEM_REUSE_DEBUG
#include "backend/optimizer/mem_reuse/mem_reuse_checker.h"
#endif
#ifdef ENABLE_DUMP_IR
#include "debug/rdr/running_data_recorder.h"
#endif
namespace mindspore {
namespace device {
@ -424,6 +427,9 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph, bool is_task_sink
profiler_inst->OpDataProducerEnd();
}
if (!ret) {
#ifdef ENABLE_DUMP_IR
mindspore::RDR::TriggerAll();
#endif
MS_LOG(EXCEPTION) << "Launch kernel failed. Trace:" << trace::DumpSourceLines(kernel);
}
static_cast<CPUMemoryManager *>(mem_manager_.get())->DecreaseAddressRefCount(kernel);