diff --git a/mindspore/ccsrc/backend/session/ascend_session.cc b/mindspore/ccsrc/backend/session/ascend_session.cc index 5e9c1b827b0..ae770fb363e 100644 --- a/mindspore/ccsrc/backend/session/ascend_session.cc +++ b/mindspore/ccsrc/backend/session/ascend_session.cc @@ -1017,7 +1017,7 @@ void AscendSession::DumpAllGraphs(const std::vector &all_graphs) for (auto &graph : all_graphs) { MS_EXCEPTION_IF_NULL(graph); std::string tag = "graph_build"; - mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, tag, graph, true, ".ir;.pb", graph->graph_id()); + mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, tag, graph, true, ".ir;.pb"); if (save_graphs) { std::string file_name = "graph_build_" + std::to_string(graph->graph_id()) + ".ir"; DumpIR(file_name, graph, true, kWholeStack); diff --git a/mindspore/ccsrc/backend/session/gpu_session.cc b/mindspore/ccsrc/backend/session/gpu_session.cc index 8a015210c5c..0b4d00e586d 100644 --- a/mindspore/ccsrc/backend/session/gpu_session.cc +++ b/mindspore/ccsrc/backend/session/gpu_session.cc @@ -86,6 +86,9 @@ #include "ps/util.h" #include "ps/ps_cache/ps_cache_manager.h" #endif +#ifdef ENABLE_DUMP_IR +#include "debug/rdr/running_data_recorder.h" +#endif namespace mindspore { namespace session { @@ -408,6 +411,10 @@ GraphId GPUSession::CompileGraphImpl(KernelGraphPtr graph) { } // Build kernel if node is cnode BuildKernel(graph); +#ifdef ENABLE_DUMP_IR + std::string tag = "graph_build"; + mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, false, ".ir,.pb"); +#endif // Get summary nodes. SetSummaryNodes(graph.get()); // Dump .pb graph after graph optimization diff --git a/mindspore/ccsrc/debug/rdr/graph_recorder.cc b/mindspore/ccsrc/debug/rdr/graph_recorder.cc index a71d25c0ba9..1e818349013 100644 --- a/mindspore/ccsrc/debug/rdr/graph_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/graph_recorder.cc @@ -16,6 +16,7 @@ #include "debug/rdr/graph_recorder.h" #include "mindspore/core/base/base.h" #include "mindspore/core/ir/func_graph.h" +#include "backend/session/kernel_graph.h" #include "mindspore/core/utils/log_adapter.h" #include "debug/anf_ir_dump.h" #include "debug/anf_ir_utils.h" @@ -57,16 +58,17 @@ void DumpIRProto(const std::string &, const FuncGraphPtr &) { void GraphRecorder::Export() { bool save_flag = false; - - auto tmp_realpath = GetFileRealPath(); + int graph_id = -1; + if (func_graph_->isa()) { + auto kernel_graph = func_graph_->cast(); + graph_id = kernel_graph->graph_id(); + } + std::string suffix = graph_id >= 0 ? std::to_string(graph_id) : ""; + auto tmp_realpath = GetFileRealPath(suffix); if (!tmp_realpath.has_value()) { return; } - std::string realpath = tmp_realpath.value(); - if (graph_id_ >= 0) { - realpath += "_" + std::to_string(graph_id_); - } if (graph_type_.find(".dat") != std::string::npos) { save_flag = true; AnfExporter exporter(""); @@ -81,7 +83,7 @@ void GraphRecorder::Export() { if (full_name_) { DumpIRForRDR(realpath_ir, func_graph_, true, kTopStack); } else { - DumpIRForRDR(realpath_ir, func_graph_, false, kOff); + DumpIRForRDR(realpath_ir, func_graph_, false, kWholeStack); } } if (graph_type_.find(".pb") != std::string::npos) { diff --git a/mindspore/ccsrc/debug/rdr/graph_recorder.h b/mindspore/ccsrc/debug/rdr/graph_recorder.h index ba8a382cf27..cda58039b01 100644 --- a/mindspore/ccsrc/debug/rdr/graph_recorder.h +++ b/mindspore/ccsrc/debug/rdr/graph_recorder.h @@ -28,8 +28,8 @@ class GraphRecorder : public BaseRecorder { public: GraphRecorder() : BaseRecorder(), func_graph_(nullptr), graph_type_("") {} GraphRecorder(const std::string &module, const std::string &tag, const FuncGraphPtr &graph, - const std::string &file_type, int graph_id) - : BaseRecorder(module, tag), func_graph_(graph), graph_type_(file_type), graph_id_(graph_id) {} + const std::string &file_type) + : BaseRecorder(module, tag), func_graph_(graph), graph_type_(file_type) {} ~GraphRecorder() {} void SetModule(const std::string &module) { module_ = module; } void SetGraphType(const std::string &file_type) { graph_type_ = file_type; } @@ -41,7 +41,6 @@ class GraphRecorder : public BaseRecorder { private: FuncGraphPtr func_graph_; std::string graph_type_; - int graph_id_; bool full_name_{false}; }; using GraphRecorderPtr = std::shared_ptr; diff --git a/mindspore/ccsrc/debug/rdr/running_data_recorder.cc b/mindspore/ccsrc/debug/rdr/running_data_recorder.cc index 70a42ff03de..8b3e0cea98d 100644 --- a/mindspore/ccsrc/debug/rdr/running_data_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/running_data_recorder.cc @@ -75,9 +75,9 @@ bool RecordTaskDebugInfo(SubModuleId module, const std::string &tag, #ifdef __linux__ bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name, - const std::string &file_type, int graph_id) { + const std::string &file_type) { std::string submodule_name = std::string(GetSubModuleName(module)); - GraphRecorderPtr graph_recorder = std::make_shared(submodule_name, tag, graph, file_type, graph_id); + GraphRecorderPtr graph_recorder = std::make_shared(submodule_name, tag, graph, file_type); graph_recorder->SetDumpFlag(full_name); bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(graph_recorder)); return ans; @@ -115,7 +115,7 @@ void ClearAll() { mindspore::RecorderManager::Instance().ClearAll(); } #else bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name, - const std::string &file_type, int graph_id) { + const std::string &file_type) { static bool already_printed = false; std::string submodule_name = std::string(GetSubModuleName(module)); if (already_printed) { diff --git a/mindspore/ccsrc/debug/rdr/running_data_recorder.h b/mindspore/ccsrc/debug/rdr/running_data_recorder.h index 3233c271ce7..7ab488d9b02 100644 --- a/mindspore/ccsrc/debug/rdr/running_data_recorder.h +++ b/mindspore/ccsrc/debug/rdr/running_data_recorder.h @@ -37,7 +37,7 @@ using TaskDebugInfoPtr = std::shared_ptr &final_exec_order, int graph_id = 0); bool RecordString(SubModuleId module, const std::string &tag, const std::string &data, diff --git a/mindspore/ccsrc/pipeline/jit/pipeline.cc b/mindspore/ccsrc/pipeline/jit/pipeline.cc index 5f6ffb84990..c82a7e7fe82 100644 --- a/mindspore/ccsrc/pipeline/jit/pipeline.cc +++ b/mindspore/ccsrc/pipeline/jit/pipeline.cc @@ -724,6 +724,21 @@ void Pipeline::Run() { if (!result) { MS_LOG(EXCEPTION) << "Pipeline running to end, failed in step:" << action.first; } +#ifdef ENABLE_DUMP_IR + MS_LOG(INFO) << "Clone func_graph."; + std::string tag = GetBaseNameForIR(i, action.first); + if (resource_->func_graph() != nullptr) { + auto graph_clone = BasicClone(resource_->func_graph()); + if (graph_clone != nullptr) { + mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, tag, graph_clone, false, ".ir"); + } else { + MS_LOG(WARNING) << "Clone func_graph failed in pipeline, no func_graph recording in RDR."; + } + } else { + MS_LOG(WARNING) << "Resource's func_graph is empty in pipeline, no func_graph recording in RDR"; + } + MS_LOG(INFO) << "Clone func_graph end."; +#endif if (MsContext::GetInstance()->get_param(MS_CTX_SAVE_GRAPHS_FLAG) && resource_->func_graph() != nullptr) { auto graph = resource_->func_graph(); if (graph != nullptr) { diff --git a/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc b/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc index 8688208ff0d..969ee556119 100644 --- a/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc +++ b/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc @@ -21,6 +21,9 @@ #include "utils/ms_utils.h" #include "runtime/device/ascend/profiling/profiling_utils.h" #include "runtime/device/ascend/profiling/profiling_manager.h" +#ifdef ENABLE_DUMP_IR +#include "debug/rdr/running_data_recorder.h" +#endif namespace mindspore { namespace device { @@ -36,6 +39,10 @@ bool TaskGenerator::GenTasks(const std::vector &anf_node_list, std::ve return false; } MS_LOG(INFO) << "GenTasks end..."; +#ifdef ENABLE_DUMP_IR + string task_info_tag = "task_info_graph"; + mindspore::RDR::RecordTaskDebugInfo(SUBMODULE_ID, task_info_tag, task_debug_info_list_, graph_id); +#endif auto context_ptr = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context_ptr); bool save_graphs = context_ptr->get_param(MS_CTX_SAVE_GRAPHS_FLAG);