From 98f7912a6769eb8d076e8bf3e9f296e912eda595 Mon Sep 17 00:00:00 2001 From: laiyongqiang Date: Thu, 24 Sep 2020 16:37:18 +0800 Subject: [PATCH] save task debug info to ir --- .../device/ascend/ascend_kernel_runtime.cc | 3 +- .../device/ascend/tasksink/task_generator.cc | 96 +++++++++++++++++++ .../device/ascend/tasksink/task_generator.h | 25 ++++- 3 files changed, 118 insertions(+), 6 deletions(-) diff --git a/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc b/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc index a14b06329e3..d04affa7836 100644 --- a/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc +++ b/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc @@ -368,7 +368,8 @@ bool AscendKernelRuntime::GenTask(const session::KernelGraph *graph) { #endif vector> task_info_list; auto anf_node_list = graph->execution_order(); - TaskGenerator::GenTasks(anf_node_list, &task_info_list, graph->graph_id()); + auto task_generator = TaskGenerator(); + task_generator.GenTasks(anf_node_list, &task_info_list, graph->graph_id()); // Store the task_info_list auto insert_ret = task_map_.insert(std::make_pair(graph->graph_id(), task_info_list)); if (!insert_ret.second) { diff --git a/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc b/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc index 8b5dc8cfe28..5dc35ac6a4d 100644 --- a/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc +++ b/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc @@ -36,6 +36,14 @@ bool TaskGenerator::GenTasks(const std::vector &anf_node_list, std::ve return false; } MS_LOG(INFO) << "GenTasks end..."; + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + bool save_graphs = context_ptr->get_param(MS_CTX_SAVE_GRAPHS_FLAG); + auto save_graphs_path = context_ptr->get_param(MS_CTX_SAVE_GRAPHS_PATH); + if (save_graphs) { + std::string file_path = save_graphs_path + "/" + "task_info" + "_graph_" + std::to_string(graph_id) + ".ir"; + DumpTaskInfo(file_path); + } return true; } @@ -162,6 +170,15 @@ bool TaskGenerator::LaunchKernel(const CNodePtr &anf_node_ptr, uint32_t stream_i std::vector task_info_ptrs = ascend_kernel_mod->GenTask(kernel_inputs, kernel_workspaces, kernel_outputs, stream_id); task_info_list->insert(task_info_list->end(), task_info_ptrs.begin(), task_info_ptrs.end()); + auto debug_info = std::make_shared(); + debug_info->op_name_ = anf_node_ptr->fullname_with_scope(); + debug_info->task_num_ = task_info_ptrs.size(); + debug_info->stream_id_ = task_info_ptrs[0]->stream_id(); + debug_info->dump_flag_ = task_info_ptrs[0]->dump_flag(); + debug_info->input_addrs_ = kernel_inputs; + debug_info->output_addrs_ = kernel_outputs; + debug_info->workspace_addrs_ = kernel_workspaces; + task_debug_info_list_.push_back(debug_info); return true; } @@ -191,8 +208,87 @@ bool TaskGenerator::LaunchAllKernel(const std::vector &anf_node_list, if (ProfilingManager::GetInstance().IsProfiling()) { ProfilingUtils::SetGraphProfilingCNode(graph_id, profiling_cnode_list); } + return true; } + +#ifdef ENABLE_DUMP_IR +void TaskGenerator::DumpTaskInfo(const std::string &real_filename) { + if (real_filename.size() > PATH_MAX) { + MS_LOG(ERROR) << "File path " << real_filename << " is too long."; + return; + } + char real_path[PATH_MAX] = {0}; +#if defined(_WIN32) || defined(_WIN64) + if (_fullpath(real_path, filename.c_str(), PATH_MAX) == nullptr) { + MS_LOG(DEBUG) << "dir " << filename << " does not exit."; + } +#else + if (nullptr == realpath(real_filename.c_str(), real_path)) { + MS_LOG(DEBUG) << "Dir " << real_filename << " does not exit."; + } +#endif + + OrderedMap para_map; + std::string path_string = real_path; + ChangeFileMode(path_string, S_IRWXU); + std::ofstream fout(real_path); + + if (!fout.is_open()) { + MS_LOG(ERROR) << "Open dump file '" << real_path << "' failed!"; + return; + } + + size_t index = 0; + for (auto &task_debug_info : task_debug_info_list_) { + fout << "op_name:" << task_debug_info->op_name_ << "\n" + << "task_index:" << index << "\t" + << "task_num:" << task_debug_info->task_num_ << "\t" + << "task0_stream_id:" << task_debug_info->stream_id_ << "\t" + << "task0_type:" << task_debug_info->type_ << "\t" + << "task0_dump_flag:" << task_debug_info->dump_flag_ << "\n"; + index++; + if (task_debug_info->input_addrs_.size()) { + fout << "input address:"; + for (auto &input : task_debug_info->input_addrs_) { + fout << input->addr << "(" << input->size << ")\t"; + } + fout << "\n"; + } + + if (task_debug_info->output_addrs_.size()) { + fout << "output address:"; + for (auto &output : task_debug_info->output_addrs_) { + fout << output->addr << "(" << output->size << ")\t"; + } + fout << "\n"; + } + + if (task_debug_info->workspace_addrs_.size()) { + fout << "workspace address:"; + for (auto &workspace : task_debug_info->workspace_addrs_) { + fout << workspace->addr << "(" << workspace->size << ")\t"; + } + fout << "\n"; + } + fout << "\n"; + } + + fout.close(); + // set file mode to read only by user + ChangeFileMode(path_string, S_IRUSR); +} +#else +void TaskGenerator::DumpTaskInfo(const std::string &real_filename) { + static bool already_printed = false; + if (already_printed) { + return; + } + already_printed = true; + MS_LOG(WARNING) << "The functionality of dumping function graph IR is disabled, " + << "please recompile source to enable it. See help of building script."; +} +#endif } // namespace tasksink } // namespace ascend } // namespace device diff --git a/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.h b/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.h index e4206ac5a71..b058fbbebeb 100644 --- a/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.h +++ b/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.h @@ -36,6 +36,19 @@ using mindspore::kernel::AddressPtr; using AddressPtrList = std::vector; using ge::model_runner::TaskInfo; using TaskInfoPtr = std::shared_ptr; +class TaskDebugInfo { + public: + std::string op_name_; + std::size_t task_num_{0}; + uint32_t stream_id_{0}; + uint32_t type_{0}; + bool dump_flag_{false}; + std::vector input_addrs_; + std::vector output_addrs_; + std::vector workspace_addrs_; +}; +using TaskDebugInfoPtr = std::shared_ptr; + class TaskGenerator { public: TaskGenerator() = default; @@ -43,15 +56,17 @@ class TaskGenerator { TaskGenerator(const TaskGenerator &in) = delete; TaskGenerator &operator=(const TaskGenerator &in) = delete; - static bool GenTasks(const std::vector &anf_node_list, std::vector *task_info_list, - uint32_t graph_id); + bool GenTasks(const std::vector &anf_node_list, std::vector *task_info_list, + uint32_t graph_id); private: + std::vector task_debug_info_list_; static void LaunchAddrCleanKernel(const CNodePtr &anf_node_ptr, AddressPtrList *kernel_inputs); static void LaunchAddrCleanAkgKernel(const CNodePtr &anf_node_ptr, AddressPtrList *kernel_inputs); - static bool LaunchKernel(const CNodePtr &anf_node_ptr, uint32_t stream_id, std::vector *task_info_list); - static bool LaunchAllKernel(const std::vector &anf_node_list, std::vector *task_info_list, - uint32_t graph_id); + bool LaunchKernel(const CNodePtr &anf_node_ptr, uint32_t stream_id, std::vector *task_info_list); + bool LaunchAllKernel(const std::vector &anf_node_list, std::vector *task_info_list, + uint32_t graph_id); + void DumpTaskInfo(const string &real_filename); }; } // namespace tasksink } // namespace ascend