diff --git a/mindspore/ccsrc/backend/optimizer/somas/somas.cc b/mindspore/ccsrc/backend/optimizer/somas/somas.cc index 122c6d6129e..3dc798b618d 100644 --- a/mindspore/ccsrc/backend/optimizer/somas/somas.cc +++ b/mindspore/ccsrc/backend/optimizer/somas/somas.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,6 +34,7 @@ #include "backend/optimizer/common/helper.h" #include "utils/ms_context.h" #include "debug/common.h" +#include "debug/rdr/running_data_recorder.h" #include "common/thread_pool.h" #include "profiler/device/common/memory_profiling.h" @@ -98,6 +99,15 @@ bool Somas::InitSomasTensors(const session::KernelGraph *graph) { << nodes_list_.size() << " nodes, " << tensors_list_.size() << " tensors, and " << contiguous_tensors_list_.size() << " contiguous lists"; +#ifdef ENABLE_DUMP_IR + SubModuleId module = SubModuleId::SM_OPTIMIZER; + std::string tag = "somas"; + std::string filename = "somas_pre_processed_info_" + std::to_string(graph->graph_id()) + ".ir"; + mindspore::RDR::RecordString(module, tag, SomasInfo(), filename); + filename = "somas_offline_log_" + std::to_string(graph->graph_id()) + ".ir"; + mindspore::RDR::RecordString(module, tag, Offline(), filename); +#endif + if (save_graphs_) { std::string file_path = save_graphs_path_ + "/" + "somas_pre_processed_info_" + std::to_string(graph->graph_id()) + ".ir"; @@ -380,6 +390,14 @@ void Somas::InitBasicInfo(const session::KernelGraph *graph) { auto context_ptr = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context_ptr); + +#ifdef ENABLE_DUMP_IR + SubModuleId module = SubModuleId::SM_OPTIMIZER; + std::string tag = "somas"; + std::string filename = "somas_initial_info_" + std::to_string(graph->graph_id()) + ".ir"; + mindspore::RDR::RecordString(module, tag, SomasInfo(), filename); +#endif + save_graphs_ = context_ptr->get_param(MS_CTX_SAVE_GRAPHS_FLAG); save_graphs_path_ = context_ptr->get_param(MS_CTX_SAVE_GRAPHS_PATH); if (save_graphs_path_.empty()) { @@ -1089,6 +1107,121 @@ std::string Somas::GetSplitName(const std::string &scope_name) const { } } +std::string Somas::SomasInfo() { + std::ostringstream oss; + + DumpParameters(oss); + DumpTensors(oss); + DumpNodes(oss); + + oss << "\n\nAll Stream Groups:\n\n"; + for (const auto &stream_group : streams_groups_) { + for (const auto &stream : stream_group) { + oss << "stm" << stream << " "; + } + oss << "\n"; + } + + if (!ref_node_constraints_.empty()) { + oss << "\n\nAll Ref Node Info:\n\n"; + for (const auto &ref_in_out : ref_node_constraints_) { + oss << "refnode input-output:"; + for (const auto &item : ref_in_out) { + oss << "%" << item << "T "; + } + oss << "\n"; + } + } + return oss.str(); +} + +void Somas::DumpNodes(std::ostringstream &oss) const { + oss << "\n\nAll Nodes:\n\n"; + for (const auto &node : nodes_list_) { + auto scope_name = node->scope_full_name_; + std::string split_name = GetSplitName(scope_name); + oss << "$" << node->GetId() << "\t" << split_name << "\t" << static_cast(node->GetType()) << "\t"; + auto input_num = node->input_tensors_.size() + node->input_parameters_map_.size(); + oss << "inputs["; + size_t tensor_index = 0; + for (size_t input_index = 0; input_index < input_num; input_index++) { + auto iter = node->input_parameters_map_.find(input_index); + if (iter != node->input_parameters_map_.end()) { + oss << "%" << iter->second->id_ << "P" + << ", "; + } else { + oss << "%" << node->input_tensors_[tensor_index]->GetId() << "T" + << ", "; + tensor_index++; + } + } + + oss << "]"; + oss << "\toutputs["; + for (const auto &out : node->output_tensors_) { + oss << "%" << out->GetId() << "T" + << ", "; + } + oss << "]"; + oss << "\tworkspace["; + for (const auto &wk : node->workspace_tensors_) { + oss << "%" << wk->GetId() << "T" + << ", "; + } + oss << "]"; + oss << "\tstreamID[" + << "@" << node->GetStream()->GetId() << "]\n"; + } +} + +void Somas::DumpTensors(std::ostringstream &oss) const { + oss << "\n\nAll Tensors:\n\n"; + oss << "index:" + << "\tsize:" + << "\treal_size:" + << "\toffset:" + << "\taddr:" + << "\ttype:" + << "\tlifelong:" + << "\tlife_start:" + << "\tlife_end:" + << "\tsource node name:\n"; + + for (const auto &tensor : tensors_list_) { + auto scope_name = tensor->GetSourceNode()->scope_full_name_; + std::string split_name = GetSplitName(scope_name); + oss << "%" << tensor->GetId() << "T" + << "\t" + << "#" << tensor->GetAlignedSize() << "S" + << "\t" + << "#" << tensor->GetOriginalSize() << "S" + << "\t" + << "&" << tensor->GetOffset() << "" + << "\t" + << "&" << static_cast(tensor->GetOffset() + mem_base_addr_) << "\t" + << tensor_type_name_map[tensor->type_] << "\t" << tensor->IsLifelong() << "\t" << tensor->lifetime_.start_ + << "\t" << tensor->lifetime_.end_ << "\t" << split_name << "\n"; + } +} + +void Somas::DumpParameters(std::ostringstream &oss) const { + oss << "All Parameters:\n\n"; + oss << "index:" + << "\tsize:" + << "\tstart_addr:" + << "\tsource node name:" + << "\tnode out index:\n"; + + for (const auto ¶m : parameters_list_) { + oss << "%" << param->id_ << "P" + << "\t" + << "#" << param->size_ << "S" + << "\t" + << "&" << param->addr_ << "\t" << param->source_node_->fullname_with_scope() << "\t" << param->output_index_ + << "\n"; + } +} + void Somas::DumpSomasInfoIR(const string filename) { if (filename.size() > PATH_MAX) { MS_LOG(ERROR) << "File path " << filename << " is too long."; @@ -1108,115 +1241,52 @@ void Somas::DumpSomasInfoIR(const string filename) { return; } - DumpParameters(ofs); - DumpTensors(ofs); - DumpNodes(ofs); - - ofs << "\n\nAll Stream Groups:\n\n"; - for (const auto &stream_group : streams_groups_) { - for (const auto &stream : stream_group) { - ofs << "stm" << stream << " "; - } - ofs << "\n"; - } - - if (!ref_node_constraints_.empty()) { - ofs << "\n\nAll Ref Node Info:\n\n"; - for (const auto &ref_in_out : ref_node_constraints_) { - ofs << "refnode input-output:"; - for (const auto &item : ref_in_out) { - ofs << "%" << item << "T "; - } - ofs << "\n"; - } - } + ofs << SomasInfo(); + ofs.close(); } -void Somas::DumpNodes(std::ofstream &ofs) const { - ofs << "\n\nAll Nodes:\n\n"; - for (const auto &node : nodes_list_) { - auto scope_name = node->scope_full_name_; - std::string split_name = GetSplitName(scope_name); - ofs << "$" << node->GetId() << "\t" << split_name << "\t" << static_cast(node->GetType()) << "\t"; - auto input_num = node->input_tensors_.size() + node->input_parameters_map_.size(); - ofs << "inputs["; - size_t tensor_index = 0; - for (size_t input_index = 0; input_index < input_num; input_index++) { - auto iter = node->input_parameters_map_.find(input_index); - if (iter != node->input_parameters_map_.end()) { - ofs << "%" << iter->second->id_ << "P" - << ", "; - } else { - ofs << "%" << node->input_tensors_[tensor_index]->GetId() << "T" - << ", "; - tensor_index++; +std::string Somas::Offline() { + std::ostringstream oss; + + for (auto tensor : tensors_list_) { + if (tensor->IsOutputOnly() || tensor->type_ == TensorType::kRefNodeOutput) { + oss << "Somas EDGE ERROR src=n" << tensor->GetSourceNode()->GetId() + << ", srcstm=" << tensor->GetSourceStream()->GetId() << ", dst=nc" + << ", dststm=nc" + << ", workspace=0, size=" << tensor->GetOriginalSize() + << ", lifelong=" << static_cast(tensor->lifelong_value_) << ", tid=" << tensor->GetId() + << ", start=" << tensor->lifetime_.start_ << ", end=" << tensor->lifetime_.end_ << std::endl; + } else { + std::map dest_infos; + for (SomasNodePtr dest_node : tensor->destinations_) { + dest_infos.insert(std::make_pair(dest_node->GetId(), dest_node->GetStream()->GetId())); + } + + for (auto dest_info : dest_infos) { + oss << "Somas EDGE src=n" << tensor->GetSourceNode()->GetId() + << ", srcstm=" << tensor->GetSourceStream()->GetId() << ", dst=n" << dest_info.first + << ", dststm=" << dest_info.second << ", workspace=" << static_cast(tensor->type_ == kWorkspace) + << ", size=" << tensor->GetOriginalSize() << ", lifelong=" << static_cast(tensor->lifelong_value_) + << ", tid=" << tensor->GetId() << ", start=" << tensor->lifetime_.start_ + << ", end=" << tensor->lifetime_.end_ << std::endl; } } - - ofs << "]"; - ofs << "\toutputs["; - for (const auto &out : node->output_tensors_) { - ofs << "%" << out->GetId() << "T" - << ", "; + } + for (vector tList : contiguous_tensors_list_) { + oss << "Somas CONTIGUOUS"; + for (size_t tid : tList) { + oss << " " << tid; } - ofs << "]"; - ofs << "\tworkspace["; - for (const auto &wk : node->workspace_tensors_) { - ofs << "%" << wk->GetId() << "T" - << ", "; + oss << std::endl; + } + for (const auto &group : streams_groups_) { + oss << "Somas GROUP"; + for (int64_t sid : group) { + oss << " " << sid; } - ofs << "]"; - ofs << "\tstreamID[" - << "@" << node->GetStream()->GetId() << "]\n"; - } -} - -void Somas::DumpTensors(std::ofstream &ofs) const { - ofs << "\n\nAll Tensors:\n\n"; - ofs << "index:" - << "\tsize:" - << "\treal_size:" - << "\toffset:" - << "\taddr:" - << "\ttype:" - << "\tlifelong:" - << "\tlife_start:" - << "\tlife_end:" - << "\tsource node name:\n"; - - for (const auto &tensor : tensors_list_) { - auto scope_name = tensor->GetSourceNode()->scope_full_name_; - std::string split_name = GetSplitName(scope_name); - ofs << "%" << tensor->GetId() << "T" - << "\t" - << "#" << tensor->GetAlignedSize() << "S" - << "\t" - << "#" << tensor->GetOriginalSize() << "S" - << "\t" - << "&" << tensor->GetOffset() << "" - << "\t" - << "&" << static_cast(tensor->GetOffset() + mem_base_addr_) << "\t" - << tensor_type_name_map[tensor->type_] << "\t" << tensor->IsLifelong() << "\t" << tensor->lifetime_.start_ - << "\t" << tensor->lifetime_.end_ << "\t" << split_name << "\n"; - } -} - -void Somas::DumpParameters(std::ofstream &ofs) const { - ofs << "All Parameters:\n\n"; - ofs << "index:" - << "\tsize:" - << "\tstart_addr:" - << "\tsource node name:" - << "\tnode out index:\n"; - - for (const auto ¶m : parameters_list_) { - ofs << "%" << param->id_ << "P" - << "\t" - << "#" << param->size_ << "S" - << "\t" - << "&" << param->addr_ << "\t" << param->source_node_->fullname_with_scope() << "\t" << param->output_index_ - << "\n"; + oss << std::endl; } + return oss.str(); } void Somas::DumpOfflineIR(const string filename) { @@ -1240,66 +1310,12 @@ void Somas::DumpOfflineIR(const string filename) { return; } - for (auto tensor : tensors_list_) { - if (tensor->IsOutputOnly() || tensor->type_ == TensorType::kRefNodeOutput) { - ofs << "Somas EDGE ERROR src=n" << tensor->GetSourceNode()->GetId() - << ", srcstm=" << tensor->GetSourceStream()->GetId() << ", dst=nc" - << ", dststm=nc" - << ", workspace=0, size=" << tensor->GetOriginalSize() - << ", lifelong=" << static_cast(tensor->lifelong_value_) << ", tid=" << tensor->GetId() - << ", start=" << tensor->lifetime_.start_ << ", end=" << tensor->lifetime_.end_ << std::endl; - } else { - std::map dest_infos; - for (SomasNodePtr dest_node : tensor->destinations_) { - dest_infos.insert(std::make_pair(dest_node->GetId(), dest_node->GetStream()->GetId())); - } - - for (auto dest_info : dest_infos) { - ofs << "Somas EDGE src=n" << tensor->GetSourceNode()->GetId() - << ", srcstm=" << tensor->GetSourceStream()->GetId() << ", dst=n" << dest_info.first - << ", dststm=" << dest_info.second << ", workspace=" << static_cast(tensor->type_ == kWorkspace) - << ", size=" << tensor->GetOriginalSize() << ", lifelong=" << static_cast(tensor->lifelong_value_) - << ", tid=" << tensor->GetId() << ", start=" << tensor->lifetime_.start_ - << ", end=" << tensor->lifetime_.end_ << std::endl; - } - } - } - for (vector tList : contiguous_tensors_list_) { - ofs << "Somas CONTIGUOUS"; - for (size_t tid : tList) { - ofs << " " << tid; - } - ofs << std::endl; - } - for (const auto &group : streams_groups_) { - ofs << "Somas GROUP"; - for (int64_t sid : group) { - ofs << " " << sid; - } - ofs << std::endl; - } + ofs << Offline(); ofs.close(); } -void Somas::DumpSomasMemoryIR(const string filename) { - if (filename.size() > PATH_MAX) { - MS_LOG(ERROR) << "File path " << filename << " is too long."; - return; - } - - auto real_path = Common::GetRealPath(filename); - if (!real_path.has_value()) { - MS_LOG(ERROR) << "Get real path failed. path=" << filename; - return; - } - - ChangeFileMode(real_path.value(), S_IRWXU); - std::ofstream ofs(real_path.value()); - - if (!ofs.is_open()) { - MS_LOG(ERROR) << "Open dump file '" << real_path.value() << "' failed!"; - return; - } +std::string Somas::SomasMemory() { + std::ostringstream oss; std::map mem_map; for (auto tensor : tensors_list_) { @@ -1325,7 +1341,7 @@ void Somas::DumpSomasMemoryIR(const string filename) { } } - ofs << "mem_id:" + oss << "mem_id:" << "\tstart_offset:" << "\tend_offset:" << "\ttensor_id:" @@ -1353,7 +1369,7 @@ void Somas::DumpSomasMemoryIR(const string filename) { } std::string split_name = GetSplitName(scope_name); - ofs << "#" << mem_map[tensor->GetOffset()] << "\t" << tensor->GetOffset() << "\t" + oss << "#" << mem_map[tensor->GetOffset()] << "\t" << tensor->GetOffset() << "\t" << tensor->GetOffset() + tensor->GetAlignedSize() << "\t%" << tensor->GetId() << "T\t" << tensor->GetOriginalSize() << "\t" << tensor->GetAlignedSize() << "\t&" << static_cast(tensor->GetOffset() + mem_base_addr_) << "\t&" @@ -1362,6 +1378,31 @@ void Somas::DumpSomasMemoryIR(const string filename) { << tensor->lifetime_.start_ << "\t" << tensor->lifetime_.end_ << "\n"; } } + return oss.str(); +} + +void Somas::DumpSomasMemoryIR(const string filename) { + if (filename.size() > PATH_MAX) { + MS_LOG(ERROR) << "File path " << filename << " is too long."; + return; + } + + auto real_path = Common::GetRealPath(filename); + if (!real_path.has_value()) { + MS_LOG(ERROR) << "Get real path failed. path=" << filename; + return; + } + + ChangeFileMode(real_path.value(), S_IRWXU); + std::ofstream ofs(real_path.value()); + + if (!ofs.is_open()) { + MS_LOG(ERROR) << "Open dump file '" << real_path.value() << "' failed!"; + return; + } + + ofs << SomasMemory(); + ofs.close(); } size_t Somas::CalcLowerBound() const { diff --git a/mindspore/ccsrc/backend/optimizer/somas/somas.h b/mindspore/ccsrc/backend/optimizer/somas/somas.h index 7e7269e298d..74a8a4e980b 100644 --- a/mindspore/ccsrc/backend/optimizer/somas/somas.h +++ b/mindspore/ccsrc/backend/optimizer/somas/somas.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,6 +49,8 @@ class Somas { uint8_t *GetNodeOutputPtr(const AnfNodePtr &node, size_t index) const; uint8_t *GetNodeWorkSpacePtr(const AnfNodePtr &node, size_t index) const; + std::string SomasInfo(); + std::string SomasMemory(); void DumpSomasInfoIR(const string filename); void DumpSomasMemoryIR(const string filename); @@ -121,8 +123,8 @@ class Somas { bool Assign(const session::KernelGraph *graph); + std::string Offline(); void DumpOfflineIR(const string filename); - void DumpSomasMemoryPoolInfoIR(const string filename); std::string GetSplitName(const string &scope_name) const; size_t CalcLowerBound() const; void GenGraphStatisticInfo(); @@ -143,9 +145,9 @@ class Somas { void UpdateRefOverlapTensorsConflicts(); void UpdateRefTensorsOffset(); void UpdateContiguousTensorsOffset(const std::map &contiguous_ref_list_map); - void DumpParameters(std::ofstream &ofs) const; - void DumpTensors(std::ofstream &ofs) const; - void DumpNodes(std::ofstream &ofs) const; + void DumpParameters(std::ostringstream &oss) const; + void DumpTensors(std::ostringstream &oss) const; + void DumpNodes(std::ostringstream &oss) const; std::map GetContiguousListContainRefTensor(); std::map GetRefTensorsInContiguousList(); }; diff --git a/mindspore/ccsrc/debug/CMakeLists.txt b/mindspore/ccsrc/debug/CMakeLists.txt index a1bca2f9fca..81ccb31530f 100644 --- a/mindspore/ccsrc/debug/CMakeLists.txt +++ b/mindspore/ccsrc/debug/CMakeLists.txt @@ -8,7 +8,7 @@ set(_DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/env_config_parser.cc" "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_exec_order_recorder.cc" "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_recorder.cc" - "${CMAKE_CURRENT_SOURCE_DIR}/rdr/somas_recorder.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/rdr/string_recorder.cc" "${CMAKE_CURRENT_SOURCE_DIR}/rdr/recorder_manager.cc" "${CMAKE_CURRENT_SOURCE_DIR}/rdr/running_data_recorder.cc" ) diff --git a/mindspore/ccsrc/debug/rdr/running_data_recorder.cc b/mindspore/ccsrc/debug/rdr/running_data_recorder.cc index b9c1a406f35..a140dc69b11 100644 --- a/mindspore/ccsrc/debug/rdr/running_data_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/running_data_recorder.cc @@ -16,9 +16,9 @@ #include "debug/rdr/running_data_recorder.h" #include #include "debug/rdr/graph_recorder.h" -#include "debug/rdr/somas_recorder.h" #include "debug/rdr/graph_exec_order_recorder.h" #include "debug/rdr/recorder_manager.h" +#include "debug/rdr/string_recorder.h" #include "mindspore/core/ir/func_graph.h" #include "mindspore/core/ir/anf.h" @@ -75,15 +75,16 @@ bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, return ans; } -bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id) { +bool RecordString(SubModuleId module, const std::string &tag, const std::string &data, const std::string &filename) { std::string submodule_name = std::string(GetSubModuleName(module)); - SomasRecorderPtr somas_recorder = std::make_shared(submodule_name, tag, somas_ptr, graph_id); - somas_recorder->GenString(); - bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(somas_recorder)); + StringRecorderPtr string_recorder = std::make_shared(submodule_name, tag, data, filename); + string_recorder->SetFilename(filename); + bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(string_recorder)); return ans; } void TriggerAll() { mindspore::RecorderManager::Instance().TriggerAll(); } + #else bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, const std::string &file_type, int graph_id) { @@ -107,7 +108,7 @@ bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, std: return false; } -bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id) { +bool RecordString(SubModuleId module, const std::string &tag, const std::string &data, const std::string &filename) { static bool already_printed = false; if (already_printed) { return false; @@ -116,6 +117,7 @@ bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const Som MS_LOG(WARNING) << "The RDR presently only support linux os."; return false; } + void TriggerAll() { static bool already_printed = false; if (already_printed) { diff --git a/mindspore/ccsrc/debug/rdr/running_data_recorder.h b/mindspore/ccsrc/debug/rdr/running_data_recorder.h index 36241b99fa4..fdc09a39d8a 100644 --- a/mindspore/ccsrc/debug/rdr/running_data_recorder.h +++ b/mindspore/ccsrc/debug/rdr/running_data_recorder.h @@ -13,28 +13,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef RUNNING_DATA_RECORDER_H_ -#define RUNNING_DATA_RECORDER_H_ +#ifndef MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_ +#define MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_ #include #include #include -#include "backend/optimizer/somas/somas.h" #include "mindspore/core/utils/log_adapter.h" namespace mindspore { class FuncGraph; class CNode; using FuncGraphPtr = std::shared_ptr; using CNodePtr = std::shared_ptr; -using SomasPtr = std::shared_ptr; namespace RDR { bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, const std::string &file_type = ".ir;.pb;.dat", int graph_id = 0); bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, const std::vector &&final_exec_order); -bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id); +bool RecordString(SubModuleId module, const std::string &tag, const std::string &data, + const std::string &filename = ""); void TriggerAll(); } // namespace RDR } // namespace mindspore -#endif // RUNNING_DATA_RECORDER_H_ +#endif // MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_ diff --git a/mindspore/ccsrc/debug/rdr/somas_recorder.cc b/mindspore/ccsrc/debug/rdr/somas_recorder.cc deleted file mode 100644 index 9b0b4cbc691..00000000000 --- a/mindspore/ccsrc/debug/rdr/somas_recorder.cc +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2021 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "debug/rdr/somas_recorder.h" -#include "backend/optimizer/somas/somas.h" -namespace mindspore { -void SomasRecorder::Export() { - if (filename_.empty()) { - filename_ = directory_ + module_ + "_" + tag_; - } - std::string filename = filename_ + "_somas_after_allocate_" + std::to_string(graph_id_) + "_" + timestamp_ + ".ir"; - somas_reuse_util_ptr_->DumpSomasInfoIR(filename); - std::string mem_filename = filename_ + "_somas_mem_info_" + std::to_string(graph_id_) + "_" + timestamp_ + ".ir"; - somas_reuse_util_ptr_->DumpSomasMemoryIR(mem_filename); -} - -bool SomasRecorder::GenString() { return true; } -} // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/string_recorder.cc b/mindspore/ccsrc/debug/rdr/string_recorder.cc new file mode 100644 index 00000000000..9d75c8620c8 --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/string_recorder.cc @@ -0,0 +1,51 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "debug/rdr/string_recorder.h" +#include +#include +#include "debug/common.h" +#include "utils/utils.h" +#include "mindspore/core/utils/log_adapter.h" + +namespace mindspore { +void StringRecorder::Export() { + if (directory_.back() != '/') { + directory_ += "/"; + } + + if (filename_.empty()) { + filename_ = module_ + "_" + tag_ + "_" + timestamp_ + ".txt"; + } + std::string file_path = directory_ + filename_; + + auto realpath = Common::GetRealPath(file_path); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed. path=" << file_path; + return; + } + + ChangeFileMode(realpath.value(), S_IRWXU); + std::ofstream fout(realpath.value(), std::ofstream::app); + if (!fout.is_open()) { + MS_LOG(WARNING) << "Open file for saving string failed."; + return; + } + fout << data_; + fout.close(); + // set file mode to read only by user + ChangeFileMode(realpath.value(), S_IRUSR); +} +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/somas_recorder.h b/mindspore/ccsrc/debug/rdr/string_recorder.h similarity index 51% rename from mindspore/ccsrc/debug/rdr/somas_recorder.h rename to mindspore/ccsrc/debug/rdr/string_recorder.h index 32fb3f8a535..578251adeb0 100644 --- a/mindspore/ccsrc/debug/rdr/somas_recorder.h +++ b/mindspore/ccsrc/debug/rdr/string_recorder.h @@ -13,34 +13,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef SOMAS_RECORDER_H_ -#define SOMAS_RECORDER_H_ +#ifndef MINDSPORE_CCSRC_DEBUG_RDR_STRING_RECORDER_H_ +#define MINDSPORE_CCSRC_DEBUG_RDR_STRING_RECORDER_H_ #include -#include #include -#include "debug/rdr/base_recorder.h" +#include "debug/rdr/base_recorder.h" namespace mindspore { -namespace somas { -class Somas; -using SomasPtr = std::shared_ptr; -} // namespace somas -class SomasRecorder : public BaseRecorder { +class StringRecorder : public BaseRecorder { public: - SomasRecorder() : BaseRecorder(), somas_reuse_util_ptr_(nullptr) {} - SomasRecorder(const std::string &module, const std::string &tag, const somas::SomasPtr &somas_reuse_util_ptr, - int graph_id) - : BaseRecorder(module, tag), somas_reuse_util_ptr_(somas_reuse_util_ptr), graph_id_(graph_id) {} + StringRecorder() : BaseRecorder() {} + StringRecorder(const std::string &module, const std::string &tag, const std::string &data, + const std::string &file_type) + : BaseRecorder(module, tag), data_(data) {} + ~StringRecorder() {} void SetModule(const std::string &module) { module_ = module; } - void SetSomas(const somas::SomasPtr &somas_reuse_util_ptr) { somas_reuse_util_ptr_ = somas_reuse_util_ptr; } - bool GenString(); + void SetFilename(const std::string &filename) { filename_ = filename; } virtual void Export(); private: - somas::SomasPtr somas_reuse_util_ptr_; - int graph_id_{0}; + std::string data_; }; -using SomasRecorderPtr = std::shared_ptr; +using StringRecorderPtr = std::shared_ptr; } // namespace mindspore -#endif // SOMAS_RECORDER_H +#endif // MINDSPORE_CCSRC_DEBUG_RDR_STRING_RECORDER_H_ diff --git a/mindspore/ccsrc/runtime/device/memory_manager.cc b/mindspore/ccsrc/runtime/device/memory_manager.cc index b7422fca4e7..1b57e99495d 100644 --- a/mindspore/ccsrc/runtime/device/memory_manager.cc +++ b/mindspore/ccsrc/runtime/device/memory_manager.cc @@ -17,6 +17,7 @@ #include "runtime/device/memory_manager.h" #include #include "backend/session/anf_runtime_algorithm.h" +#include "debug/rdr/running_data_recorder.h" #include "utils/ms_context.h" using mindspore::memreuse::BestFitMemReuse; @@ -69,6 +70,16 @@ void MemoryManager::MallocSomasDynamicMem(const session::KernelGraph *graph) { auto context_ptr = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context_ptr); +#ifdef ENABLE_DUMP_IR + SubModuleId module = SubModuleId::SM_OPTIMIZER; + std::string tag = "somas"; + + std::string filename = "somas_allocate_info_" + std::to_string(graph->graph_id()) + ".ir"; + mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasInfo(), filename); + + filename = "somas_mem_info_" + std::to_string(graph->graph_id()) + ".ir"; + mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasMemory(), filename); +#endif bool save_graphs = context_ptr->get_param(MS_CTX_SAVE_GRAPHS_FLAG); auto save_graphs_path = context_ptr->get_param(MS_CTX_SAVE_GRAPHS_PATH); if (save_graphs_path.empty()) {