From 7c3994e48e9bc540a3c3b1cb7cf537209c305d54 Mon Sep 17 00:00:00 2001 From: yelihua Date: Thu, 9 Sep 2021 19:25:29 +0800 Subject: [PATCH] use Common::CreatePrefixPath instead of Common::GetRealPath --- .../ccsrc/backend/session/session_basic.cc | 10 ++----- mindspore/ccsrc/debug/anf_ir_dump.cc | 4 +-- mindspore/ccsrc/debug/anf_ir_utils.cc | 2 +- mindspore/ccsrc/debug/common.cc | 4 +-- .../ccsrc/debug/data_dump/dump_json_parser.cc | 13 ++++----- mindspore/ccsrc/debug/data_dump/e2e_dump.cc | 10 +++---- mindspore/ccsrc/debug/debug_services.cc | 2 +- .../ccsrc/debug/debugger/proto_exporter.cc | 2 +- mindspore/ccsrc/debug/draw.cc | 6 ++--- mindspore/ccsrc/debug/dump_proto.cc | 4 +-- mindspore/ccsrc/debug/rdr/base_recorder.cc | 2 +- mindspore/ccsrc/debug/trace.cc | 4 +-- mindspore/ccsrc/utils/file_utils.cc | 2 +- tests/st/dump/dump_test_utils.py | 1 - tests/st/dump/test_data_dump.py | 27 +++++++++++++++++++ 15 files changed, 55 insertions(+), 38 deletions(-) diff --git a/mindspore/ccsrc/backend/session/session_basic.cc b/mindspore/ccsrc/backend/session/session_basic.cc index 3a1c19cf15d..49cf768eda1 100644 --- a/mindspore/ccsrc/backend/session/session_basic.cc +++ b/mindspore/ccsrc/backend/session/session_basic.cc @@ -2677,14 +2677,8 @@ void SessionBasic::InitPSParamAndOptim(const KernelGraphPtr &kernel_graph, } // namespace session void DumpGraphExeOrder(const std::string &file_name, const std::string &target_dir, const std::vector &execution_order) { - auto dir_path = FileUtils::CreateNotExistDirs(target_dir + "/execution_order/"); - if (!dir_path.has_value()) { - MS_LOG(ERROR) << "Failed to CreateNotExistDirs: " << target_dir << "/execution_order/ in DumpGraphExeOrder"; - return; - } - - std::string file_path = dir_path.value() + "/" + file_name; - auto realpath = FileUtils::GetRealPath(common::SafeCStr(file_path)); + std::string file_path = target_dir + "/execution_order/" + file_name; + auto realpath = Common::CreatePrefixPath(file_path); if (!realpath.has_value()) { MS_LOG(ERROR) << "Get real path: " << file_path << " filed."; return; diff --git a/mindspore/ccsrc/debug/anf_ir_dump.cc b/mindspore/ccsrc/debug/anf_ir_dump.cc index 992747a575d..c9d82737fce 100644 --- a/mindspore/ccsrc/debug/anf_ir_dump.cc +++ b/mindspore/ccsrc/debug/anf_ir_dump.cc @@ -598,7 +598,7 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu if (!target_file.empty()) { path = target_file; } - auto realpath = Common::GetRealPath(path); + auto realpath = Common::CreatePrefixPath(path); if (!realpath.has_value()) { MS_LOG(ERROR) << "Get real path failed, path=" << path; return; @@ -641,7 +641,7 @@ void DumpIRForRDR(const std::string &filename, const FuncGraphPtr &graph, bool d return; } auto path = Common::AddId(filename, ".ir"); - auto realpath = Common::GetRealPath(path); + auto realpath = Common::CreatePrefixPath(path); if (!realpath.has_value()) { MS_LOG(ERROR) << "Get real path failed. path=" << path; return; diff --git a/mindspore/ccsrc/debug/anf_ir_utils.cc b/mindspore/ccsrc/debug/anf_ir_utils.cc index 3e24b7ec834..96ff0724fcc 100644 --- a/mindspore/ccsrc/debug/anf_ir_utils.cc +++ b/mindspore/ccsrc/debug/anf_ir_utils.cc @@ -644,7 +644,7 @@ void ExportIR(const std::string &filename, const FuncGraphPtr &func_graph) { } auto filepath = GetSaveGraphsPathName(Common::AddId(filename, ".dat")); - auto real_filepath = Common::GetRealPath(filepath); + auto real_filepath = Common::CreatePrefixPath(filepath); if (!real_filepath.has_value()) { MS_LOG(ERROR) << "The export ir path: " << filepath << " is not illegal."; return; diff --git a/mindspore/ccsrc/debug/common.cc b/mindspore/ccsrc/debug/common.cc index f27ddddfed8..2ff59e17137 100644 --- a/mindspore/ccsrc/debug/common.cc +++ b/mindspore/ccsrc/debug/common.cc @@ -382,14 +382,14 @@ struct GlogLogDirRegister { both_exist = true; } log_dir_str += "/rank_" + rank + "/logs"; - auto real_log_dir_str = Common::GetRealPath(log_dir_str); + auto real_log_dir_str = Common::CreatePrefixPath(log_dir_str); // While 'GLOG_logtostderr' = 0, logs output to files. 'GLOG_log_dir' must be specified as the path of log files. // Here can not throw exception and use python to catch, because the PYBIND11_MODULE is not yet been initialed. if (logtostderr_str == "0" && real_log_dir_str.has_value()) { if (!Common::IsPathValid(real_log_dir_str.value(), MAX_DIRECTORY_LENGTH, "")) { MS_LOG(ERROR) << "The path of log files, which set by 'GLOG_log_dir', is invalid"; exit(EXIT_FAILURE); - } else if (!Common::CreateNotExistDirs(real_log_dir_str.value())) { + } else if (!FileUtils::CreateNotExistDirs(real_log_dir_str.value())) { MS_LOG(ERROR) << "Create the path of log files, which set by 'GLOG_log_dir', failed."; exit(EXIT_FAILURE); } diff --git a/mindspore/ccsrc/debug/data_dump/dump_json_parser.cc b/mindspore/ccsrc/debug/data_dump/dump_json_parser.cc index 3cde1cb107d..9a676aecb5d 100644 --- a/mindspore/ccsrc/debug/data_dump/dump_json_parser.cc +++ b/mindspore/ccsrc/debug/data_dump/dump_json_parser.cc @@ -90,11 +90,7 @@ void DumpJsonParser::Parse() { MS_LOG(EXCEPTION) << "Get dump config file failed"; } - auto dump_file_realpath = Common::GetRealPath(dump_config_file.value()); - if (!dump_file_realpath.has_value()) { - MS_LOG(EXCEPTION) << "Get real path failed in Parse."; - } - std::ifstream json_file(dump_file_realpath.value()); + std::ifstream json_file(dump_config_file.value()); if (!json_file.is_open()) { MS_LOG(EXCEPTION) << "Dump file:" << dump_config_file.value() << " open failed." << " Errno:" << errno << " ErrInfo:" << strerror(errno); @@ -132,7 +128,8 @@ void DumpJsonParser::CopyJsonToDir(uint32_t rank_id) { } std::ifstream json_file(dump_config_file.value()); if (async_dump_enabled_ || e2e_dump_enabled_) { - auto realpath = Common::GetRealPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/data_dump.json"); + auto realpath = + Common::CreatePrefixPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/data_dump.json"); if (!realpath.has_value()) { MS_LOG(ERROR) << "Get real path failed in CopyJsonDir."; } else { @@ -162,7 +159,7 @@ void DumpJsonParser::CopyHcclJsonToDir(uint32_t rank_id) { } } std::ifstream json_file(config_path); - auto realpath = Common::GetRealPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/hccl.json"); + auto realpath = Common::CreatePrefixPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/hccl.json"); if (!realpath.has_value()) { MS_LOG(ERROR) << "Get real path failed in CopyHcclJsonToDir."; } else { @@ -182,7 +179,7 @@ void DumpJsonParser::CopyMSCfgJsonToDir(uint32_t rank_id) { if (!IsDumpEnabled()) { return; } - auto realpath = Common::GetRealPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/config.json"); + auto realpath = Common::CreatePrefixPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/config.json"); if (!realpath.has_value()) { MS_LOG(ERROR) << "Get real path failed in CopyMSConfigJsonToDir."; } else { diff --git a/mindspore/ccsrc/debug/data_dump/e2e_dump.cc b/mindspore/ccsrc/debug/data_dump/e2e_dump.cc index 543c4ddcb2d..86833df684f 100644 --- a/mindspore/ccsrc/debug/data_dump/e2e_dump.cc +++ b/mindspore/ccsrc/debug/data_dump/e2e_dump.cc @@ -28,6 +28,7 @@ #include "utils/ms_context.h" #include "runtime/device/kernel_runtime_manager.h" #include "utils/config_manager.h" +#include "utils/file_utils.h" #ifdef ENABLE_DEBUGGER #include "debug/debug_services.h" #include "debug/tensor_load.h" @@ -301,10 +302,9 @@ void E2eDump::DumpSetup(const session::KernelGraph *graph, uint32_t rank_id) { MS_LOG(INFO) << "cur_iter_dump_path: " << cur_iter_dump_path; // create cur_iter_dump_path dirs - bool status = Common::CreateNotExistDirs(root_cur_iter_dump_path); - if (!status) { + auto dir_path = FileUtils::CreateNotExistDirs(root_cur_iter_dump_path); + if (!dir_path.has_value()) { MS_LOG(EXCEPTION) << "Failed at CreateNotExistDirs for " << root_cur_iter_dump_path; - return; } } } @@ -340,8 +340,8 @@ void E2eDump::DumpData(const session::KernelGraph *graph, uint32_t rank_id, cons if (dump_json_parser.IsDumpIter(current_iter)) { // create actual dir for iteration in final dump dir - bool status = Common::CreateNotExistDirs(cur_iter_dump_path); - if (!status) { + auto dir_path = FileUtils::CreateNotExistDirs(cur_iter_dump_path); + if (!dir_path.has_value()) { MS_LOG(EXCEPTION) << "failed at CreateNotExistDirs for " << cur_iter_dump_path; } diff --git a/mindspore/ccsrc/debug/debug_services.cc b/mindspore/ccsrc/debug/debug_services.cc index 50dc1af66d2..1559703a00f 100644 --- a/mindspore/ccsrc/debug/debug_services.cc +++ b/mindspore/ccsrc/debug/debug_services.cc @@ -1285,7 +1285,7 @@ bool DebugServices::CheckOpOverflow(std::string node_name_to_find, unsigned int overflow_bin_path = DumpJsonParser::GetInstance().GetOpOverflowBinPath(debugger->GetGraphPtr()->graph_id()); std::string check_overflow_bin_path = RealPath(overflow_bin_path); if (check_overflow_bin_path.empty()) { - MS_LOG(WARNING) << "Get real path failed for overflow_bin_path."; + MS_LOG(INFO) << "Get real path failed for overflow_bin_path."; return false; } overflow_bin_path = check_overflow_bin_path; diff --git a/mindspore/ccsrc/debug/debugger/proto_exporter.cc b/mindspore/ccsrc/debug/debugger/proto_exporter.cc index 3db363edcab..ca90c4cdeb5 100644 --- a/mindspore/ccsrc/debug/debugger/proto_exporter.cc +++ b/mindspore/ccsrc/debug/debugger/proto_exporter.cc @@ -563,7 +563,7 @@ void DumpIRProtoWithSrcInfo(const FuncGraphPtr &func_graph, const std::string &s return; } std::string file_path = target_dir + "/" + "ms_output_" + suffix + ".pb"; - auto realpath = Common::GetRealPath(file_path); + auto realpath = Common::CreatePrefixPath(file_path); if (!realpath.has_value()) { MS_LOG(ERROR) << "Get real path failed, path=" << file_path; return; diff --git a/mindspore/ccsrc/debug/draw.cc b/mindspore/ccsrc/debug/draw.cc index 769ee812a66..82391890ebb 100644 --- a/mindspore/ccsrc/debug/draw.cc +++ b/mindspore/ccsrc/debug/draw.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019-2020 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -189,7 +189,7 @@ void Draw(const std::string &filename, const FuncGraphPtr &func_graph) { const std::string filename_with_suffix = (filename.rfind(dot_suffix) != (filename.size() - dot_suffix.size())) ? (filename + dot_suffix) : filename; const std::string filepath = GetSaveGraphsPathName(Common::AddId(filename_with_suffix, dot_suffix)); - auto real_filepath = Common::GetRealPath(filepath); + auto real_filepath = Common::CreatePrefixPath(filepath); if (!real_filepath.has_value()) { MS_LOG(EXCEPTION) << "The export ir path: " << filepath << " is not illegal."; } @@ -199,7 +199,7 @@ void Draw(const std::string &filename, const FuncGraphPtr &func_graph) { void DrawUserFuncGraph(const std::string &filename, const FuncGraphPtr &func_graph) { const std::string dot_suffix = ".dot"; const std::string filepath = GetSaveGraphsPathName(Common::AddId(filename, dot_suffix)); - auto real_filepath = Common::GetRealPath(filepath); + auto real_filepath = Common::CreatePrefixPath(filepath); if (!real_filepath.has_value()) { MS_LOG(EXCEPTION) << "The export ir path: " << filepath << " is not illegal."; } diff --git a/mindspore/ccsrc/debug/dump_proto.cc b/mindspore/ccsrc/debug/dump_proto.cc index 8ce69fae311..c232296d64f 100644 --- a/mindspore/ccsrc/debug/dump_proto.cc +++ b/mindspore/ccsrc/debug/dump_proto.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -548,7 +548,7 @@ void DumpIRProto(const FuncGraphPtr &func_graph, const std::string &suffix) { return; } std::string file_path = GetSaveGraphsPathName("ms_output_" + suffix + ".pb"); - auto realpath = Common::GetRealPath(file_path); + auto realpath = Common::CreatePrefixPath(file_path); if (!realpath.has_value()) { MS_LOG(ERROR) << "Get real path failed, path=" << file_path; return; diff --git a/mindspore/ccsrc/debug/rdr/base_recorder.cc b/mindspore/ccsrc/debug/rdr/base_recorder.cc index 11a4fc08736..fea8be548b7 100644 --- a/mindspore/ccsrc/debug/rdr/base_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/base_recorder.cc @@ -43,7 +43,7 @@ std::optional BaseRecorder::GetFileRealPath(const std::string &suff if (config_file.empty()) { file_path = directory_ + "rank_" + std::to_string(GetRank()) + "/rdr/" + filename; } - auto realpath = Common::GetRealPath(file_path); + auto realpath = Common::CreatePrefixPath(file_path); if (!realpath.has_value()) { MS_LOG(ERROR) << "Get real path failed. " << "Info: module=" << module_ << ", name=" << name_ << ", " diff --git a/mindspore/ccsrc/debug/trace.cc b/mindspore/ccsrc/debug/trace.cc index 51737df9290..92ee04c7446 100644 --- a/mindspore/ccsrc/debug/trace.cc +++ b/mindspore/ccsrc/debug/trace.cc @@ -350,7 +350,7 @@ bool AnalyzeFailExporter::ExportFuncGraph(const std::string &filename, const Tra MS_LOG(DEBUG) << "Node configs is empty"; return false; } - auto real_filepath = Common::GetRealPath(filename); + auto real_filepath = Common::CreatePrefixPath(filename); if (!real_filepath.has_value()) { MS_LOG(ERROR) << "The export ir path: " << filename << " is not illegal."; return false; @@ -413,7 +413,7 @@ std::string GetEvalFailDatPath() { path = "."; } path += "/rank_" + std::to_string(GetRank()) + "/om/analyze_fail.dat"; - auto realpath = Common::GetRealPath(path); + auto realpath = Common::CreatePrefixPath(path); if (!realpath.has_value()) { MS_EXCEPTION(ValueError) << "Get real path failed. path=" << path; } diff --git a/mindspore/ccsrc/utils/file_utils.cc b/mindspore/ccsrc/utils/file_utils.cc index 3c9fc1ba3f6..3714886ff3a 100644 --- a/mindspore/ccsrc/utils/file_utils.cc +++ b/mindspore/ccsrc/utils/file_utils.cc @@ -59,7 +59,7 @@ void FileUtils::SplitDirAndFileName(const std::string &path, std::optional