p60975123
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!23186 use Common::CreatePrefixPath instead of Common::GetRealPath 

Merge pull request !23186 from yelihua/dev
This commit is contained in:
i-robot 2021-09-10 03:35:49 +00:00 committed by Gitee
parent d7cc1a1723 7c3994e48e
commit f1e92aeb00
15 changed files with 55 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
mindspore/ccsrc/backend/session/session_basic.cc
View File

@ -2677,14 +2677,8 @@ void SessionBasic::InitPSParamAndOptim(const KernelGraphPtr &kernel_graph,
} // namespace session
void DumpGraphExeOrder(const std::string &file_name, const std::string &target_dir,
const std::vector<CNodePtr> &execution_order) {
auto dir_path = FileUtils::CreateNotExistDirs(target_dir + "/execution_order/");
if (!dir_path.has_value()) {
MS_LOG(ERROR) << "Failed to CreateNotExistDirs: " << target_dir << "/execution_order/ in DumpGraphExeOrder";
return;
}
std::string file_path = dir_path.value() + "/" + file_name;
auto realpath = FileUtils::GetRealPath(common::SafeCStr(file_path));
std::string file_path = target_dir + "/execution_order/" + file_name;
auto realpath = Common::CreatePrefixPath(file_path);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path: " << file_path << " filed.";
return;

4
mindspore/ccsrc/debug/anf_ir_dump.cc
View File

@ -598,7 +598,7 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu
if (!target_file.empty()) {
path = target_file;
}
auto realpath = Common::GetRealPath(path);
auto realpath = Common::CreatePrefixPath(path);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << path;
return;
@ -641,7 +641,7 @@ void DumpIRForRDR(const std::string &filename, const FuncGraphPtr &graph, bool d
return;
}
auto path = Common::AddId(filename, ".ir");
auto realpath = Common::GetRealPath(path);
auto realpath = Common::CreatePrefixPath(path);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed. path=" << path;
return;

2
mindspore/ccsrc/debug/anf_ir_utils.cc
View File

@ -644,7 +644,7 @@ void ExportIR(const std::string &filename, const FuncGraphPtr &func_graph) {
}
auto filepath = GetSaveGraphsPathName(Common::AddId(filename, ".dat"));
auto real_filepath = Common::GetRealPath(filepath);
auto real_filepath = Common::CreatePrefixPath(filepath);
if (!real_filepath.has_value()) {
MS_LOG(ERROR) << "The export ir path: " << filepath << " is not illegal.";
return;

4
mindspore/ccsrc/debug/common.cc
View File

@ -382,14 +382,14 @@ struct GlogLogDirRegister {
both_exist = true;
}
log_dir_str += "/rank_" + rank + "/logs";
auto real_log_dir_str = Common::GetRealPath(log_dir_str);
auto real_log_dir_str = Common::CreatePrefixPath(log_dir_str);
// While 'GLOG_logtostderr' = 0, logs output to files. 'GLOG_log_dir' must be specified as the path of log files.
// Here can not throw exception and use python to catch, because the PYBIND11_MODULE is not yet been initialed.
if (logtostderr_str == "0" && real_log_dir_str.has_value()) {
if (!Common::IsPathValid(real_log_dir_str.value(), MAX_DIRECTORY_LENGTH, "")) {
MS_LOG(ERROR) << "The path of log files, which set by 'GLOG_log_dir', is invalid";
exit(EXIT_FAILURE);
} else if (!Common::CreateNotExistDirs(real_log_dir_str.value())) {
} else if (!FileUtils::CreateNotExistDirs(real_log_dir_str.value())) {
MS_LOG(ERROR) << "Create the path of log files, which set by 'GLOG_log_dir', failed.";
exit(EXIT_FAILURE);
}

13
mindspore/ccsrc/debug/data_dump/dump_json_parser.cc
View File

@ -90,11 +90,7 @@ void DumpJsonParser::Parse() {
MS_LOG(EXCEPTION) << "Get dump config file failed";
}
auto dump_file_realpath = Common::GetRealPath(dump_config_file.value());
if (!dump_file_realpath.has_value()) {
MS_LOG(EXCEPTION) << "Get real path failed in Parse.";
}
std::ifstream json_file(dump_file_realpath.value());
std::ifstream json_file(dump_config_file.value());
if (!json_file.is_open()) {
MS_LOG(EXCEPTION) << "Dump file:" << dump_config_file.value() << " open failed."
<< " Errno:" << errno << " ErrInfo:" << strerror(errno);
@ -132,7 +128,8 @@ void DumpJsonParser::CopyJsonToDir(uint32_t rank_id) {
}
std::ifstream json_file(dump_config_file.value());
if (async_dump_enabled_ || e2e_dump_enabled_) {
auto realpath = Common::GetRealPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/data_dump.json");
auto realpath =
Common::CreatePrefixPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/data_dump.json");
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed in CopyJsonDir.";
} else {
@ -162,7 +159,7 @@ void DumpJsonParser::CopyHcclJsonToDir(uint32_t rank_id) {
}
}
std::ifstream json_file(config_path);
auto realpath = Common::GetRealPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/hccl.json");
auto realpath = Common::CreatePrefixPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/hccl.json");
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed in CopyHcclJsonToDir.";
} else {
@ -182,7 +179,7 @@ void DumpJsonParser::CopyMSCfgJsonToDir(uint32_t rank_id) {
if (!IsDumpEnabled()) {
return;
}
auto realpath = Common::GetRealPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/config.json");
auto realpath = Common::CreatePrefixPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/config.json");
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed in CopyMSConfigJsonToDir.";
} else {

10
mindspore/ccsrc/debug/data_dump/e2e_dump.cc
View File

@ -28,6 +28,7 @@
#include "utils/ms_context.h"
#include "runtime/device/kernel_runtime_manager.h"
#include "utils/config_manager.h"
#include "utils/file_utils.h"
#ifdef ENABLE_DEBUGGER
#include "debug/debug_services.h"
#include "debug/tensor_load.h"
@ -301,10 +302,9 @@ void E2eDump::DumpSetup(const session::KernelGraph *graph, uint32_t rank_id) {
MS_LOG(INFO) << "cur_iter_dump_path: " << cur_iter_dump_path;
// create cur_iter_dump_path dirs
bool status = Common::CreateNotExistDirs(root_cur_iter_dump_path);
if (!status) {
auto dir_path = FileUtils::CreateNotExistDirs(root_cur_iter_dump_path);
if (!dir_path.has_value()) {
MS_LOG(EXCEPTION) << "Failed at CreateNotExistDirs for " << root_cur_iter_dump_path;
return;
}
}
}
@ -340,8 +340,8 @@ void E2eDump::DumpData(const session::KernelGraph *graph, uint32_t rank_id, cons
if (dump_json_parser.IsDumpIter(current_iter)) {
// create actual dir for iteration in final dump dir
bool status = Common::CreateNotExistDirs(cur_iter_dump_path);
if (!status) {
auto dir_path = FileUtils::CreateNotExistDirs(cur_iter_dump_path);
if (!dir_path.has_value()) {
MS_LOG(EXCEPTION) << "failed at CreateNotExistDirs for " << cur_iter_dump_path;
}

2
mindspore/ccsrc/debug/debug_services.cc
View File

@ -1285,7 +1285,7 @@ bool DebugServices::CheckOpOverflow(std::string node_name_to_find, unsigned int
overflow_bin_path = DumpJsonParser::GetInstance().GetOpOverflowBinPath(debugger->GetGraphPtr()->graph_id());
std::string check_overflow_bin_path = RealPath(overflow_bin_path);
if (check_overflow_bin_path.empty()) {
MS_LOG(WARNING) << "Get real path failed for overflow_bin_path.";
MS_LOG(INFO) << "Get real path failed for overflow_bin_path.";
return false;
}
overflow_bin_path = check_overflow_bin_path;

2
mindspore/ccsrc/debug/debugger/proto_exporter.cc
View File

@ -563,7 +563,7 @@ void DumpIRProtoWithSrcInfo(const FuncGraphPtr &func_graph, const std::string &s
return;
}
std::string file_path = target_dir + "/" + "ms_output_" + suffix + ".pb";
auto realpath = Common::GetRealPath(file_path);
auto realpath = Common::CreatePrefixPath(file_path);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file_path;
return;

6
mindspore/ccsrc/debug/draw.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -189,7 +189,7 @@ void Draw(const std::string &filename, const FuncGraphPtr &func_graph) {
const std::string filename_with_suffix =
(filename.rfind(dot_suffix) != (filename.size() - dot_suffix.size())) ? (filename + dot_suffix) : filename;
const std::string filepath = GetSaveGraphsPathName(Common::AddId(filename_with_suffix, dot_suffix));
auto real_filepath = Common::GetRealPath(filepath);
auto real_filepath = Common::CreatePrefixPath(filepath);
if (!real_filepath.has_value()) {
MS_LOG(EXCEPTION) << "The export ir path: " << filepath << " is not illegal.";
}
@ -199,7 +199,7 @@ void Draw(const std::string &filename, const FuncGraphPtr &func_graph) {
void DrawUserFuncGraph(const std::string &filename, const FuncGraphPtr &func_graph) {
const std::string dot_suffix = ".dot";
const std::string filepath = GetSaveGraphsPathName(Common::AddId(filename, dot_suffix));
auto real_filepath = Common::GetRealPath(filepath);
auto real_filepath = Common::CreatePrefixPath(filepath);
if (!real_filepath.has_value()) {
MS_LOG(EXCEPTION) << "The export ir path: " << filepath << " is not illegal.";
}

4
mindspore/ccsrc/debug/dump_proto.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -548,7 +548,7 @@ void DumpIRProto(const FuncGraphPtr &func_graph, const std::string &suffix) {
return;
}
std::string file_path = GetSaveGraphsPathName("ms_output_" + suffix + ".pb");
auto realpath = Common::GetRealPath(file_path);
auto realpath = Common::CreatePrefixPath(file_path);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file_path;
return;

2
mindspore/ccsrc/debug/rdr/base_recorder.cc
View File

@ -43,7 +43,7 @@ std::optional<std::string> BaseRecorder::GetFileRealPath(const std::string &suff
if (config_file.empty()) {
file_path = directory_ + "rank_" + std::to_string(GetRank()) + "/rdr/" + filename;
}
auto realpath = Common::GetRealPath(file_path);
auto realpath = Common::CreatePrefixPath(file_path);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed. "
<< "Info: module=" << module_ << ", name=" << name_ << ", "

4
mindspore/ccsrc/debug/trace.cc
View File

@ -350,7 +350,7 @@ bool AnalyzeFailExporter::ExportFuncGraph(const std::string &filename, const Tra
MS_LOG(DEBUG) << "Node configs is empty";
return false;
}
auto real_filepath = Common::GetRealPath(filename);
auto real_filepath = Common::CreatePrefixPath(filename);
if (!real_filepath.has_value()) {
MS_LOG(ERROR) << "The export ir path: " << filename << " is not illegal.";
return false;
@ -413,7 +413,7 @@ std::string GetEvalFailDatPath() {
path = ".";
}
path += "/rank_" + std::to_string(GetRank()) + "/om/analyze_fail.dat";
auto realpath = Common::GetRealPath(path);
auto realpath = Common::CreatePrefixPath(path);
if (!realpath.has_value()) {
MS_EXCEPTION(ValueError) << "Get real path failed. path=" << path;
}

2
mindspore/ccsrc/utils/file_utils.cc
View File

@ -59,7 +59,7 @@ void FileUtils::SplitDirAndFileName(const std::string &path, std::optional<std::
if (path_split_pos != std::string::npos) {
*prefix_path = path.substr(0, path_split_pos);
*file_name = path.substr(path_split_pos);
*file_name = path.substr(path_split_pos + 1);
} else {
*prefix_path = std::nullopt;
*file_name = path;

1
tests/st/dump/dump_test_utils.py
View File

@ -73,7 +73,6 @@ def generate_dump_json(dump_path, json_file_name, test_key):
"""
Util function to generate dump configuration json file.
"""
data = dict()
if test_key == "test_async_dump":
data = async_dump_dict
data["common_dump_settings"]["path"] = dump_path

27
tests/st/dump/test_data_dump.py
View File

@ -237,3 +237,30 @@ def test_dump_with_diagnostic_path():
add = Net()
add(Tensor(x), Tensor(y))
assert len(os.listdir(dump_file_path)) == 5
def run_e2e_dump_execution_graph():
"""Run e2e dump and check execution order."""
if sys.platform != 'linux':
return
pwd = os.getcwd()
with tempfile.TemporaryDirectory(dir=pwd) as tmp_dir:
dump_path = os.path.join(tmp_dir, 'e2e_dump_exe_graph')
dump_config_path = os.path.join(tmp_dir, 'e2e_dump.json')
generate_dump_json(dump_path, dump_config_path, 'test_e2e_dump')
os.environ['MINDSPORE_DUMP_CONFIG'] = dump_config_path
if os.path.isdir(dump_path):
shutil.rmtree(dump_path)
add = Net()
add(Tensor(x), Tensor(y))
exe_graph_path = os.path.join(dump_path, 'rank_0', 'execution_order')
assert len(os.listdir(exe_graph_path)) == 1
@pytest.mark.level0
@pytest.mark.platform_x86_gpu_training
@pytest.mark.env_onecard
def test_dump_with_execution_graph():
"""Test dump with execution graph on GPU."""
context.set_context(mode=context.GRAPH_MODE, device_target='GPU')
run_e2e_dump_execution_graph()