!23186 use Common::CreatePrefixPath instead of Common::GetRealPath

Merge pull request !23186 from yelihua/dev

mindspore/ccsrc/backend/session/session_basic.cc

@@ -2677,14 +2677,8 @@ void SessionBasic::InitPSParamAndOptim(const KernelGraphPtr &kernel_graph,
 }  // namespace session
 void DumpGraphExeOrder(const std::string &file_name, const std::string &target_dir,
                        const std::vector<CNodePtr> &execution_order) {
-  auto dir_path = FileUtils::CreateNotExistDirs(target_dir + "/execution_order/");
+  std::string file_path = target_dir + "/execution_order/" + file_name;
-  if (!dir_path.has_value()) {
+  auto realpath = Common::CreatePrefixPath(file_path);
-    MS_LOG(ERROR) << "Failed to CreateNotExistDirs: " << target_dir << "/execution_order/ in DumpGraphExeOrder";
-    return;
-  }
-
-  std::string file_path = dir_path.value() + "/" + file_name;
-  auto realpath = FileUtils::GetRealPath(common::SafeCStr(file_path));
   if (!realpath.has_value()) {
     MS_LOG(ERROR) << "Get real path: " << file_path << " filed.";
     return;

mindspore/ccsrc/debug/anf_ir_dump.cc

@@ -598,7 +598,7 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu
   if (!target_file.empty()) {
     path = target_file;
   }
-  auto realpath = Common::GetRealPath(path);
+  auto realpath = Common::CreatePrefixPath(path);
   if (!realpath.has_value()) {
     MS_LOG(ERROR) << "Get real path failed, path=" << path;
     return;
@@ -641,7 +641,7 @@ void DumpIRForRDR(const std::string &filename, const FuncGraphPtr &graph, bool d
     return;
   }
   auto path = Common::AddId(filename, ".ir");
-  auto realpath = Common::GetRealPath(path);
+  auto realpath = Common::CreatePrefixPath(path);
   if (!realpath.has_value()) {
     MS_LOG(ERROR) << "Get real path failed. path=" << path;
     return;

mindspore/ccsrc/debug/anf_ir_utils.cc

@@ -644,7 +644,7 @@ void ExportIR(const std::string &filename, const FuncGraphPtr &func_graph) {
   }
 
   auto filepath = GetSaveGraphsPathName(Common::AddId(filename, ".dat"));
-  auto real_filepath = Common::GetRealPath(filepath);
+  auto real_filepath = Common::CreatePrefixPath(filepath);
   if (!real_filepath.has_value()) {
     MS_LOG(ERROR) << "The export ir path: " << filepath << " is not illegal.";
     return;

mindspore/ccsrc/debug/common.cc

@@ -382,14 +382,14 @@ struct GlogLogDirRegister {
         both_exist = true;
       }
       log_dir_str += "/rank_" + rank + "/logs";
-      auto real_log_dir_str = Common::GetRealPath(log_dir_str);
+      auto real_log_dir_str = Common::CreatePrefixPath(log_dir_str);
       // While 'GLOG_logtostderr' = 0, logs output to files. 'GLOG_log_dir' must be specified as the path of log files.
       // Here can not throw exception and use python to catch, because the PYBIND11_MODULE is not yet been initialed.
       if (logtostderr_str == "0" && real_log_dir_str.has_value()) {
         if (!Common::IsPathValid(real_log_dir_str.value(), MAX_DIRECTORY_LENGTH, "")) {
           MS_LOG(ERROR) << "The path of log files, which set by 'GLOG_log_dir', is invalid";
           exit(EXIT_FAILURE);
-        } else if (!Common::CreateNotExistDirs(real_log_dir_str.value())) {
+        } else if (!FileUtils::CreateNotExistDirs(real_log_dir_str.value())) {
           MS_LOG(ERROR) << "Create the path of log files, which set by 'GLOG_log_dir', failed.";
           exit(EXIT_FAILURE);
         }

mindspore/ccsrc/debug/data_dump/dump_json_parser.cc

@@ -90,11 +90,7 @@ void DumpJsonParser::Parse() {
     MS_LOG(EXCEPTION) << "Get dump config file failed";
   }
 
-  auto dump_file_realpath = Common::GetRealPath(dump_config_file.value());
+  std::ifstream json_file(dump_config_file.value());
-  if (!dump_file_realpath.has_value()) {
-    MS_LOG(EXCEPTION) << "Get real path failed in Parse.";
-  }
-  std::ifstream json_file(dump_file_realpath.value());
   if (!json_file.is_open()) {
     MS_LOG(EXCEPTION) << "Dump file:" << dump_config_file.value() << " open failed."
                       << " Errno:" << errno << " ErrInfo:" << strerror(errno);
@@ -132,7 +128,8 @@ void DumpJsonParser::CopyJsonToDir(uint32_t rank_id) {
   }
   std::ifstream json_file(dump_config_file.value());
   if (async_dump_enabled_ || e2e_dump_enabled_) {
-    auto realpath = Common::GetRealPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/data_dump.json");
+    auto realpath =
+      Common::CreatePrefixPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/data_dump.json");
     if (!realpath.has_value()) {
       MS_LOG(ERROR) << "Get real path failed in CopyJsonDir.";
     } else {
@@ -162,7 +159,7 @@ void DumpJsonParser::CopyHcclJsonToDir(uint32_t rank_id) {
     }
   }
   std::ifstream json_file(config_path);
-  auto realpath = Common::GetRealPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/hccl.json");
+  auto realpath = Common::CreatePrefixPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/hccl.json");
   if (!realpath.has_value()) {
     MS_LOG(ERROR) << "Get real path failed in CopyHcclJsonToDir.";
   } else {
@@ -182,7 +179,7 @@ void DumpJsonParser::CopyMSCfgJsonToDir(uint32_t rank_id) {
   if (!IsDumpEnabled()) {
     return;
   }
-  auto realpath = Common::GetRealPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/config.json");
+  auto realpath = Common::CreatePrefixPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/config.json");
   if (!realpath.has_value()) {
     MS_LOG(ERROR) << "Get real path failed in CopyMSConfigJsonToDir.";
   } else {

mindspore/ccsrc/debug/data_dump/e2e_dump.cc

@@ -28,6 +28,7 @@
 #include "utils/ms_context.h"
 #include "runtime/device/kernel_runtime_manager.h"
 #include "utils/config_manager.h"
+#include "utils/file_utils.h"
 #ifdef ENABLE_DEBUGGER
 #include "debug/debug_services.h"
 #include "debug/tensor_load.h"
@@ -301,10 +302,9 @@ void E2eDump::DumpSetup(const session::KernelGraph *graph, uint32_t rank_id) {
     MS_LOG(INFO) << "cur_iter_dump_path: " << cur_iter_dump_path;
 
     // create cur_iter_dump_path dirs
-    bool status = Common::CreateNotExistDirs(root_cur_iter_dump_path);
+    auto dir_path = FileUtils::CreateNotExistDirs(root_cur_iter_dump_path);
-    if (!status) {
+    if (!dir_path.has_value()) {
       MS_LOG(EXCEPTION) << "Failed at CreateNotExistDirs for " << root_cur_iter_dump_path;
-      return;
     }
   }
 }
@@ -340,8 +340,8 @@ void E2eDump::DumpData(const session::KernelGraph *graph, uint32_t rank_id, cons
 
     if (dump_json_parser.IsDumpIter(current_iter)) {
       // create actual dir for iteration in final dump dir
-      bool status = Common::CreateNotExistDirs(cur_iter_dump_path);
+      auto dir_path = FileUtils::CreateNotExistDirs(cur_iter_dump_path);
-      if (!status) {
+      if (!dir_path.has_value()) {
         MS_LOG(EXCEPTION) << "failed at CreateNotExistDirs for " << cur_iter_dump_path;
       }
 

mindspore/ccsrc/debug/debug_services.cc

@@ -1285,7 +1285,7 @@ bool DebugServices::CheckOpOverflow(std::string node_name_to_find, unsigned int
   overflow_bin_path = DumpJsonParser::GetInstance().GetOpOverflowBinPath(debugger->GetGraphPtr()->graph_id());
   std::string check_overflow_bin_path = RealPath(overflow_bin_path);
   if (check_overflow_bin_path.empty()) {
-    MS_LOG(WARNING) << "Get real path failed for overflow_bin_path.";
+    MS_LOG(INFO) << "Get real path failed for overflow_bin_path.";
     return false;
   }
   overflow_bin_path = check_overflow_bin_path;

mindspore/ccsrc/debug/debugger/proto_exporter.cc

@@ -563,7 +563,7 @@ void DumpIRProtoWithSrcInfo(const FuncGraphPtr &func_graph, const std::string &s
     return;
   }
   std::string file_path = target_dir + "/" + "ms_output_" + suffix + ".pb";
-  auto realpath = Common::GetRealPath(file_path);
+  auto realpath = Common::CreatePrefixPath(file_path);
   if (!realpath.has_value()) {
     MS_LOG(ERROR) << "Get real path failed, path=" << file_path;
     return;

mindspore/ccsrc/debug/draw.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -189,7 +189,7 @@ void Draw(const std::string &filename, const FuncGraphPtr &func_graph) {
   const std::string filename_with_suffix =
     (filename.rfind(dot_suffix) != (filename.size() - dot_suffix.size())) ? (filename + dot_suffix) : filename;
   const std::string filepath = GetSaveGraphsPathName(Common::AddId(filename_with_suffix, dot_suffix));
-  auto real_filepath = Common::GetRealPath(filepath);
+  auto real_filepath = Common::CreatePrefixPath(filepath);
   if (!real_filepath.has_value()) {
     MS_LOG(EXCEPTION) << "The export ir path: " << filepath << " is not illegal.";
   }
@@ -199,7 +199,7 @@ void Draw(const std::string &filename, const FuncGraphPtr &func_graph) {
 void DrawUserFuncGraph(const std::string &filename, const FuncGraphPtr &func_graph) {
   const std::string dot_suffix = ".dot";
   const std::string filepath = GetSaveGraphsPathName(Common::AddId(filename, dot_suffix));
-  auto real_filepath = Common::GetRealPath(filepath);
+  auto real_filepath = Common::CreatePrefixPath(filepath);
   if (!real_filepath.has_value()) {
     MS_LOG(EXCEPTION) << "The export ir path: " << filepath << " is not illegal.";
   }

mindspore/ccsrc/debug/dump_proto.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -548,7 +548,7 @@ void DumpIRProto(const FuncGraphPtr &func_graph, const std::string &suffix) {
     return;
   }
   std::string file_path = GetSaveGraphsPathName("ms_output_" + suffix + ".pb");
-  auto realpath = Common::GetRealPath(file_path);
+  auto realpath = Common::CreatePrefixPath(file_path);
   if (!realpath.has_value()) {
     MS_LOG(ERROR) << "Get real path failed, path=" << file_path;
     return;

mindspore/ccsrc/debug/rdr/base_recorder.cc

@@ -43,7 +43,7 @@ std::optional<std::string> BaseRecorder::GetFileRealPath(const std::string &suff
   if (config_file.empty()) {
     file_path = directory_ + "rank_" + std::to_string(GetRank()) + "/rdr/" + filename;
   }
-  auto realpath = Common::GetRealPath(file_path);
+  auto realpath = Common::CreatePrefixPath(file_path);
   if (!realpath.has_value()) {
     MS_LOG(ERROR) << "Get real path failed. "
                   << "Info: module=" << module_ << ", name=" << name_ << ", "

mindspore/ccsrc/debug/trace.cc

@@ -350,7 +350,7 @@ bool AnalyzeFailExporter::ExportFuncGraph(const std::string &filename, const Tra
     MS_LOG(DEBUG) << "Node configs is empty";
     return false;
   }
-  auto real_filepath = Common::GetRealPath(filename);
+  auto real_filepath = Common::CreatePrefixPath(filename);
   if (!real_filepath.has_value()) {
     MS_LOG(ERROR) << "The export ir path: " << filename << " is not illegal.";
     return false;
@@ -413,7 +413,7 @@ std::string GetEvalFailDatPath() {
     path = ".";
   }
   path += "/rank_" + std::to_string(GetRank()) + "/om/analyze_fail.dat";
-  auto realpath = Common::GetRealPath(path);
+  auto realpath = Common::CreatePrefixPath(path);
   if (!realpath.has_value()) {
     MS_EXCEPTION(ValueError) << "Get real path failed. path=" << path;
   }

mindspore/ccsrc/utils/file_utils.cc

@@ -59,7 +59,7 @@ void FileUtils::SplitDirAndFileName(const std::string &path, std::optional<std::
 
   if (path_split_pos != std::string::npos) {
     *prefix_path = path.substr(0, path_split_pos);
-    *file_name = path.substr(path_split_pos);
+    *file_name = path.substr(path_split_pos + 1);
   } else {
     *prefix_path = std::nullopt;
     *file_name = path;

tests/st/dump/dump_test_utils.py

@@ -73,7 +73,6 @@ def generate_dump_json(dump_path, json_file_name, test_key):
     """
     Util function to generate dump configuration json file.
     """
-    data = dict()
     if test_key == "test_async_dump":
         data = async_dump_dict
         data["common_dump_settings"]["path"] = dump_path

tests/st/dump/test_data_dump.py

@@ -237,3 +237,30 @@ def test_dump_with_diagnostic_path():
         add = Net()
         add(Tensor(x), Tensor(y))
         assert len(os.listdir(dump_file_path)) == 5
+
+
+def run_e2e_dump_execution_graph():
+    """Run e2e dump and check execution order."""
+    if sys.platform != 'linux':
+        return
+    pwd = os.getcwd()
+    with tempfile.TemporaryDirectory(dir=pwd) as tmp_dir:
+        dump_path = os.path.join(tmp_dir, 'e2e_dump_exe_graph')
+        dump_config_path = os.path.join(tmp_dir, 'e2e_dump.json')
+        generate_dump_json(dump_path, dump_config_path, 'test_e2e_dump')
+        os.environ['MINDSPORE_DUMP_CONFIG'] = dump_config_path
+        if os.path.isdir(dump_path):
+            shutil.rmtree(dump_path)
+        add = Net()
+        add(Tensor(x), Tensor(y))
+        exe_graph_path = os.path.join(dump_path, 'rank_0', 'execution_order')
+        assert len(os.listdir(exe_graph_path)) == 1
+
+
+@pytest.mark.level0
+@pytest.mark.platform_x86_gpu_training
+@pytest.mark.env_onecard
+def test_dump_with_execution_graph():
+    """Test dump with execution graph on GPU."""
+    context.set_context(mode=context.GRAPH_MODE, device_target='GPU')
+    run_e2e_dump_execution_graph()