!12146 add funtions to check for directory and filename

From: @luopengting
Reviewed-by: 
Signed-off-by:

mindspore/ccsrc/backend/optimizer/somas/somas.cc

@@ -381,9 +381,9 @@ bool Somas::InitSomasTensors(const session::KernelGraph *graph) {
 #ifdef ENABLE_DUMP_IR
   SubModuleId module = SubModuleId::SM_OPTIMIZER;
   std::string tag = "somas";
-  std::string filename = "somas_pre_processed_info_" + std::to_string(graph->graph_id()) + ".ir";
+  std::string filename = "somas_pre_processed_info_" + std::to_string(graph->graph_id());
   mindspore::RDR::RecordString(module, tag, SomasInfo(), filename);
-  filename = "somas_offline_log_" + std::to_string(graph->graph_id()) + ".ir";
+  filename = "somas_offline_log_" + std::to_string(graph->graph_id());
   mindspore::RDR::RecordString(module, tag, Offline(), filename);
 #endif
 
@@ -673,7 +673,7 @@ void Somas::InitBasicInfo(const session::KernelGraph *graph) {
 #ifdef ENABLE_DUMP_IR
   SubModuleId module = SubModuleId::SM_OPTIMIZER;
   std::string tag = "somas";
-  std::string filename = "somas_initial_info_" + std::to_string(graph->graph_id()) + ".ir";
+  std::string filename = "somas_initial_info_" + std::to_string(graph->graph_id());
   mindspore::RDR::RecordString(module, tag, SomasInfo(), filename);
 #endif
 

mindspore/ccsrc/debug/CMakeLists.txt

@@ -10,6 +10,7 @@ set(_DEBUG_SRC_LIST
 
 if(ENABLE_DUMP_IR)
     list(APPEND _DEBUG_SRC_LIST
+        "${CMAKE_CURRENT_SOURCE_DIR}/rdr/base_recorder.cc"
         "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_exec_order_recorder.cc"
         "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_recorder.cc"
         "${CMAKE_CURRENT_SOURCE_DIR}/rdr/string_recorder.cc"

mindspore/ccsrc/debug/common.cc

@@ -170,4 +170,105 @@ std::optional<std::string> Common::GetEnvConfigFile() {
   }
   return config_file;
 }
+
+bool Common::IsStrLengthValid(const std::string &str, const int &length_limit, const std::string &error_message,
+                              const bool &print_str) {
+  const int len_str = str.length();
+  if (len_str > length_limit) {
+    std::ostringstream msg;
+    if (print_str) {
+      msg << error_message << "The string is " << str << ", its length is " << str.length();
+    } else {
+      msg << error_message << "The length is " << str.length();
+    }
+    msg << ", exceeding the limit of " << length_limit << ".";
+    MS_LOG(WARNING) << msg.str();
+    return false;
+  }
+  return true;
+}
+
+bool Common::IsEveryFilenameValid(const std::string &path, const int &length_limit, const std::string &error_message) {
+  int left_pos = 0;
+  int len_path = path.length();
+  for (int i = 0; i < len_path; i++) {
+    if (i != 0) {
+      if (path[i] == '\\' || path[i] == '/') {
+        int cur_len = i - left_pos;
+        if (cur_len > length_limit) {
+          MS_LOG(WARNING) << error_message << "The name length of '" << path.substr(left_pos, cur_len) << "' is "
+                          << cur_len << ". It is out of the limit which is " << length_limit << ".";
+          return false;
+        }
+        left_pos = i + 1;
+      }
+    }
+  }
+  if (!(path[len_path - 1] == '\\' || path[len_path - 1] == '/')) {
+    int cur_len = len_path - left_pos;
+    if (cur_len > length_limit) {
+      MS_LOG(WARNING) << error_message << "The name length of '" << path.substr(left_pos, cur_len) << "' is " << cur_len
+                      << ". It is out of the limit which is " << length_limit << ".";
+      return false;
+    }
+  }
+  return true;
+}
+
+bool Common::IsPathValid(const std::string &path, const int &length_limit, const std::string &error_message,
+                         const bool &print_str) {
+  std::string err_msg = "Detail: ";
+  if (!error_message.empty()) {
+    err_msg = error_message + " " + err_msg;
+  }
+
+  if (path.empty()) {
+    MS_LOG(WARNING) << err_msg << "The path is empty.";
+    return false;
+  }
+
+  if (!IsStrLengthValid(path, length_limit, err_msg, print_str)) {
+    return false;
+  }
+
+  if (!std::all_of(path.begin(), path.end(),
+                   [](char c) { return ::isalpha(c) || ::isdigit(c) || c == '-' || c == '_' || c == '/'; })) {
+    MS_LOG(WARNING) << err_msg << "The path only support alphabets, digit or {'-', '_', '/'}, but got:" << path << ".";
+    return false;
+  }
+
+  if (path[0] != '/') {
+    MS_LOG(WARNING) << err_msg << "The path only support absolute path and should start with '/'.";
+    return false;
+  }
+
+  if (!IsEveryFilenameValid(path, maxOSFilenameLength, err_msg)) {
+    return false;
+  }
+  return true;
+}
+
+bool Common::IsFilenameValid(const std::string &filename, const int &length_limit, const std::string &error_message) {
+  std::string err_msg = "Detail: ";
+  if (!error_message.empty()) {
+    err_msg = error_message + " " + err_msg;
+  }
+
+  if (filename.empty()) {
+    MS_LOG(WARNING) << err_msg << "The filename is empty.";
+    return false;
+  }
+
+  if (!IsStrLengthValid(filename, length_limit, err_msg)) {
+    return false;
+  }
+
+  if (!std::all_of(filename.begin(), filename.end(),
+                   [](char c) { return ::isalpha(c) || ::isdigit(c) || c == '-' || c == '_' || c == '.'; })) {
+    MS_LOG(WARNING) << err_msg << "The filename only support alphabets, digit or {'-', '_', '.'}, but got:" << filename
+                    << ".";
+    return false;
+  }
+  return true;
+}
 }  // namespace mindspore

mindspore/ccsrc/debug/common.h

@@ -22,6 +22,9 @@
 #include "utils/contract.h"
 
 namespace mindspore {
+static const int maxDirectoryLength = 1024;
+static const int maxFilenameLength = 128;
+static const int maxOSFilenameLength = 255;
 class Common {
  public:
   Common() = default;
@@ -29,9 +32,16 @@ class Common {
   static std::optional<std::string> GetRealPath(const std::string &input_path);
   static std::optional<std::string> GetConfigFile(const std::string &env);
   static std::optional<std::string> GetEnvConfigFile();
+  static bool IsStrLengthValid(const std::string &str, const int &length_limit, const std::string &error_message = "",
+                               const bool &print_str = true);
+  static bool IsPathValid(const std::string &path, const int &length_limit, const std::string &error_message = "",
+                          const bool &print_str = true);
+  static bool IsFilenameValid(const std::string &filename, const int &length_limit,
+                              const std::string &error_message = "");
 
  private:
   static bool CreateNotExistDirs(const std::string &path);
+  static bool IsEveryFilenameValid(const std::string &path, const int &length_limit, const std::string &error_message);
 };
 }  // namespace mindspore
 #endif  // MINDSPORE_CCSRC_DEBUG_COMMON_H_

mindspore/ccsrc/debug/env_config_parser.cc

@@ -116,28 +116,20 @@ void EnvConfigParser::ParseRdrSetting(const nlohmann::json &content) {
 }
 
 void EnvConfigParser::ParseRdrPath(const nlohmann::json &content) {
+  std::string err_msg = "RDR path parse failed. The RDR path will be a default value: '" + rdr_path_ +
+                        "'. Please check the settings about '" + kRdrSettings + "' in config file '" + config_file_ +
+                        "' set by 'env_config_path' in context.";
+
   if (!CheckJsonStringType(content, kRdrSettings, kPath)) {
-    MS_LOG(WARNING) << "The RDR path will be a default value: '" << rdr_path_ << "'.";
+    MS_LOG(WARNING) << err_msg;
     return;
   }
+
   std::string path = content;
-  if (!std::all_of(path.begin(), path.end(),
-                   [](char c) { return ::isalpha(c) || ::isdigit(c) || c == '-' || c == '_' || c == '/'; })) {
-    MS_LOG(WARNING) << "The path in " << kRdrSettings
-                    << " only support alphabets, digit or {'-', '_', '/'}, but got:" << path << "."
-                    << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
-    return;
-  }
-  if (path.empty()) {
-    MS_LOG(WARNING) << "The path in " << kRdrSettings << " is empty."
-                    << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
-    return;
-  }
-  if (path[0] != '/') {
-    MS_LOG(WARNING) << "The path in " << kRdrSettings << " only support absolute path and should start with '/'."
-                    << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
+  if (!Common::IsPathValid(path, maxDirectoryLength, err_msg, false)) {
     return;
   }
+
   if (path.back() != '/') {
     path += '/';
   }
@@ -146,7 +138,7 @@ void EnvConfigParser::ParseRdrPath(const nlohmann::json &content) {
 
 void EnvConfigParser::ParseRdrEnable(const nlohmann::json &content) {
   if (!content.is_boolean()) {
-    MS_LOG(WARNING) << "Json Parse Failed. 'enable' in " << kRdrSettings << " should be boolean."
+    MS_LOG(WARNING) << "Json parse failed. 'enable' in " << kRdrSettings << " should be boolean."
                     << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
     rdr_enabled_ = false;
     return;

mindspore/ccsrc/debug/rdr/base_recorder.cc

@@ -0,0 +1,54 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "debug/rdr/base_recorder.h"
+#include <sys/stat.h>
+#include <fstream>
+#include "debug/common.h"
+#include "utils/utils.h"
+
+namespace mindspore {
+void BaseRecorder::SetDirectory(const std::string &directory) {
+  std::string error_message = module_ + ":" + tag_ + " set directory failed.";
+  if (Common::IsPathValid(directory, maxDirectoryLength, error_message)) {
+    directory_ = directory;
+    if (directory_.back() != '/') {
+      directory_ += "/";
+    }
+  }
+}
+
+void BaseRecorder::SetFilename(const std::string &filename) {
+  std::string error_message = module_ + ":" + tag_ + " set filename failed.";
+  if (Common::IsFilenameValid(filename, maxDirectoryLength, error_message)) {
+    filename_ = filename;
+  }
+}
+
+std::optional<std::string> BaseRecorder::GetFileRealPath() {
+  if (filename_.empty()) {
+    filename_ = module_ + "_" + tag_ + "_" + timestamp_;
+  }
+  std::string file_path = directory_ + filename_;
+  auto realpath = Common::GetRealPath(file_path);
+  if (!realpath.has_value()) {
+    MS_LOG(ERROR) << "Get real path failed. "
+                  << "Info: module=" << module_ << ", tag=" << tag_ << ", "
+                  << "path=" << file_path << ".";
+  }
+
+  return realpath;
+}
+}  // namespace mindspore

mindspore/ccsrc/debug/rdr/base_recorder.h

@@ -21,7 +21,11 @@
 #include <sstream>
 #include <chrono>
 #include <iomanip>
+#include "debug/common.h"
 #include "debug/env_config_parser.h"
+#include "mindspore/core/utils/log_adapter.h"
+
+const int maxTagLength = 32;
 namespace mindspore {
 class BaseRecorder {
  public:
@@ -31,6 +35,17 @@ class BaseRecorder {
     config_parser_ptr.Parse();
     directory_ = config_parser_ptr.rdr_path();
 
+    if (tag.length() > maxTagLength) {
+      tag_ = tag.substr(0, maxTagLength);
+      MS_LOG(WARNING) << "The tag length is " << tag.length() << ", exceeding the limit " << maxTagLength
+                      << ". It will be intercepted as '" << tag_ << "'.";
+    }
+
+    std::string err_msg = module_ + ":" + tag_ + " set filename failed.";
+    if (!filename_.empty() && !Common::IsFilenameValid(filename_, maxFilenameLength, err_msg)) {
+      filename_ = "";
+    }
+
     auto sys_time = std::chrono::system_clock::now();
     auto t_time = std::chrono::system_clock::to_time_t(sys_time);
     std::tm *l_time = std::localtime(&t_time);
@@ -47,9 +62,10 @@ class BaseRecorder {
   std::string GetModule() const { return module_; }
   std::string GetTag() const { return tag_; }
   std::string GetTimeStamp() const { return timestamp_; }
+  std::optional<std::string> GetFileRealPath();
 
-  void SetDirectory(const std::string &directory) { directory_ = directory; }
-  std::string GetDirectory() const { return directory_; }
+  void SetDirectory(const std::string &directory);
+  void SetFilename(const std::string &filename);
   virtual void Export() {}
 
  protected:

mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc

@@ -46,10 +46,11 @@ bool DumpGraphExeOrder(const std::string &filename, const std::vector<CNodePtr>
 }  // namespace
 
 void GraphExecOrderRecorder::Export() {
-  if (filename_.empty()) {
-    filename_ = module_ + "_" + tag_ + "_" + timestamp_;
+  auto realpath = GetFileRealPath();
+  if (!realpath.has_value()) {
+    return;
   }
-  std::string filename = directory_ + filename_ + ".txt";
-  DumpGraphExeOrder(filename, exec_order_);
+  std::string real_file_path = realpath.value() + ".txt";
+  DumpGraphExeOrder(real_file_path, exec_order_);
 }
 }  // namespace mindspore

mindspore/ccsrc/debug/rdr/graph_recorder.cc

@@ -25,33 +25,22 @@
 namespace mindspore {
 namespace protobuf {
 #ifdef ENABLE_DUMP_IR
-void DumpIRProto(const std::string &filename, const FuncGraphPtr &func_graph) {
+void DumpIRProto(const std::string &real_path, const FuncGraphPtr &func_graph) {
   if (func_graph == nullptr) {
-    MS_LOG(ERROR) << "Func graph is nullptr";
+    MS_LOG(ERROR) << "Func graph is nullptr.";
     return;
   }
 
-  if (filename.size() > PATH_MAX) {
-    MS_LOG(ERROR) << "File path " << filename << " is too long.";
-    return;
-  }
-
-  auto real_path = Common::GetRealPath(filename);
-  if (!real_path.has_value()) {
-    MS_LOG(ERROR) << "Get real path failed. path=" << filename;
-    return;
-  }
-  ChangeFileMode(real_path.value(), S_IRWXU);
   // write to pb file
-  std::ofstream ofs(real_path.value());
+  std::ofstream ofs(real_path);
   if (!ofs.is_open()) {
-    MS_LOG(ERROR) << "Open file '" << real_path.value() << "' failed!";
+    MS_LOG(ERROR) << "Open file '" << real_path << "' failed!";
     return;
   }
   ofs << GetFuncGraphProtoString(func_graph);
   ofs.close();
   // set file mode to read only by user
-  ChangeFileMode(real_path.value(), S_IRUSR);
+  ChangeFileMode(real_path, S_IRUSR);
 }
 #else
 void DumpIRProto(const std::string &, const FuncGraphPtr &) {
@@ -68,31 +57,35 @@ void DumpIRProto(const std::string &, const FuncGraphPtr &) {
 
 void GraphRecorder::Export() {
   bool save_flag = false;
-  if (filename_.empty()) {
-    filename_ = module_ + "_" + tag_ + "_" + timestamp_;
+
+  auto tmp_realpath = GetFileRealPath();
+  if (!tmp_realpath.has_value()) {
+    return;
   }
-  std::string file_path = directory_ + filename_ + std::to_string(id_);
+
+  std::string realpath = tmp_realpath.value() + std::to_string(id_);
+
   if (graph_type_.find(".dat") != std::string::npos) {
     save_flag = true;
-    AnfExporter exporter("");
-    std::string real_path = file_path + ".dat";
-    ChangeFileMode(real_path, S_IRWXU);
-    exporter.ExportFuncGraph(real_path, func_graph_);
-    ChangeFileMode(real_path, S_IRUSR);
+    AnfExporter exporter(std::to_string(id_));
+    std::string realpath_dat = realpath + ".dat";
+    ChangeFileMode(realpath_dat, S_IRWXU);
+    exporter.ExportFuncGraph(realpath_dat, func_graph_);
+    ChangeFileMode(realpath_dat, S_IRUSR);
   }
   if (graph_type_.find(".ir") != std::string::npos) {
     save_flag = true;
-    std::string real_path = file_path + ".ir";
+    std::string realpath_ir = realpath + ".ir";
     if (full_name_) {
-      DumpIRForRDR(real_path, func_graph_, true, kTopStack);
+      DumpIRForRDR(realpath_ir, func_graph_, true, kTopStack);
     } else {
-      DumpIRForRDR(real_path, func_graph_, false, kOff);
+      DumpIRForRDR(realpath_ir, func_graph_, false, kOff);
     }
   }
   if (graph_type_.find(".pb") != std::string::npos) {
     save_flag = true;
-    std::string real_path = file_path + ".pb";
-    protobuf::DumpIRProto(real_path, func_graph_);  // save *.pb file
+
+    protobuf::DumpIRProto(realpath + ".pb", func_graph_);  // save *.pb file
   }
   if (!save_flag) {
     MS_LOG(WARNING) << "Unknown save graph type: " << graph_type_;

mindspore/ccsrc/debug/rdr/string_recorder.cc

@@ -22,23 +22,13 @@
 
 namespace mindspore {
 void StringRecorder::Export() {
-  if (directory_.back() != '/') {
-    directory_ += "/";
-  }
-
-  if (filename_.empty()) {
-    filename_ = module_ + "_" + tag_ + "_" + timestamp_ + ".txt";
-  }
-  std::string file_path = directory_ + filename_;
-
-  auto realpath = Common::GetRealPath(file_path);
+  auto realpath = GetFileRealPath();
   if (!realpath.has_value()) {
-    MS_LOG(ERROR) << "Get real path failed. path=" << file_path;
     return;
   }
-
-  ChangeFileMode(realpath.value(), S_IRWXU);
-  std::ofstream fout(realpath.value(), std::ofstream::app);
+  std::string file_path = realpath.value() + ".txt";
+  ChangeFileMode(file_path, S_IRWXU);
+  std::ofstream fout(file_path, std::ofstream::app);
   if (!fout.is_open()) {
     MS_LOG(WARNING) << "Open file for saving string failed.";
     return;
@@ -46,6 +36,6 @@ void StringRecorder::Export() {
   fout << data_;
   fout.close();
   // set file mode to read only by user
-  ChangeFileMode(realpath.value(), S_IRUSR);
+  ChangeFileMode(file_path, S_IRUSR);
 }
 }  // namespace mindspore

mindspore/ccsrc/debug/rdr/string_recorder.h

@@ -25,15 +25,15 @@ class StringRecorder : public BaseRecorder {
  public:
   StringRecorder() : BaseRecorder() {}
   StringRecorder(const std::string &module, const std::string &tag, const std::string &data,
-                 const std::string &file_type)
-      : BaseRecorder(module, tag), data_(data) {}
+                 const std::string &filename)
+      : BaseRecorder(module, tag), data_(data), filename_(filename) {}
   ~StringRecorder() {}
   void SetModule(const std::string &module) { module_ = module; }
-  void SetFilename(const std::string &filename) { filename_ = filename; }
   virtual void Export();
 
  private:
   std::string data_;
+  std::string filename_;
 };
 using StringRecorderPtr = std::shared_ptr<StringRecorder>;
 }  // namespace mindspore

mindspore/ccsrc/runtime/device/memory_manager.cc

@@ -76,10 +76,10 @@ void MemoryManager::MallocSomasDynamicMem(const session::KernelGraph *graph) {
   SubModuleId module = SubModuleId::SM_OPTIMIZER;
   std::string tag = "somas";
 
-  std::string filename = "somas_allocate_info_" + std::to_string(graph->graph_id()) + ".ir";
+  std::string filename = "somas_allocate_info_" + std::to_string(graph->graph_id());
   mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasInfo(), filename);
 
-  filename = "somas_mem_info_" + std::to_string(graph->graph_id()) + ".ir";
+  filename = "somas_mem_info_" + std::to_string(graph->graph_id());
   mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasMemory(), filename);
 #endif
   bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG);

mindspore/ops/operations/nn_ops.py

@@ -18,6 +18,7 @@
 import math
 import operator
 from functools import reduce, partial
+from mindspore import log as logger
 from mindspore._checkparam import _check_3d_int_or_tuple
 import numpy as np
 from ... import context