p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Move some ir files which backend optpass dumped to the fold: verbose_ir_files

This commit is contained in:
huanghui 2020-10-31 17:43:43 +08:00
parent 02a942ef56
commit fa6c23358a
11 changed files with 240 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
mindspore/ccsrc/backend/optimizer/common/pass_manager.cc
View File

@ -67,8 +67,11 @@ bool PassManager::Run(const FuncGraphPtr &func_graph, const std::vector<PassPtr>
MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost << " us";
#endif
if (save_graphs) {
auto file_name = "hwopt_" + name() + "_" + std::to_string(num) + "_" + pass->name() + ".ir";
DumpIR(file_name, func_graph, true);
std::ostringstream oss;
oss << "verbose_ir_files"
<< "/";
oss << "hwopt_" + name() + "_" + std::to_string(num) + "_" + pass->name() + ".ir";
DumpIR(oss.str(), func_graph, true);
}
num++;
}

2
mindspore/ccsrc/debug/CMakeLists.txt
View File

@ -4,6 +4,7 @@ set(_DEBUG_SRC_LIST
"${CMAKE_CURRENT_SOURCE_DIR}/draw.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/dump_proto.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/trace.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
)
if (ENABLE_DEBUGGER)
@ -12,7 +13,6 @@ if (ENABLE_DEBUGGER)
"${CMAKE_CURRENT_SOURCE_DIR}/debugger/grpc_client.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/debugger/proto_exporter.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/debug_services.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
)
endif (ENABLE_DEBUGGER)

31
mindspore/ccsrc/debug/anf_ir_dump.cc
View File

@ -28,6 +28,7 @@
#include "backend/session/anf_runtime_algorithm.h"
#include "frontend/parallel/ops_info/operator_info.h"
#include "pipeline/jit/base.h"
#include "debug/common.h"
namespace mindspore {
const std::string ToShortString(const TypeId &typeId) {
@ -483,35 +484,23 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu
if (graph == nullptr) {
return;
}
auto real_filename = pipeline::GetSaveGraphsPathName(AddGlobalId(filename));
if (real_filename.size() > PATH_MAX) {
MS_LOG(ERROR) << "File path " << real_filename << " is too long.";
auto path = pipeline::GetSaveGraphsPathName(AddGlobalId(filename));
auto realpath = Common::GetRealPath(path);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed. path=" << path;
return;
}
char real_path[PATH_MAX] = {0};
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(real_path, filename.c_str(), PATH_MAX) == nullptr) {
MS_LOG(DEBUG) << "dir " << filename << " does not exit.";
}
#else
if (nullptr == realpath(real_filename.c_str(), real_path)) {
MS_LOG(DEBUG) << "Dir " << real_filename << " does not exit.";
}
#endif
OrderedMap<AnfNodePtr, int32_t> para_map;
std::string path_string = real_path;
ChangeFileMode(path_string, S_IRWXU);
std::ofstream fout(real_path);
ChangeFileMode(realpath.value(), S_IRWXU);
std::ofstream fout(realpath.value());
std::ostringstream buffer;
if (!fout.is_open()) {
MS_LOG(ERROR) << "Open dump file '" << real_path << "' failed!";
MS_LOG(ERROR) << "Open dump file '" << realpath.value() << "' failed!";
return;
}
auto nodes = TopoSort(graph->get_return(), SuccDeeperSimple, AlwaysInclude);
OrderedMap<AnfNodePtr, int32_t> para_map;
// dump global info
DumpGlobalInfoEntry(graph, buffer);
DumpParams(graph, buffer, &para_map);
@ -528,7 +517,7 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu
fout.close();
// set file mode to read only by user
ChangeFileMode(path_string, S_IRUSR);
ChangeFileMode(realpath.value(), S_IRUSR);
}
#else
void DumpIR(const std::string &, const FuncGraphPtr &, bool) {

22
mindspore/ccsrc/debug/common.cc
View File

@ -43,11 +43,19 @@ std::optional<std::string> Common::GetRealPath(const std::string &input_path) {
MS_LOG(ERROR) << "CreateNotExistDirs Failed!";
return std::nullopt;
}
#if defined(SYSTEM_ENV_POSIX)
if (nullptr == realpath(prefix_path.c_str(), real_path)) {
MS_LOG(ERROR) << "dir " << prefix_path << " does not exit.";
MS_LOG(ERROR) << "dir " << prefix_path << " does not exist.";
return std::nullopt;
}
#elif defined(SYSTEM_ENV_WINDOWS)
if (nullptr == _fullpath(real_path, prefix_path.c_str(), PATH_MAX)) {
MS_LOG(ERROR) << "dir " << prefix_path << " does not exist.";
return std::nullopt;
}
#else
MS_LOG(EXCEPTION) << "Unsupported platform.";
#endif
out_path = std::string(real_path) + last_path;
}
@ -56,9 +64,17 @@ std::optional<std::string> Common::GetRealPath(const std::string &input_path) {
MS_LOG(ERROR) << "Prefix path is too longer!";
return std::nullopt;
}
#if defined(SYSTEM_ENV_POSIX)
if (nullptr == realpath(input_path.c_str(), real_path)) {
MS_LOG(ERROR) << "File " << input_path << " does not exit, it will be created.";
MS_LOG(ERROR) << "File " << input_path << " does not exist, it will be created.";
}
#elif defined(SYSTEM_ENV_WINDOWS)
if (nullptr == _fullpath(real_path, input_path.c_str(), PATH_MAX)) {
MS_LOG(ERROR) << "File " << input_path << " does not exist, it will be created.";
}
#else
MS_LOG(EXCEPTION) << "Unsupported platform.";
#endif
out_path = std::string(real_path);
}
return out_path;

3
mindspore/ccsrc/pipeline/jit/pipeline.cc
View File

@ -23,6 +23,7 @@
#include <unordered_map>
#include <cstdlib>
#include <algorithm>
#include <iomanip>
#include "ir/param_info.h"
#include "pipeline/jit/pass.h"
@ -85,7 +86,7 @@ std::unordered_map<abstract::AbstractBasePtrList, int, abstract::AbstractBasePtr
namespace {
std::string GetBaseNameForIR(int stage_idx, const std::string &action_name) {
std::ostringstream oss;
oss << stage_idx << "_" << action_name;
oss << std::setfill('0') << std::setw(2) << stage_idx << "_" << action_name;
return oss.str();
}

2
mindspore/ccsrc/utils/system/base.h
View File

@ -49,7 +49,7 @@ using uint64 = uint64_t;
#define SYSTEM_ENV_POSIX
#elif defined(_WIN32)
#elif defined(_WIN32) || defined(_WIN64)
#define SYSTEM_ENV_WINDOWS

3
mindspore/ccsrc/utils/system/env.h
View File

@ -34,6 +34,9 @@ class Env {
#if defined(SYSTEM_ENV_POSIX)
auto fs = std::make_shared<PosixFileSystem>();
return fs;
#elif defined(SYSTEM_ENV_WINDOWS)
auto fs = std::make_shared<WinFileSystem>();
return fs;
#else
MS_LOG(EXCEPTION) << "Now not support the platform.";
#endif

154
mindspore/ccsrc/utils/system/file_system.cc
View File

@ -16,14 +16,18 @@
#include "utils/system/file_system.h"
#if defined(SYSTEM_ENV_POSIX)
#include <sys/stat.h>
#include <unistd.h>
#elif defined(SYSTEM_ENV_WINDOWS)
#include <direct.h>
#endif
namespace mindspore {
namespace system {
#if defined(SYSTEM_ENV_POSIX)
// Implement the Posix file systen
// Implement the Posix file system
WriteFilePtr PosixFileSystem::CreateWriteFile(const string &file_name) {
if (file_name.empty()) {
MS_LOG(ERROR) << "Create write file failed because the file name is null.";
@ -49,7 +53,7 @@ bool PosixFileSystem::FileExist(const string &file_name) {
}
auto result = access(file_name.c_str(), F_OK);
if (result != 0) {
MS_LOG(INFO) << "The file(" << file_name << ") not exist.";
MS_LOG(DEBUG) << "The file(" << file_name << ") not exist.";
return false;
}
return true;
@ -62,7 +66,7 @@ bool PosixFileSystem::DeleteFile(const string &file_name) {
}
auto result = unlink(file_name.c_str());
if (result != 0) {
MS_LOG(ERROR) << "Delete the file(" << file_name << ") is falire, error(" << errno << ").";
MS_LOG(ERROR) << "Delete the file(" << file_name << ") is failed, error(" << errno << ").";
return false;
}
return true;
@ -77,7 +81,7 @@ bool PosixFileSystem::CreateDir(const string &dir_name) {
auto result = mkdir(dir_name.c_str(), DEFAULT_MKDIR_MODE);
if (result != 0) {
if (errno != EEXIST) {
MS_LOG(ERROR) << "Create the dir(" << dir_name << ") is falire, error(" << errno << ").";
MS_LOG(ERROR) << "Create the dir(" << dir_name << ") is failed, error(" << errno << ").";
return false;
}
}
@ -98,5 +102,147 @@ bool PosixFileSystem::DeleteDir(const string &dir_name) {
}
#endif
#if defined(SYSTEM_ENV_WINDOWS)
// Implement the Windows file system
WriteFilePtr WinFileSystem::CreateWriteFile(const string &file_name) {
if (file_name.empty()) {
MS_LOG(ERROR) << "Create write file failed because the file name is null.";
return nullptr;
}
auto fp = std::make_shared<WinWriteFile>(file_name);
if (fp == nullptr) {
MS_LOG(ERROR) << "Create write file(" << file_name << ") failed.";
return nullptr;
}
bool result = fp->Open();
if (!result) {
MS_LOG(ERROR) << "Open the write file(" << file_name << ") failed.";
return nullptr;
}
return fp;
}
bool WinFileSystem::FileExist(const string &file_name) {
if (file_name.empty()) {
MS_LOG(WARNING) << "The file name is null.";
return false;
}
auto result = access(file_name.c_str(), F_OK);
if (result != 0) {
MS_LOG(DEBUG) << "The file(" << file_name << ") not exist.";
return false;
}
return true;
}
bool WinFileSystem::CreateDir(const string &dir_name) {
if (dir_name.empty()) {
MS_LOG(WARNING) << "The directory name is null.";
return false;
}
auto result = mkdir(dir_name.c_str());
if (result != 0) {
MS_LOG(ERROR) << "Create the dir(" << dir_name << ") is failed, error(" << result << ").";
return false;
}
return true;
}
bool WinFileSystem::DeleteDir(const string &dir_name) {
if (dir_name.empty()) {
MS_LOG(WARNING) << "The directory name is null.";
return false;
}
auto result = rmdir(dir_name.c_str());
if (result != 0) {
MS_LOG(ERROR) << "Delete the dir(" << dir_name << ") is failed, error(" << result << ").";
return false;
}
return true;
}
bool WinFileSystem::DeleteFile(const string &file_name) {
if (file_name.empty()) {
MS_LOG(WARNING) << "The file name is null.";
return false;
}
auto result = unlink(file_name.c_str());
if (result != 0) {
MS_LOG(ERROR) << "Delete the file(" << file_name << ") is failed, error(" << errno << ").";
return false;
}
return true;
}
bool WinWriteFile::Open() {
if (file_ != nullptr) {
MS_LOG(WARNING) << "The File(" << file_name_ << ") already open.";
return true;
}
// check the path
if (nullptr == file_name_.c_str()) {
MS_LOG(EXCEPTION) << "The file path is null.";
}
char path[PATH_MAX + 1] = {0x00};
if (file_name_.size() > PATH_MAX || nullptr == _fullpath(path, file_name_.c_str(), PATH_MAX)) {
MS_LOG(EXCEPTION) << "Convert to real path fail, file name is " << file_name_ << ".";
}
// open the file
file_ = fopen(path, "w+");
if (file_ == nullptr) {
MS_LOG(ERROR) << "File(" << path << ") IO ERROR: " << errno << ".";
return false;
}
return true;
}
bool WinWriteFile::Write(const std::string &data) {
MS_LOG(DEBUG) << "Write data(" << data.size() << ") to file(" << this->file_name_ << ").";
size_t r = fwrite(data.data(), 1, data.size(), file_);
if (r != data.size()) {
MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR: " << errno << ".";
return false;
}
return true;
}
bool WinWriteFile::Close() {
if (file_ == nullptr) {
MS_LOG(WARNING) << "File(" << file_name_ << ") already close.";
return true;
}
bool result = true;
if (fclose(file_) != 0) {
MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR: " << errno << ".";
result = false;
}
file_ = nullptr;
return result;
}
bool WinWriteFile::Flush() {
if (fflush(file_) != 0) {
MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR: " << EBADF << ".";
return false;
}
return true;
}
bool WinWriteFile::Sync() { return Flush(); }
WinWriteFile::~WinWriteFile() {
try {
if (file_ != nullptr) {
(void)fclose(file_);
file_ = nullptr;
}
} catch (const std::exception &e) {
MS_LOG(ERROR) << "Exception when closing file.";
} catch (...) {
MS_LOG(ERROR) << "Non standard exception when closing file.";
}
}
#endif
} // namespace system
} // namespace mindspore

45
mindspore/ccsrc/utils/system/file_system.h
View File

@ -198,6 +198,51 @@ class PosixWriteFile : public WriteFile {
};
#endif
#if defined(SYSTEM_ENV_WINDOWS)
// File system of create or delete directory for windows system
class WinFileSystem : public FileSystem {
public:
WinFileSystem() = default;
~WinFileSystem() override = default;
// create a new write file
WriteFilePtr CreateWriteFile(const string &file_name) override;
// check the file is exist?
bool FileExist(const string &file_name) override;
// delete the file
bool DeleteFile(const string &file_name) override;
// Create a Directory
bool CreateDir(const string &dir_name) override;
// Delete the specified directory.
bool DeleteDir(const string &dir_name) override;
};
// A file that can be read and write for windows
class WinWriteFile : public WriteFile {
public:
explicit WinWriteFile(const string &file_name) : WriteFile(file_name), file_(nullptr) {}
~WinWriteFile() override;
bool Open() override;
bool Write(const std::string &data) override;
bool Close() override;
bool Flush() override;
bool Sync() override;
private:
FILE *file_;
};
#endif
} // namespace system
} // namespace mindspore

3
tests/ut/python/pynative_mode/test_backend.py
View File

@ -14,6 +14,7 @@
# ============================================================================
""" test_backend """
import os
import shutil
import pytest
import mindspore.nn as nn
@ -86,6 +87,6 @@ def teardown_module():
if not os.path.exists(item_name):
continue
if os.path.isdir(item_name):
os.rmdir(item_name)
shutil.rmtree(item_name)
elif os.path.isfile(item_name):
os.remove(item_name)

3
tests/ut/python/pynative_mode/test_context.py
View File

@ -14,6 +14,7 @@
# ============================================================================
""" test_context """
import os
import shutil
import pytest
from mindspore import context
@ -150,6 +151,6 @@ def teardown_module():
if not os.path.exists(item_name):
continue
if os.path.isdir(item_name):
os.rmdir(item_name)
shutil.rmtree(item_name)
elif os.path.isfile(item_name):
os.remove(item_name)