forked from mindspore-Ecosystem/mindspore
!17873 [MS][LOG] Make directorys when the path set by GLOG_log_dir is not exist
Merge pull request !17873 from louie5/master
This commit is contained in:
commit
7da28cef6f
|
@ -119,5 +119,20 @@ bool UpdateGPUMemAddressInfo(const SubModuleId module, const std::string &name,
|
|||
void TriggerAll() { mindspore::RecorderManager::Instance().TriggerAll(); }
|
||||
|
||||
void ClearAll() { mindspore::RecorderManager::Instance().ClearAll(); }
|
||||
|
||||
void ClearGPUMemAddressInfo() {
|
||||
if (!mindspore::RecorderManager::Instance().RdrEnable()) {
|
||||
return;
|
||||
}
|
||||
if (RecorderManager::Instance().CheckRdrGPUMemIsRecord()) {
|
||||
std::string name = "mem_address_list";
|
||||
std::string submodule_name = "KERNEL";
|
||||
auto recorder = RecorderManager::Instance().GetRecorder(submodule_name, name);
|
||||
if (recorder != nullptr) {
|
||||
auto mem_recorder = std::dynamic_pointer_cast<GPUMemAddressRecorder>(recorder);
|
||||
mem_recorder->CleanUp();
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace RDR
|
||||
} // namespace mindspore
|
||||
|
|
|
@ -61,6 +61,7 @@ bool RecordTaskDebugInfo(SubModuleId module, const std::string &name,
|
|||
#endif // ENABLE_D
|
||||
void TriggerAll();
|
||||
void ClearAll();
|
||||
void ClearGPUMemAddressInfo();
|
||||
} // namespace RDR
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_
|
||||
|
|
|
@ -35,8 +35,7 @@
|
|||
#include "profiler/device/gpu/gpu_profiling_utils.h"
|
||||
#include "backend/session/kernel_graph.h"
|
||||
#include "backend/kernel_compiler/gpu/gpu_kernel.h"
|
||||
#include "debug/rdr/recorder_manager.h"
|
||||
#include "debug/rdr/mem_address_recorder.h"
|
||||
#include "debug/rdr/running_data_recorder.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace device {
|
||||
|
@ -389,18 +388,10 @@ bool GPUDeviceContext::SyncStream(size_t stream_id) const {
|
|||
bool result = GPUDeviceManager::GetInstance().SyncStream(streams_[stream_id]);
|
||||
#ifdef ENABLE_DUMP_IR
|
||||
if (!result) {
|
||||
RecorderManager::Instance().TriggerAll();
|
||||
mindspore::RDR::TriggerAll();
|
||||
}
|
||||
// clear RDR gpu memory info
|
||||
if (RecorderManager::Instance().CheckRdrGPUMemIsRecord()) {
|
||||
std::string name = "mem_address_list";
|
||||
std::string submodule_name = "KERNEL";
|
||||
auto recorder = RecorderManager::Instance().GetRecorder(submodule_name, name);
|
||||
if (recorder != nullptr) {
|
||||
auto mem_recorder = std::dynamic_pointer_cast<GPUMemAddressRecorder>(recorder);
|
||||
mem_recorder->CleanUp();
|
||||
}
|
||||
}
|
||||
mindspore::RDR::ClearGPUMemAddressInfo();
|
||||
#endif
|
||||
return result;
|
||||
}
|
||||
|
|
|
@ -17,6 +17,7 @@ log module
|
|||
"""
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import time
|
||||
import logging
|
||||
|
@ -289,6 +290,37 @@ def _get_env_config():
|
|||
config_dict[key] = value.strip()
|
||||
return config_dict
|
||||
|
||||
def check_directory_by_regular(target, reg=None, flag=re.ASCII, prim_name=None):
|
||||
"""Check whether directory is legitimate."""
|
||||
if not isinstance(target, str):
|
||||
raise ValueError("Args directory {} must be string, please check it".format(target))
|
||||
if reg is None:
|
||||
reg = r"^[\/0-9a-zA-Z\_\-\.\:\\]+$"
|
||||
if re.match(reg, target, flag) is None:
|
||||
prim_name = f'in `{prim_name}`' if prim_name else ""
|
||||
raise ValueError("'{}' {} is illegal, it should be match regular'{}' by flags'{}'".format(
|
||||
target, prim_name, reg, flag))
|
||||
return True
|
||||
|
||||
def _make_directory(path: str):
|
||||
"""Make directory."""
|
||||
if path is None or not isinstance(path, str) or path.strip() == "":
|
||||
raise TypeError("Input path '{}' is invalid type".format(path))
|
||||
|
||||
path = os.path.realpath(path)
|
||||
check_directory_by_regular(path)
|
||||
if os.path.exists(path):
|
||||
real_path = path
|
||||
else:
|
||||
try:
|
||||
permissions = os.R_OK | os.W_OK | os.X_OK
|
||||
os.umask(permissions << 3 | permissions)
|
||||
mode = permissions << 6
|
||||
os.makedirs(path, mode=mode, exist_ok=True)
|
||||
real_path = path
|
||||
except PermissionError:
|
||||
raise TypeError("No write permission on the directory `{path}`.")
|
||||
return real_path
|
||||
|
||||
def _verify_config(kwargs):
|
||||
"""
|
||||
|
@ -330,8 +362,7 @@ def _verify_config(kwargs):
|
|||
if console == _std_off and file_path is not None:
|
||||
file_real_path = os.path.realpath(file_path)
|
||||
if not os.path.exists(file_real_path):
|
||||
raise ValueError(f'The file path does not exist. '
|
||||
f'{_confmap_dict["filepath"]}:{file_path}')
|
||||
_make_directory(file_real_path)
|
||||
|
||||
# Check the input value of maxBytes
|
||||
max_bytes = kwargs.get('maxBytes', None)
|
||||
|
|
Loading…
Reference in New Issue