p27049618
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!31539 Fix codecheck warnings for debug module 

Merge pull request !31539 from maning202007/master
This commit is contained in:
i-robot 2022-03-21 01:58:50 +00:00 committed by Gitee
parent f5010ec8b8 c35fe12a12
commit 9bede7a97b
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
11 changed files with 53 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
mindspore/ccsrc/common/debug/env_config_parser.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
* Copyright 2021-2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -287,14 +287,14 @@ void EnvConfigParser::ParseRdrPath(const nlohmann::json &content) {
void EnvConfigParser::ConfigToString() {
std::string cur_config;
#ifdef ENABLE_DUMP_IR
cur_config.append("After parsed, ");
cur_config.append("rdr_enable: ");
(void)cur_config.append("After parsed, ");
(void)cur_config.append("rdr_enable: ");
std::string rdr_enable_flag = rdr_enabled_ ? "1" : "0";
(void)cur_config.append(rdr_enable_flag);
cur_config.append(", rdr mode: ");
cur_config.append(std::to_string(rdr_mode_));
cur_config.append(", rdr path: ");
cur_config.append(rdr_path_);
(void)cur_config.append(", rdr mode: ");
(void)cur_config.append(std::to_string(rdr_mode_));
(void)cur_config.append(", rdr path: ");
(void)cur_config.append(rdr_path_);
#endif
MS_LOG(INFO) << cur_config;
}

10
mindspore/ccsrc/debug/data_dump/dump_json_parser.cc
View File

@ -365,10 +365,10 @@ void DumpJsonParser::ParseDumpMode(const nlohmann::json &content) {
MS_EXCEPTION_IF_NULL(context);
CheckJsonUnsignedType(content, kDumpMode);
dump_mode_ = content;
if (dump_mode_ < DUMP_ALL || dump_mode_ > DUMP_KERNELS_WITH_FLAG) {
if (dump_mode_ > static_cast<uint32_t>(DUMP_KERNELS_WITH_FLAG)) {
MS_LOG(EXCEPTION) << "Dump config parse failed, dump_mode should be 0, 1 or 2, but got " << dump_mode_;
}
if (dump_mode_ == DUMP_KERNELS_WITH_FLAG) {
if (dump_mode_ == static_cast<uint32_t>(DUMP_KERNELS_WITH_FLAG)) {
if (context->get_param<std::string>(MS_CTX_DEVICE_TARGET) != kAscendDevice || e2e_dump_enabled_) {
MS_LOG(EXCEPTION) << "Cell dump is only supported in Ascend async dump. Please set dump_mode to 0 or 1.";
}
@ -537,7 +537,7 @@ void DumpJsonParser::ParseInputOutput(const nlohmann::json &content) {
void DumpJsonParser::ParseKernels(const nlohmann::json &content) {
CheckJsonArrayType(content, kKernels);
if (dump_mode_ != DUMP_KERNEL) {
if (dump_mode_ != static_cast<uint32_t>(DUMP_KERNEL)) {
MS_LOG(INFO) << "Dump config field <" << kKernels << "> is not used as the dump mode is not 1.";
return;
}
@ -688,7 +688,7 @@ void DumpJsonParser::MatchKernel(const std::string &kernel_name) {
}
void DumpJsonParser::PrintUnusedKernel() {
if ((!e2e_dump_enabled_ && !async_dump_enabled_) || dump_mode_ != DUMP_KERNEL) {
if ((!e2e_dump_enabled_ && !async_dump_enabled_) || dump_mode_ != static_cast<uint32_t>(DUMP_KERNEL)) {
return;
}
for (const auto &iter : kernels_) {
@ -748,7 +748,7 @@ bool DumpJsonParser::OutputNeedDump() const {
* Description: Obtain the cell dump flag of each operators in the given kernel graph.
*/
void DumpJsonParser::GetCellDumpFlag(const session::KernelGraph &kernel_graph) {
if (dump_mode_ != DUMP_KERNELS_WITH_FLAG) {
if (dump_mode_ != static_cast<uint32_t>(DUMP_KERNELS_WITH_FLAG)) {
return;
}
for (const auto &kernel : kernel_graph.execution_order()) {

2
mindspore/ccsrc/debug/data_dump/dump_json_parser.h
View File

@ -100,7 +100,7 @@ class BACKEND_EXPORT DumpJsonParser {
std::vector<std::string> cell_dump_kernels_;
std::set<uint32_t> support_devices_;
uint32_t op_debug_mode_{0};
JsonFileFormat file_format_;
JsonFileFormat file_format_{FORMAT_BIN};
bool trans_flag_{false};
uint32_t cur_dump_iter_{0};
bool already_parsed_{false};

27
mindspore/ccsrc/debug/data_dump/e2e_dump.cc
View File

@ -124,15 +124,14 @@ bool E2eDump::IsDeviceTargetGPU() {
* Runtime category: Old runtime, MindRT.
* Description: This function is for dumping tensor in memory to disk in GPU machine.
*/
void E2eDump::DumpGPUMemToFile(const std::string &file_path, const std::string &original_kernel_name,
const device::DeviceAddress &addr, const ShapeVector &int_shapes,
const TypeId &host_type, const TypeId &device_type, bool trans_flag, size_t slot,
const Debugger *debugger) {
void E2eDump::DumpGPUMemToFile(const Debugger *debugger, const std::string &file_path, bool trans_flag,
const device::DeviceAddress &addr, const std::string &original_kernel_name, size_t slot,
const ShapeVector &int_shapes, const TypeId &host_type) {
#ifdef ENABLE_DEBUGGER
auto format = kOpFormat_DEFAULT;
MS_EXCEPTION_IF_NULL(debugger);
auto ret = debugger->DumpTensorToFile(original_kernel_name, trans_flag, file_path, format, int_shapes, host_type,
device_type, addr.format(), slot);
auto ret = debugger->DumpTensorToFile(file_path, trans_flag, format, addr.format(), original_kernel_name, slot,
int_shapes, host_type);
if (!ret) {
MS_LOG(INFO) << "DumpTensorToFile Failed: flag:" << trans_flag << ", path:" << file_path
<< ", host_format:" << format;
@ -189,7 +188,6 @@ void E2eDump::DumpOutputImpl(const CNodePtr &node, bool trans_flag, const std::s
ShapeVector int_shapes;
GetDumpIntShape(node, j, NOT_NULL(&int_shapes), trans_flag);
auto type = common::AnfAlgo::GetOutputInferDataType(node, j);
auto device_type = AnfAlgo::GetOutputDeviceDataType(node, j);
std::string op_type = common::AnfAlgo::GetCNodeName(node);
std::string op_name = GetOpNameWithoutScope(*kernel_name);
uint32_t task_id = 0;
@ -205,8 +203,7 @@ void E2eDump::DumpOutputImpl(const CNodePtr &node, bool trans_flag, const std::s
}
if (DumpJsonParser::GetInstance().IsTensorDump()) {
if (IsDeviceTargetGPU()) {
DumpGPUMemToFile(file_path, GetKernelNodeName(node), *addr, int_shapes, type, device_type, trans_flag, j,
debugger);
DumpGPUMemToFile(debugger, file_path, trans_flag, *addr, GetKernelNodeName(node), j, int_shapes, type);
} else {
DumpMemToFile(file_path, *addr, int_shapes, type, trans_flag);
}
@ -312,7 +309,6 @@ void E2eDump::DumpInputImpl(const CNodePtr &node, bool trans_flag, const std::st
ShapeVector int_shapes;
GetDumpIntShape(input, index, NOT_NULL(&int_shapes), trans_flag);
auto type = common::AnfAlgo::GetOutputInferDataType(input, index);
auto device_type = AnfAlgo::GetOutputDeviceDataType(input, index);
std::string op_type = common::AnfAlgo::GetCNodeName(node);
std::string op_name = GetOpNameWithoutScope(*kernel_name);
uint64_t timestamp = GetTimeStamp();
@ -329,7 +325,7 @@ void E2eDump::DumpInputImpl(const CNodePtr &node, bool trans_flag, const std::st
}
if (DumpJsonParser::GetInstance().IsTensorDump()) {
if (IsDeviceTargetGPU()) {
DumpGPUMemToFile(file_path, tensor_name, *addr, int_shapes, type, device_type, trans_flag, slot, debugger);
DumpGPUMemToFile(debugger, file_path, trans_flag, *addr, tensor_name, slot, int_shapes, type);
} else if (Debugger::GetInstance()->GetAscendKernelByKernelFlag()) {
// load address from launch_info when it's Ascend Kernel by kernel mode.
auto ascend_device_addr = CreateAscendDeviceAddress(launch_info, j, type);
@ -402,7 +398,6 @@ void E2eDump::DumpSingleAnfNode(const AnfNodePtr &anf_node, const size_t output_
ShapeVector int_shapes;
GetDumpIntShape(anf_node, output_index, NOT_NULL(&int_shapes), trans_flag);
auto type = common::AnfAlgo::GetOutputInferDataType(anf_node, output_index);
auto device_type = AnfAlgo::GetOutputDeviceDataType(anf_node, output_index);
uint64_t timestamp = GetTimeStamp();
uint32_t task_id = 0;
uint32_t stream_id = 0;
@ -414,7 +409,7 @@ void E2eDump::DumpSingleAnfNode(const AnfNodePtr &anf_node, const size_t output_
(void)stat_dump.DumpTensorStatsToFile(node_name, dump_path, debugger);
}
if (dump_json_parser.IsTensorDump()) {
DumpGPUMemToFile(file_path, node_name, *addr, int_shapes, type, device_type, trans_flag, 0, debugger);
DumpGPUMemToFile(debugger, file_path, trans_flag, *addr, node_name, 0, int_shapes, type);
}
} else {
DumpMemToFile(file_path, *addr, int_shapes, type, trans_flag);
@ -457,7 +452,7 @@ void E2eDump::DumpSingleParameterNode(const AnfNodePtr &anf_node, const std::str
(void)stat_dump.DumpTensorStatsToFile(node_name, dump_path, debugger);
}
if (dump_json_parser.IsTensorDump()) {
DumpGPUMemToFile(file_path, node_name, *addr, int_shapes, type, device_type, trans_flag, 0, debugger);
DumpGPUMemToFile(debugger, file_path, trans_flag, *addr, node_name, 0, int_shapes, type);
}
} else {
DumpMemToFile(file_path, *addr, int_shapes, type, trans_flag);
@ -771,7 +766,9 @@ void E2eDump::DumpTensorToFile(const std::string &dump_path, const debugger::dum
// If the total tensor size is less than 1Mb, do it in single thread.
ConvertFormatForTensors(&dump_tensor_vec, 0, dump_tensor_vec.size() - 1);
} else {
auto default_num_workers = std::max<uint32_t>(1, std::thread::hardware_concurrency() / 4);
// In multi_thread process, we only use 1/4 of the total concurrent threads.
uint32_t ratio_divider = 4;
auto default_num_workers = std::max<uint32_t>(1, std::thread::hardware_concurrency() / ratio_divider);
auto num_threads = std::min<uint32_t>(default_num_workers, dump_tensor_vec.size());
uint32_t task_size = dump_tensor_vec.size() / num_threads;
uint32_t remainder = dump_tensor_vec.size() % num_threads;

7
mindspore/ccsrc/debug/data_dump/e2e_dump.h
View File

@ -105,10 +105,9 @@ class E2eDump {
static void DumpParameters(const session::KernelGraph *graph, const std::string &dump_path, const Debugger *debugger);
static void DumpGPUMemToFile(const std::string &file_path, const std::string &original_kernel_name,
const device::DeviceAddress &addr, const ShapeVector &int_shapes,
const TypeId &host_type, const TypeId &device_type, bool trans_flag, size_t slot,
const Debugger *debugger);
static void DumpGPUMemToFile(const Debugger *debugger, const std::string &file_path, bool trans_flag,
const device::DeviceAddress &addr, const std::string &original_kernel_name, size_t slot,
const ShapeVector &int_shapes, const TypeId &host_type);
static bool IsDeviceTargetGPU();
static void DumpSingleAnfNode(const AnfNodePtr &anf_node, const size_t output_index, const std::string &dump_path,
bool trans_flag, const Debugger *debugger);

21
mindspore/ccsrc/debug/debug_services.cc
View File

@ -43,8 +43,8 @@
namespace mindspore {
namespace {
static constexpr const char *constant_prefix = "Default--data-";
static constexpr const char *kNpyExt = ".npy";
static constexpr const char constant_prefix[] = "Default--data-";
static constexpr const char kNpyExt[] = ".npy";
constexpr float ms_to_s = 1000.0;
constexpr int precision = 2;
static constexpr int32_t wp_progress_period = 300;
@ -1155,8 +1155,8 @@ void DebugServices::ProcessConvertList(const DumpFileMap &dump_dir_mapped_files,
}
// Add the already converted npy files to result_list
if (npy_files.find(prefix_dump_file_name) != npy_files.end()) {
std::copy(npy_files[prefix_dump_file_name].begin(), npy_files[prefix_dump_file_name].end(),
std::inserter(*result_list, result_list->end()));
(void)std::copy(npy_files[prefix_dump_file_name].begin(), npy_files[prefix_dump_file_name].end(),
std::inserter(*result_list, result_list->end()));
}
}
@ -1535,7 +1535,6 @@ void DebugServices::ReadDumpedTensor(std::vector<std::string> backend_name, std:
bool is_cst = false;
// prefix_dump_to_check is node name used to find corresponding dump file.
std::string prefix_dump_to_check = GetNodeNameWithoutScope(dump_style_kernel_name);
// if node name has prefix of "Default--data-", consider as constant, search in cst folder
if (prefix_dump_to_check.length() > (unsigned)strlen(constant_prefix) &&
prefix_dump_to_check.substr(0, (unsigned)strlen(constant_prefix)).compare(constant_prefix) == 0) {
@ -1837,12 +1836,11 @@ std::shared_ptr<TensorData> DebugServices::GetTensor(const std::string &tensor_n
void DebugServices::EmptyCurrentTensor() { tensor_loader_->EmptyCurrentTensor(); }
#ifdef ONLINE_DBG_MODE
bool DebugServices::DumpTensorToFile(const std::string &tensor_name, bool trans_flag, const std::string &filepath,
const std::string &host_fmt, const std::vector<int64_t> &host_shape,
TypeId host_type, TypeId device_type, const std::string &addr_format,
size_t slot) const {
return tensor_loader_->DumpTensorToFile(tensor_name, trans_flag, filepath, host_fmt, host_shape, host_type,
device_type, addr_format, slot);
bool DebugServices::DumpTensorToFile(const std::string &filepath, bool trans_flag, const std::string &host_fmt,
const std::string &addr_format, const std::string &tensor_name, size_t slot,
const std::vector<int64_t> &host_shape, TypeId host_type) const {
return tensor_loader_->DumpTensorToFile(filepath, trans_flag, host_fmt, addr_format, tensor_name, slot, host_shape,
host_type);
}
#endif
@ -2130,7 +2128,6 @@ bool DebugServices::GetAttrsFromFilename(const std::string &file_name, std::stri
}
size_t third_dot = file_name.rfind(".", fourth_dot - 1);
size_t second_dot = file_name.rfind(".", third_dot - 1);
// check if dots were found
if (first_dot == std::string::npos || second_dot == std::string::npos || third_dot == std::string::npos ||
fourth_dot == std::string::npos) {

14
mindspore/ccsrc/debug/debug_services.h
View File

@ -112,10 +112,6 @@ class DebugServices {
}
};
struct MappedFiles {
MappedFiles(std::vector<std::string> bin_files, std::map<std::string, std::vector<std::string>> npy_files)
: bin_files(bin_files), npy_files(npy_files) {}
MappedFiles() = default;
std::vector<std::string> bin_files;
// key is op_name and value is the vector of matched npy files to that op name.
std::map<std::string, std::vector<std::string>> npy_files;
@ -131,7 +127,7 @@ class DebugServices {
};
struct ProtoDump {
bool operator==(ProtoDump obj) {
bool operator==(const ProtoDump obj) {
return (origin_node_name == obj.origin_node_name && dump_name == obj.dump_name && is_output == obj.is_output);
}
// name_to_match is the op_name between first and second dot in file_name
@ -465,9 +461,9 @@ class DebugServices {
void EmptyCurrentTensor();
#ifdef ONLINE_DBG_MODE
bool DumpTensorToFile(const std::string &tensor_name, bool trans_flag, const std::string &filepath,
const std::string &host_fmt, const std::vector<int64_t> &host_shape, TypeId host_type,
TypeId device_type, const std::string &addr_format, size_t slot) const;
bool DumpTensorToFile(const std::string &filepath, bool trans_flag, const std::string &host_fmt,
const std::string &addr_format, const std::string &tensor_name, size_t slot,
const std::vector<int64_t> &host_shape, TypeId host_type) const;
#endif
bool LoadNewTensor(const std::shared_ptr<TensorData> &tensor, bool keep_prev);
@ -531,8 +527,6 @@ class DebugServices {
std::unordered_map<std::string, std::vector<std::string>> overflow_ops_;
std::string net_name_;
std::string dump_dir_;
// DumpFileMap for each specific dump dir (including rank, graph_id and iteration)
DumpFileMap dump_dir_mapped_files_;
// store history of graphs that have been run (rank_id, graph_id)
std::map<std::tuple<uint32_t, uint32_t>, std::vector<uint32_t>> graphs_run_history_;
bool is_sync_mode_{false};

12
mindspore/ccsrc/debug/debugger/debugger.cc
View File

@ -369,7 +369,6 @@ void Debugger::StoreRunGraphIdList(uint32_t graph_id) {
// collect rungrap_ids to update step number in multigraph case for GPU old runtime
if (!rungraph_id_list_.size()) {
rungraph_id_list_.push_back(graph_id);
} else {
if (std::find(rungraph_id_list_.begin(), rungraph_id_list_.end(), graph_id) == rungraph_id_list_.end()) {
rungraph_id_list_.push_back(graph_id);
@ -1336,11 +1335,11 @@ void Debugger::SendWatchpoints(const std::list<WatchpointHit> &points) {
}
}
bool Debugger::DumpTensorToFile(const std::string &tensor_name, bool trans_flag, const std::string &filepath,
const std::string &host_fmt, const std::vector<int64_t> &host_shape, TypeId host_type,
TypeId device_type, const std::string &addr_format, size_t slot) const {
return debug_services_.get()->DumpTensorToFile(tensor_name, trans_flag, filepath, host_fmt, host_shape, host_type,
device_type, addr_format, slot);
bool Debugger::DumpTensorToFile(const std::string &filepath, bool trans_flag, const std::string &host_fmt,
const std::string &addr_format, const std::string &tensor_name, size_t slot,
const std::vector<int64_t> &host_shape, TypeId host_type) const {
return debug_services_.get()->DumpTensorToFile(filepath, trans_flag, host_fmt, addr_format, tensor_name, slot,
host_shape, host_type);
}
bool Debugger::LoadNewTensor(const std::shared_ptr<TensorData> &tensor, bool keep_prev) {
@ -1574,7 +1573,6 @@ void Debugger::LoadSingleParameterMindRT(const AnfNodePtr &node) {
// Keep_prev is True for parameters.
// force update for parameters.
bool ret = device_addr->LoadMemToHost(tensor_name, 0, format, int_shapes, type, 0, true, root_graph_id, true);
if (!ret) {
MS_LOG(ERROR) << "LoadMemToHost:"
<< ", tensor_name:" << tensor_name << ", host_format:" << format << ".!";

6
mindspore/ccsrc/debug/debugger/debugger.h
View File

@ -117,9 +117,9 @@ class BACKEND_EXPORT Debugger : public std::enable_shared_from_this<Debugger> {
void PostExecuteNode(const CNodePtr &kernel, bool last_kernel);
bool DumpTensorToFile(const std::string &tensor_name, bool trans_flag, const std::string &filepath,
const std::string &host_fmt, const std::vector<int64_t> &host_shape, TypeId host_type,
TypeId device_type, const std::string &addr_format, size_t slot) const;
bool DumpTensorToFile(const std::string &filepath, bool trans_flag, const std::string &host_fmt,
const std::string &addr_format, const std::string &tensor_name, size_t slot,
const std::vector<int64_t> &host_shape, TypeId host_type) const;
bool LoadNewTensor(const std::shared_ptr<TensorData> &tensor, bool keep_prev);

6
mindspore/ccsrc/debug/tensor_load.h
View File

@ -244,9 +244,9 @@ class TensorLoader {
* Runtime category: Old runtime, MindRT.
* Description: Load tensor data from debugger backend cache (tensor_list_map_) and dump to file in npy format.
*/
bool DumpTensorToFile(const std::string &tensor_name, bool trans_flag, const std::string &filepath,
const std::string &host_fmt, const std::vector<int64_t> &host_shape, TypeId host_type,
TypeId device_type, const std::string &addr_format, size_t slot) {
bool DumpTensorToFile(const std::string &filepath, bool trans_flag, const std::string &host_fmt,
const std::string &addr_format, const std::string &tensor_name, size_t slot,
const std::vector<int64_t> &host_shape, TypeId host_type) {
if (filepath.empty()) {
MS_LOG(ERROR) << "Dump file path is null!";
return false;

2
mindspore/ccsrc/debug/utils.cc
View File

@ -58,7 +58,7 @@ bool CheckStoi(int64_t *const output_digit, const std::string &input_str) {
void CheckStringMatch(size_t start, size_t end, std::string *matched_str, const std::string &input_str) {
if (start != std::string::npos && end != std::string::npos && end > start && start + 1 < input_str.length()) {
*matched_str = input_str.substr(start + 1, end - start - 1);
*matched_str = input_str.substr(start + 1, end - (start + 1));
}
}
} // namespace mindspore