p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!26865 Refactor original kernal name so all output tensors are included in stat dump 

Merge pull request !26865 from Jimmy Qi/stat-hotfix
This commit is contained in:
i-robot 2021-11-27 04:22:42 +00:00 committed by Gitee
parent 4a82477626 2a4a98ce9d
commit e2b5ee71c2
6 changed files with 51 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
mindspore/ccsrc/debug/data_dump/e2e_dump.cc
View File

@ -176,8 +176,8 @@ void E2eDump::DumpOutputImpl(const CNodePtr &node, bool trans_flag, const std::s
std::to_string(j);
if (IsDeviceTargetGPU()) {
if (DumpJsonParser::GetInstance().IsStatisticDump()) {
TensorStatDump stat_dump(GetKernelNodeName(node), op_type, op_name, task_id, stream_id, timestamp, false, j);
stat_dump.DumpTensorStatsToFile(dump_path, debugger);
TensorStatDump stat_dump(op_type, op_name, task_id, stream_id, timestamp, false, j, j);
stat_dump.DumpTensorStatsToFile(GetKernelNodeName(node), dump_path, debugger);
}
if (DumpJsonParser::GetInstance().IsTensorDump()) {
DumpGPUMemToFile(file_path, GetKernelNodeName(node), *addr, int_shapes, type, device_type, trans_flag, j,
@ -261,8 +261,8 @@ void E2eDump::DumpInputImpl(const CNodePtr &node, bool trans_flag, const std::st
MS_EXCEPTION_IF_NULL(addr);
if (IsDeviceTargetGPU()) {
if (DumpJsonParser::GetInstance().IsStatisticDump()) {
TensorStatDump stat_dump(tensor_name, op_type, op_name, task_id, stream_id, timestamp, true, slot);
stat_dump.DumpTensorStatsToFile(dump_path, debugger);
TensorStatDump stat_dump(op_type, op_name, task_id, stream_id, timestamp, true, j, slot);
stat_dump.DumpTensorStatsToFile(tensor_name, dump_path, debugger);
}
if (DumpJsonParser::GetInstance().IsTensorDump()) {
DumpGPUMemToFile(file_path, tensor_name, *addr, int_shapes, type, device_type, trans_flag, slot, debugger);
@ -313,8 +313,8 @@ void E2eDump::DumpSingleAnfNode(const AnfNodePtr &anf_node, const size_t output_
std::to_string(stream_id) + '.' + std::to_string(timestamp) + ".output.0";
if (IsDeviceTargetGPU()) {
if (dump_json_parser.IsStatisticDump()) {
TensorStatDump stat_dump(node_name, "Parameter", dump_name, task_id, stream_id, timestamp, false, 0);
stat_dump.DumpTensorStatsToFile(dump_path, debugger);
TensorStatDump stat_dump("Parameter", dump_name, task_id, stream_id, timestamp, false, 0, 0);
stat_dump.DumpTensorStatsToFile(node_name, dump_path, debugger);
}
if (dump_json_parser.IsTensorDump()) {
DumpGPUMemToFile(file_path, node_name, *addr, int_shapes, type, device_type, trans_flag, 0, debugger);
@ -433,7 +433,7 @@ void E2eDump::DumpData(const session::KernelGraph *graph, uint32_t rank_id, cons
std::string dump_path = GenerateDumpPath(graph_id, rank_id);
if (dump_json_parser.IsStatisticDump()) {
CsvWriter::GetInstance().OpenFile(dump_path);
TensorStatDump::OpenStatisticsFile(dump_path);
}
DumpInput(graph, dump_path, debugger);
DumpOutput(graph, dump_path, debugger);

58
mindspore/ccsrc/debug/data_dump/tensor_stat_dump.cc
View File

@ -30,6 +30,7 @@ constexpr auto kCsvHeader =
"Count,Negative Zero Count,Positive Zero Count,NaN Count,Negative Inf Count,Positive Inf Count,Zero Count\n";
constexpr auto kCsvFileName = "statistic.csv";
} // namespace
namespace mindspore {
bool CsvWriter::OpenFile(const std::string &path, const std::string &header) {
if (file_.is_open() && path == file_path_str_) {
@ -38,13 +39,20 @@ bool CsvWriter::OpenFile(const std::string &path, const std::string &header) {
if (file_.is_open()) {
CloseFile();
}
auto file_path = Common::CreatePrefixPath(path);
if (!file_path.has_value()) {
MS_LOG(WARNING) << "CreatePrefixPath failed.";
return false;
}
// try to open file
std::string file_path_value = file_path.value();
bool first_time_opening = file_path_str_ != path;
ChangeFileMode(path, S_IWUSR);
ChangeFileMode(file_path_value, S_IWUSR);
if (first_time_opening) {
// remove any possible output from previous runs
file_.open(path, std::ios::out | std::ios::trunc | std::ios::binary);
file_.open(file_path_value, std::ios::out | std::ios::trunc | std::ios::binary);
} else {
file_.open(path, std::ios::out | std::ios::app | std::ios::binary);
file_.open(file_path_value, std::ios::out | std::ios::app | std::ios::binary);
}
if (!file_.is_open()) {
MS_LOG(WARNING) << "Open file " << path << " failed." << ErrnoToString(errno);
@ -55,7 +63,7 @@ bool CsvWriter::OpenFile(const std::string &path, const std::string &header) {
file_.flush();
file_path_str_ = path;
}
MS_LOG(INFO) << "Opened statistics file: " << path;
MS_LOG(INFO) << "Opened file: " << path;
return true;
}
@ -80,16 +88,16 @@ void CsvWriter::WriteToCsv(const T &val, bool end_line) {
CsvWriter::~CsvWriter() { CloseFile(); }
TensorStatDump::TensorStatDump(const std::string &original_kernel_name, const std::string &op_type,
const std::string &op_name, uint32_t task_id, uint32_t stream_id, uint64_t timestamp,
bool input, size_t slot)
: original_kernel_name_{original_kernel_name},
op_type_{op_type},
TensorStatDump::TensorStatDump(const std::string &op_type, const std::string &op_name, uint32_t task_id,
uint32_t stream_id, uint64_t timestamp, bool input, size_t slot,
size_t tensor_loader_slot)
: op_type_{op_type},
op_name_{op_name},
task_id_{task_id},
stream_id_{stream_id},
timestamp_{timestamp},
slot_{slot} {
slot_{slot},
tensor_loader_slot_{tensor_loader_slot} {
if (input) {
io_ = kInput;
} else {
@ -97,35 +105,37 @@ TensorStatDump::TensorStatDump(const std::string &original_kernel_name, const st
}
}
void TensorStatDump::DumpTensorStatsToFile(const std::string &dump_path, const Debugger *debugger) {
bool TensorStatDump::OpenStatisticsFile(const std::string &dump_path) {
std::string filename = dump_path + "/" + kCsvFileName;
auto file_path = Common::CreatePrefixPath(filename);
if (!file_path.has_value()) {
MS_LOG(WARNING) << "CreatePrefixPath failed.";
return;
}
// try to open file
CsvWriter &csv = CsvWriter::GetInstance();
std::string file_path_value = file_path.value();
int retry = 2;
while (retry > 0) {
if (csv.OpenFile(file_path_value, kCsvHeader)) {
if (csv.OpenFile(filename, kCsvHeader)) {
break;
}
retry--;
}
if (!retry) {
MS_LOG(WARNING) << "Open statistic dump file failed, skipping current statistics";
return;
return false;
}
return true;
}
bool TensorStatDump::DumpTensorStatsToFile(const std::string &original_kernel_name, const std::string &dump_path,
const Debugger *debugger) {
if (!OpenStatisticsFile(dump_path)) {
return false;
}
// get tensor statistics using debugger
std::string tensor_loader_name = original_kernel_name_ + ":" + std::to_string(slot_);
std::string tensor_loader_name = original_kernel_name + ":" + std::to_string(tensor_loader_slot_);
std::shared_ptr<TensorData> data = debugger->GetTensor(tensor_loader_name);
if (data == nullptr) {
MS_LOG(WARNING) << "Failed to find tensor in tensor loader, skipping current statistics";
return;
MS_LOG(WARNING) << "Failed to find " << tensor_loader_name << " in tensor loader, skipping current statistics";
return false;
}
const DebugServices::TensorStat &stat = debugger->GetTensorStatistics(data);
const DebugServices::TensorStat &stat = DebugServices::GetTensorStatistics(data);
// write tensor statistics to csv file
std::ostringstream shape;
shape << "\"(";
@ -133,6 +143,7 @@ void TensorStatDump::DumpTensorStatsToFile(const std::string &dump_path, const D
shape << (i ? "," : "") << stat.shape[i];
}
shape << ")\"";
CsvWriter &csv = CsvWriter::GetInstance();
csv.WriteToCsv(op_type_);
csv.WriteToCsv(op_name_);
csv.WriteToCsv(task_id_);
@ -153,5 +164,6 @@ void TensorStatDump::DumpTensorStatsToFile(const std::string &dump_path, const D
csv.WriteToCsv(stat.neg_inf_count);
csv.WriteToCsv(stat.pos_inf_count);
csv.WriteToCsv(stat.zero_count, true);
return true;
}
} // namespace mindspore

11
mindspore/ccsrc/debug/data_dump/tensor_stat_dump.h
View File

@ -51,7 +51,6 @@ class TensorStatDump {
static const char CSV_HEADER[];
static const char CSV_FILE_NAME[];
const std::string &original_kernel_name_;
const std::string &op_type_;
const std::string &op_name_;
uint32_t task_id_;
@ -59,11 +58,15 @@ class TensorStatDump {
uint64_t timestamp_;
std::string io_;
size_t slot_;
size_t tensor_loader_slot_;
public:
TensorStatDump(const std::string &original_kernel_name, const std::string &op_type, const std::string &op_name,
uint32_t task_id, uint32_t stream_id, uint64_t timestamp, bool input, size_t slot);
void DumpTensorStatsToFile(const std::string &dump_path, const Debugger *debugger);
static bool OpenStatisticsFile(const std::string &dump_path);
TensorStatDump(const std::string &op_type, const std::string &op_name, uint32_t task_id, uint32_t stream_id,
uint64_t timestamp, bool input, size_t slot, size_t tensor_loader_slot_);
bool DumpTensorStatsToFile(const std::string &original_kernel_name, const std::string &dump_path,
const Debugger *debugger);
};
} // namespace mindspore
#endif // MINDSPORE_MINDSPORE_CCSRC_DEBUG_DATA_DUMP_TENSOR_STAT_DUMP_H_

6
mindspore/ccsrc/debug/debugger/debugger.cc
View File

@ -1140,7 +1140,7 @@ std::list<TensorSummary> Debugger::LoadTensorsStat(const ProtoVector<TensorProto
continue;
}
// tensor was found creating tensor summary object.
DebugServices::TensorStat tensor_stat = debug_services_->GetTensorStatistics(tensor);
DebugServices::TensorStat tensor_stat = DebugServices::GetTensorStatistics(tensor);
AddTensorStatInfo(tensor_stat, &tensor_summary_list);
}
return tensor_summary_list;
@ -1150,10 +1150,6 @@ std::shared_ptr<TensorData> Debugger::GetTensor(const std::string &tensor_name)
return debug_services_->GetTensor(tensor_name);
}
DebugServices::TensorStat Debugger::GetTensorStatistics(std::shared_ptr<TensorData> tensor_data) const {
return DebugServices::GetTensorStatistics(tensor_data);
}
void Debugger::Exit(bool exit_success) {
// debugger will notify main thread to exit because main thread can only exit at step boundary.
MS_LOG(INFO) << "Exit Debugger";

2
mindspore/ccsrc/debug/debugger/debugger.h
View File

@ -117,8 +117,6 @@ class Debugger : public std::enable_shared_from_this<Debugger> {
std::shared_ptr<TensorData> GetTensor(const std::string &tensor_name) const;
DebugServices::TensorStat GetTensorStatistics(std::shared_ptr<TensorData> tensor_data) const;
bool debugger_enabled() const;
bool partial_memory() const;

2
mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.cc
View File

@ -343,7 +343,7 @@ std::vector<TensorStatData> DbgServices::ReadTensorsStat(const std::vector<tenso
AddTensorStatInfo(tensor_statistics, &tensors_read_stat);
continue;
}
DebugServices::TensorStat tensor_statistics = debug_services_->GetTensorStatistics(result);
DebugServices::TensorStat tensor_statistics = DebugServices::GetTensorStatistics(result);
AddTensorStatInfo(tensor_statistics, &tensors_read_stat);
}