Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

dump statistic file in kernel by kernel mode

This commit is contained in:
TinaMengtingZhang 2022-02-14 17:48:36 -05:00
parent c45e8b5c9b
commit e576292862
2 changed files with 37 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
mindspore/ccsrc/debug/data_dump/e2e_dump.cc
View File

@ -182,17 +182,18 @@ void E2eDump::DumpOutputImpl(const CNodePtr &node, bool trans_flag, const std::s
std::string file_path = dump_path + '/' + op_type + '.' + op_name + '.' + std::to_string(task_id) + '.' +
std::to_string(stream_id) + '.' + std::to_string(timestamp) + ".output." +
std::to_string(j);
if (IsDeviceTargetGPU()) {
if (DumpJsonParser::GetInstance().IsStatisticDump()) {
TensorStatDump stat_dump(op_type, op_name, task_id, stream_id, timestamp, false, j, j);
stat_dump.DumpTensorStatsToFile(GetKernelNodeName(node), dump_path, debugger);
}
if (DumpJsonParser::GetInstance().IsTensorDump()) {
if (DumpJsonParser::GetInstance().IsStatisticDump() &&
(IsDeviceTargetGPU() || Debugger::GetInstance()->GetAscendKernelByKernelFlag())) {
TensorStatDump stat_dump(op_type, op_name, task_id, stream_id, timestamp, false, j, j);
stat_dump.DumpTensorStatsToFile(GetKernelNodeName(node), dump_path, debugger);
}
if (DumpJsonParser::GetInstance().IsTensorDump()) {
if (IsDeviceTargetGPU()) {
DumpGPUMemToFile(file_path, GetKernelNodeName(node), *addr, int_shapes, type, device_type, trans_flag, j,
debugger);
} else {
DumpMemToFile(file_path, *addr, int_shapes, type, trans_flag);
}
} else {
DumpMemToFile(file_path, *addr, int_shapes, type, trans_flag);
}
}
}
@ -249,7 +250,7 @@ void E2eDump::DumpInputImpl(const CNodePtr &node, bool trans_flag, const std::st
std::string tensor_name = GetKernelNodeName(node);
size_t slot = j;
if (IsDeviceTargetGPU()) {
if (IsDeviceTargetGPU() || Debugger::GetInstance()->GetAscendKernelByKernelFlag()) {
auto input_kernel = node->input(j + 1);
std::string input_kernel_name = GetKernelNodeName(input_kernel);
tensor_name = input_kernel_name;
@ -267,16 +268,17 @@ void E2eDump::DumpInputImpl(const CNodePtr &node, bool trans_flag, const std::st
std::string file_path = dump_path + '/' + op_type + '.' + op_name + '.' + std::to_string(task_id) + '.' +
std::to_string(stream_id) + '.' + std::to_string(timestamp) + ".input." + std::to_string(j);
MS_EXCEPTION_IF_NULL(addr);
if (IsDeviceTargetGPU()) {
if (DumpJsonParser::GetInstance().IsStatisticDump()) {
TensorStatDump stat_dump(op_type, op_name, task_id, stream_id, timestamp, true, j, slot);
stat_dump.DumpTensorStatsToFile(tensor_name, dump_path, debugger);
}
if (DumpJsonParser::GetInstance().IsTensorDump()) {
if (DumpJsonParser::GetInstance().IsStatisticDump() &&
(IsDeviceTargetGPU() || Debugger::GetInstance()->GetAscendKernelByKernelFlag())) {
TensorStatDump stat_dump(op_type, op_name, task_id, stream_id, timestamp, true, j, slot);
stat_dump.DumpTensorStatsToFile(tensor_name, dump_path, debugger);
}
if (DumpJsonParser::GetInstance().IsTensorDump()) {
if (IsDeviceTargetGPU()) {
DumpGPUMemToFile(file_path, tensor_name, *addr, int_shapes, type, device_type, trans_flag, slot, debugger);
} else {
DumpMemToFile(file_path, *addr, int_shapes, type, trans_flag);
}
} else {
DumpMemToFile(file_path, *addr, int_shapes, type, trans_flag);
}
}
}

18
tests/st/dump/test_data_dump.py
View File

@ -555,6 +555,24 @@ def test_ascend_statistic_dump():
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
run_saved_data_dump_test('test_async_dump', 'statistic')
@pytest.mark.level0
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@security_off_wrap
def test_ascend_statistic_dump_kernel_by_kernel():
"""
Feature: Ascend Statistics Dump in kernel by kernel (mindRT) mode
Description: Test Ascend statistics dump
Expectation: Statistics are stored in statistic.csv files
"""
# set env `GRAPH_OP_RUN`` to enable kernel-by-kernel mode.
os.environ['GRAPH_OP_RUN'] = "1"
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
run_saved_data_dump_test('test_async_dump', 'statistic')
del os.environ['GRAPH_OP_RUN']
@pytest.mark.level0
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training