p60975123
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!27860 The profiler framework data is changed to binary data 

Merge pull request !27860 from helloiSCSI/data_reduce
This commit is contained in:
i-robot 2021-12-20 07:47:49 +00:00 committed by Gitee
parent ac69bac952 115ec28ea0
commit 76aaf62088
42 changed files with 1161 additions and 1326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.jenkins/check/config/filter_cpplint.txt
View File

@ -7,6 +7,8 @@
"mindspore/mindspore/core/abstract/utils.cc" "build/include_what_you_use"
"mindspore/mindspore/ccsrc/backend/optimizer/ascend/buffer_fusion/ub_pattern_fusion.cc" "build/include_what_you_use"
"mindspore/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_callback_register.cc" "runtime/references"
"mindspore/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_manager.h" "runtime/references"
"mindspore/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_reporter.h" "runtime/references"
"mindspore/mindspore/core/mindrt/src/actor/actormgr.h" "runtime/references"
"mindspore/mindspore/core/mindrt/src/actor/actorpolicyinterface.h" "runtime/references"
"mindspore/mindspore/core/mindrt/src/actor/actorthread.h" "runtime/references"

6
mindspore/ccsrc/runtime/device/CMakeLists.txt
View File

@ -90,12 +90,6 @@ if(ENABLE_SECURITY)
list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/profiling_callback_register.cc")
list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/profiling_manager.cc")
list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/profiling_utils.cc")
list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/reporter/desc_reporter.cc")
list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/reporter/graph_desc_reporter.cc")
list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/reporter/op_name_task_stream_reporter.cc")
list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/reporter/point_reporter.cc")
list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/reporter/profiling_desc.cc")
list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/reporter/task_desc_reporter.cc")
endif()
set_property(SOURCE ${DEVICE_SRC_LIST} ${D_SRC_LIST} ${CPU_SRC_LIST}

16
mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc
View File

@ -60,7 +60,6 @@
#endif
#include "runtime/device/ascend/executor/hccl_dynamic_kernel.h"
#include "utils/config_manager.h"
#include "runtime/device/ascend/profiling/reporter/op_name_task_stream_reporter.h"
#include "runtime/hccl_adapter/hccl_adapter.h"
#ifdef ENABLE_TDTQUE
#include "minddata/dataset/engine/tdt/tdt_handle.h"
@ -241,17 +240,6 @@ void AsyncDataDumpUninit() {
}
}
}
void AscendKernelRuntime::ReportProfilingData() {
auto context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context);
if (ProfilingManager::GetInstance().IsProfilingStart() &&
context->get_param<int>(MS_CTX_EXECUTION_MODE) == kPynativeMode) {
// Save Profiling Framework data
OpNameTaskStreamReporter reporter(device_id_, "nonsink", stream_id_task_id_op_name_map_);
reporter.ReportData();
}
}
#endif
void AscendKernelRuntime::ReleaseDeviceRes() {
@ -269,9 +257,7 @@ void AscendKernelRuntime::ReleaseDeviceRes() {
return;
}
SetCurrentContext();
#ifndef ENABLE_SECURITY
ReportProfilingData();
#endif
// release ge runtime
ClearGraphModelMap();

1
mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.h
View File

@ -101,7 +101,6 @@ class AscendKernelRuntime : public KernelRuntime {
#ifndef ENABLE_SECURITY
void DistributeDebugTask(const session::KernelGraph &graph, const NotNull<std::function<void *()>> &model_handle);
void LaunchDataDump(GraphId graph_id);
void ReportProfilingData();
#endif
static CNodePtr GetErrorNodeName(uint32_t streamid, uint32_t taskid);
static std::string GetDumpPath();

179
mindspore/ccsrc/runtime/device/ascend/profiling/prof_common.h Normal file
View File

@ -0,0 +1,179 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MSPROFILER_PROF_COMMON_H_
#define MSPROFILER_PROF_COMMON_H_
#include <stdint.h>
#define MSPROF_DATA_HEAD_MAGIC_NUM 0x5a5a
#define MSPROF_DATA_HASH_MIN_LEN_128 128
#define MSPROF_GE_MODELLOAD_DATA_BYTES 104
#define MSPROF_DATA_HASH_MIN_LEN_64 64
#define MSPROF_MIX_DATA_RESERVE_BYTES 7
#define MSPROF_MIX_DATA_STRING_LEN 120
enum MsprofMixDataType {
MSPROF_MIX_DATA_HASH_ID = 0,
MSPROF_MIX_DATA_STRING = 1,
};
enum MsprofDataTag {
MSPROF_GE_DATA_TAG_MODEL_LOAD = 20,
MSPROF_GE_DATA_TAG_FUSION = 21,
MSPROF_GE_DATA_TAG_INFER = 22,
MSPROF_GE_DATA_TAG_TASK = 23,
MSPROF_GE_DATA_TAG_TENSOR = 24,
MSPROF_GE_DATA_TAG_STEP = 25,
MSPROF_GE_DATA_TAG_ID_MAP = 26,
};
struct MsprofMixData {
uint8_t type;
uint8_t rsv[MSPROF_MIX_DATA_RESERVE_BYTES];
union {
uint64_t hashId;
char dataStr[MSPROF_MIX_DATA_STRING_LEN];
} data;
};
using MixData = struct MsprofMixData;
struct MsprofGeProfModelLoadData {
uint16_t magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM;
uint16_t dataTag = MSPROF_GE_DATA_TAG_MODEL_LOAD;
uint32_t modelId;
MixData modelName;
uint64_t startTime;
uint64_t endTime;
uint8_t reserve[MSPROF_GE_MODELLOAD_DATA_BYTES];
};
#define MSPROF_FUSION_DATA_RESERVE_BYTES 14
#define MSPROF_GE_FUSION_OP_NUM 8
struct MsprofGeProfFusionData {
uint16_t magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM;
uint16_t dataTag = MSPROF_GE_DATA_TAG_FUSION;
uint32_t modelId;
MixData fusionName;
uint64_t inputMemSize;
uint64_t outputMemSize;
uint64_t weightMemSize;
uint64_t workspaceMemSize;
uint64_t totalMemSize;
uint16_t fusionOpNum;
uint64_t fusionOp[MSPROF_GE_FUSION_OP_NUM];
uint8_t reserve[MSPROF_FUSION_DATA_RESERVE_BYTES];
};
#define MSPROF_GE_INFER_DATA_RESERVE_BYTES 64
struct MsprofGeProfInferData {
uint16_t magicNumber;
uint16_t dataTag;
uint32_t modelId;
MixData modelName;
uint32_t requestId;
uint32_t threadId;
uint64_t inputDataStartTime;
uint64_t inputDataEndTime;
uint64_t inferStartTime;
uint64_t inferEndTime;
uint64_t outputDataStartTime;
uint64_t outputDataEndTime;
uint8_t reserve[MSPROF_GE_INFER_DATA_RESERVE_BYTES];
};
#define MSPROF_GE_TASK_DATA_RESERVE_BYTES 16
#define MSPROF_GE_OP_TYPE_LEN 56
enum MsprofGeTaskType { MSPROF_GE_TASK_TYPE_AI_CORE = 0, MSPROF_GE_TASK_TYPE_AI_CPU, MSPROF_GE_TASK_TYPE_AIV };
enum MsprofGeShapeType { MSPROF_GE_SHAPE_TYPE_STATIC = 0, MSPROF_GE_SHAPE_TYPE_DYNAMIC };
struct MsprofGeOpType {
uint8_t type;
uint8_t rsv[MSPROF_MIX_DATA_RESERVE_BYTES];
union {
uint64_t hashId;
char dataStr[MSPROF_GE_OP_TYPE_LEN];
} data;
};
using GeOpType = struct MsprofGeOpType;
struct MsprofGeProfTaskData {
uint16_t magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM;
uint16_t dataTag = MSPROF_GE_DATA_TAG_TASK;
uint32_t taskType;
MixData opName;
GeOpType opType;
uint64_t curIterNum;
uint64_t timeStamp;
uint32_t shapeType;
uint32_t blockDims;
uint32_t modelId;
uint32_t streamId;
uint32_t taskId;
uint32_t threadId;
uint8_t reserve[MSPROF_GE_TASK_DATA_RESERVE_BYTES];
};
#define MSPROF_GE_TENSOR_DATA_RESERVE_BYTES 4
#define MSPROF_GE_TENSOR_DATA_SHAPE_LEN 8
#define MSPROF_GE_TENSOR_DATA_NUM 5
enum MsprofGeTensorType { MSPROF_GE_TENSOR_TYPE_INPUT = 0, MSPROF_GE_TENSOR_TYPE_OUTPUT };
struct MsprofGeTensorData {
uint32_t tensorType;
uint32_t format;
uint32_t dataType;
uint32_t shape[MSPROF_GE_TENSOR_DATA_SHAPE_LEN];
};
using GeTensorData = struct MsprofGeTensorData;
struct MsprofGeProfTensorData {
uint16_t magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM;
uint16_t dataTag = MSPROF_GE_DATA_TAG_TENSOR;
uint32_t modelId;
uint64_t curIterNum;
uint32_t streamId;
uint32_t taskId;
uint8_t tensorNum;
GeTensorData tensorData[MSPROF_GE_TENSOR_DATA_NUM];
uint8_t reserve[MSPROF_GE_TENSOR_DATA_RESERVE_BYTES];
};
#define MSPROF_GE_STEP_DATA_RESERVE_BYTES 27
struct MsprofGeProfStepData {
uint16_t magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM;
uint16_t dataTag = MSPROF_GE_DATA_TAG_STEP;
uint32_t modelId;
uint32_t streamId;
uint32_t taskId;
uint64_t timeStamp;
uint64_t curIterNum;
uint32_t threadId;
uint8_t tag;
uint8_t reserve[MSPROF_GE_STEP_DATA_RESERVE_BYTES];
};
#define MSRPOF_GE_ID_MAP_DATA_RESERVE_BYTES 6
struct MsprofGeProfIdMapData {
uint16_t magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM;
uint16_t dataTag = MSPROF_GE_DATA_TAG_ID_MAP;
uint32_t graphId;
uint32_t modelId;
uint32_t sessionId;
uint64_t timeStamp;
uint16_t mode;
uint8_t reserve[MSRPOF_GE_ID_MAP_DATA_RESERVE_BYTES];
};
#endif

18
mindspore/ccsrc/runtime/device/ascend/profiling/profiling_manager.cc
View File

@ -241,6 +241,24 @@ Status ProfCtrlSwitchHandle(void *data) {
}
Status ProfCommandHandle(ProfCommandHandleType type) { return ProfilingManager::GetInstance().ProfCommandHandle(type); }
void ProfilingManager::QueryHashId(const int32_t &device_id, const std::string &src_str, uint64_t &hash_id) {
// when some profiling data size exceeds the specified size, query its hashId instead.
MsprofHashData hash_data{};
hash_data.deviceId = device_id;
hash_data.dataLen = src_str.size();
hash_data.data = reinterpret_cast<unsigned char *>(const_cast<char *>(src_str.c_str()));
const int32_t ret = prof_cb_.msprofReporterCallback(
static_cast<int32_t>(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK),
static_cast<int32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_HASH), &hash_data, sizeof(MsprofHashData));
if (ret != 0) {
MS_LOG(EXCEPTION) << "[Profiling] Query hash id of long string failed, src string is " << src_str.c_str();
}
hash_id = hash_data.hashId;
}
} // namespace ascend
} // namespace device
} // namespace mindspore

1
mindspore/ccsrc/runtime/device/ascend/profiling/profiling_manager.h
View File

@ -64,6 +64,7 @@ class ProfilingManager {
Status PluginInit() const;
void PluginUnInit() const;
Status CallMsprofReport(NotNull<ReporterData *> reporter_data) const;
void QueryHashId(const int32_t &device_id, const std::string &src_str, uint64_t &hash_id);
const struct MsprofCallback &GetMsprofCallback() { return prof_cb_; }
void SetMsprofCtrlCallback(MsprofCtrlCallback func) { prof_cb_.msprofCtrlCallback = func; }
void SetMsprofReporterCallback(MsprofReporterCallback func) { prof_cb_.msprofReporterCallback = func; }

256
mindspore/ccsrc/runtime/device/ascend/profiling/profiling_reporter.cc Normal file
View File

@ -0,0 +1,256 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <map>
#include <algorithm>
#include "runtime/device/ascend/profiling/profiling_reporter.h"
#include "backend/kernel_compiler/kernel.h"
#include "backend/kernel_compiler/ascend_kernel_mod.h"
#include "utils/utils.h"
namespace mindspore {
namespace device {
namespace ascend {
static std::map<enum KernelType, MsprofGeTaskType> KernelType2TaskTypeEnum{{TBE_KERNEL, MSPROF_GE_TASK_TYPE_AI_CORE},
{AKG_KERNEL, MSPROF_GE_TASK_TYPE_AI_CORE},
{AICPU_KERNEL, MSPROF_GE_TASK_TYPE_AI_CPU}};
// 0 means unknown format
static std::map<string, uint32_t> OpFormat2Index{{kOpFormat_DEFAULT, 1},
{kOpFormat_NC1KHKWHWC0, 2},
{kOpFormat_ND, 3},
{kOpFormat_NCHW, 4},
{kOpFormat_NHWC, 5},
{kOpFormat_HWCN, 6},
{kOpFormat_NC1HWC0, 7},
{kOpFormat_FRAC_Z, 8},
{kOpFormat_C1HWNCoC0, 9},
{kOpFormat_FRAC_NZ, 10},
{kOpFormat_NC1HWC0_C04, 11},
{kOpFormat_FRACTAL_Z_C04, 12},
{kOpFormat_NDHWC, 13},
{kOpFormat_FRACTAL_ZN_LSTM, 14},
{kOpFormat_FRACTAL_ZN_RNN, 15},
{kOpFormat_ND_RNN_BIAS, 16},
{kOpFormat_NDC1HWC0, 17},
{kOpFormat_NCDHW, 18},
{kOpFormat_FRACTAL_Z_3D, 19},
{kOpFormat_DHWNC, 20},
{kOpFormat_DHWCN, 21}};
bool ProfilingReporter::CheckStreamTaskValid() {
if (cnode_list_.size() != stream_ids_.size() || cnode_list_.size() != task_ids_.size()) {
MS_LOG(ERROR) << "CNode size is not equal stream size or not equal task size, "
"can not support to report profiling data. CNode size is "
<< cnode_list_.size() << ", stream size is " << stream_ids_.size() << ", task size is "
<< task_ids_.size();
return false;
}
return true;
}
void ProfilingReporter::ReportTasks() {
MS_LOG(INFO) << "Profiling start to report tasks.";
if (!CheckStreamTaskValid()) {
return;
}
size_t task_index = 0;
for (const auto &node : cnode_list_) {
MS_EXCEPTION_IF_NULL(node);
KernelType kernel_type = AnfAlgo::GetKernelType(node);
// Note: some kernel stream id or task id will conflict, such as RT_KERNEL,
// and CPU_KERNEL does not have stream id, task id.
if (kernel_type != TBE_KERNEL && kernel_type != AKG_KERNEL && kernel_type != AICPU_KERNEL &&
kernel_type != HCCL_KERNEL) {
MS_LOG(INFO) << "This node is not TBE_KERNEL, AKG_KERNEL, AICPU_KERNEL, HCCL_KERNEL, will skip, node name:"
<< node->fullname_with_scope();
++task_index;
continue;
}
auto stream_id = stream_ids_[task_index];
auto task_id = task_ids_[task_index];
(void)ReportTask(node, stream_id, task_id, kernel_type);
(void)ReportNode(node, stream_id, task_id, MSPROF_GE_TENSOR_TYPE_INPUT);
(void)ReportNode(node, stream_id, task_id, MSPROF_GE_TENSOR_TYPE_OUTPUT);
++task_index;
}
MS_LOG(INFO) << "Profiling report task data finish.";
}
void ProfilingReporter::ReportStepPoint(const std::vector<std::shared_ptr<StepPointDesc>> &points) {
MS_LOG(INFO) << "Profiling start to report step point data.";
if (!CheckStreamTaskValid()) {
return;
}
ConstructNodeNameIndexMap();
for (const auto &point : points) {
MsprofGeProfStepData step_point{};
step_point.modelId = graph_id_;
auto op_name = point->op_name();
step_point.streamId = GetStreamId(op_name);
step_point.taskId = GetTaskId(op_name);
step_point.timeStamp = 0;
step_point.curIterNum = 0;
step_point.threadId = 0;
step_point.tag = point->tag();
(void)ReportData(device_id_, reinterpret_cast<unsigned char *>(&step_point), sizeof(step_point), "step_info");
MS_LOG(WARNING) << "ReportStepPoint, graph_id: " << graph_id_ << ", op_name: " << point->op_name()
<< ", streamId: " << GetStreamId(op_name) << ", task Id: " << GetTaskId(op_name);
}
}
uint32_t ProfilingReporter::GetStreamId(const string &node_name) {
auto index = node_name_index_map_[node_name];
return stream_ids_[index];
}
uint32_t ProfilingReporter::GetTaskId(const string &node_name) {
auto index = node_name_index_map_[node_name];
return task_ids_[index];
}
void ProfilingReporter::ReportData(int32_t device_id, unsigned char *data, size_t data_size, const string &tag_name) {
ReporterData report_data{};
report_data.deviceId = device_id;
report_data.data = data;
report_data.dataLen = data_size;
auto ret = memcpy_s(report_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, tag_name.c_str(), tag_name.length());
if (ret != 0) {
MS_LOG(EXCEPTION) << "Report data failed, tag is " << tag_name.c_str() << ", ret: " << ret;
}
auto report_ret = ProfilingManager::GetInstance().CallMsprofReport(NOT_NULL(&report_data));
if (report_ret != 0) {
MS_LOG(EXCEPTION) << "Report data failed, tag is " << tag_name.c_str() << ", ret: " << ret;
}
}
void ProfilingReporter::ConstructNodeNameIndexMap() {
if (!node_name_index_map_.empty()) {
return;
}
size_t task_index = 0;
for (const auto &node : cnode_list_) {
MS_EXCEPTION_IF_NULL(node);
node_name_index_map_.insert(pair<string, uint32_t>(node->fullname_with_scope(), task_index));
++task_index;
}
}
uint32_t ProfilingReporter::GetBlockDim(const CNodePtr &node) {
auto kernel_mod = AnfAlgo::GetKernelMod(node);
auto ascend_kernel_mod = dynamic_cast<kernel::AscendKernelMod *>(kernel_mod);
MS_EXCEPTION_IF_NULL(ascend_kernel_mod);
return ascend_kernel_mod->block_dim();
}
void ProfilingReporter::ReportTask(const CNodePtr &node, const uint32_t stream_id, uint32_t task_id,
KernelType kernel_type) {
MsprofGeProfTaskData task_info{};
task_info.taskType = static_cast<uint32_t>(KernelType2TaskTypeEnum[kernel_type]);
(void)SetAlternativeValue(task_info.opName, MSPROF_MIX_DATA_STRING_LEN, node->fullname_with_scope(), device_id_);
(void)SetAlternativeValue(task_info.opType, MSPROF_GE_OP_TYPE_LEN, AnfAlgo::GetCNodeName(node), device_id_);
// Note: Currently, the profiler supports only static shapes.
task_info.shapeType = static_cast<uint32_t>(MSPROF_GE_SHAPE_TYPE_STATIC);
task_info.blockDims = GetBlockDim(node);
// Note: Currently, all steps are hardcoded to 0.
task_info.curIterNum = 0;
task_info.modelId = graph_id_;
task_info.streamId = stream_id;
task_info.taskId = task_id;
task_info.timeStamp = 0;
task_info.threadId = 0;
(void)ReportData(device_id_, reinterpret_cast<unsigned char *>(&task_info), sizeof(task_info), "task_desc_info");
}
void ProfilingReporter::ReportNode(const CNodePtr &node, uint32_t stream_id, uint32_t task_id, uint32_t tensor_type) {
const std::string tag_name = "tensor_data_info";
size_t total_size = 0;
if (tensor_type == MSPROF_GE_TENSOR_TYPE_INPUT) {
total_size = AnfAlgo::GetInputTensorNum(node);
} else {
total_size = AnfAlgo::GetOutputTensorNum(node);
}
const size_t batch_size = total_size / MSPROF_GE_TENSOR_DATA_NUM;
for (size_t i = 0U; i < batch_size; i++) {
MsprofGeProfTensorData tensor_info{};
(void)BuildProfTensorDataCommon(tensor_info, stream_id, task_id);
tensor_info.tensorNum = MSPROF_GE_TENSOR_DATA_NUM;
for (size_t j = 0U; j < MSPROF_GE_TENSOR_DATA_NUM; j++) {
size_t cur_index = i * MSPROF_GE_TENSOR_DATA_NUM + j;
MsprofGeTensorData tensor_data{};
(void)BuildTensorData(tensor_data, node, cur_index, tensor_type);
tensor_info.tensorData[j] = tensor_data;
}
(void)ReportData(device_id_, reinterpret_cast<unsigned char *>(&tensor_info), sizeof(tensor_info), tag_name);
}
size_t remain_size = total_size % MSPROF_GE_TENSOR_DATA_NUM;
if (remain_size == 0) {
return;
}
MsprofGeProfTensorData tensor_info{};
(void)BuildProfTensorDataCommon(tensor_info, stream_id, task_id);
tensor_info.tensorNum = remain_size;
for (size_t i = 0U; i < remain_size; ++i) {
MsprofGeTensorData tensor_data{};
size_t cur_index = batch_size * MSPROF_GE_TENSOR_DATA_NUM + i;
(void)BuildTensorData(tensor_data, node, cur_index, tensor_type);
tensor_info.tensorData[i] = tensor_data;
}
(void)ReportData(device_id_, reinterpret_cast<unsigned char *>(&tensor_info), sizeof(tensor_info), tag_name);
}
void ProfilingReporter::BuildProfTensorDataCommon(MsprofGeProfTensorData &tensor_info, uint32_t stream_id,
uint32_t task_id) {
tensor_info.modelId = graph_id_;
tensor_info.streamId = stream_id;
tensor_info.taskId = task_id;
// Note: Currently, all steps are hardcoded to 0.
tensor_info.curIterNum = 0;
}
void ProfilingReporter::BuildTensorData(MsprofGeTensorData &tensor_data, const CNodePtr &node, size_t index,
uint32_t tensor_type) {
tensor_data.tensorType = tensor_type;
std::vector<size_t> shape;
string data_format;
if (tensor_type == MSPROF_GE_TENSOR_TYPE_INPUT) {
auto input_node_with_index = AnfAlgo::GetPrevNodeOutput(node, index);
auto input_node = input_node_with_index.first;
auto input_index = input_node_with_index.second;
shape = AnfAlgo::GetOutputDeviceShape(input_node, input_index);
data_format = AnfAlgo::GetOutputFormat(input_node, input_index);
tensor_data.dataType = static_cast<uint32_t>(AnfAlgo::GetOutputDeviceDataType(input_node, input_index));
} else {
shape = AnfAlgo::GetOutputDeviceShape(node, index);
data_format = AnfAlgo::GetOutputFormat(node, index);
tensor_data.dataType = static_cast<uint32_t>(AnfAlgo::GetOutputDeviceDataType(node, index));
}
tensor_data.format = OpFormat2Index[data_format];
auto shape_size = std::min(static_cast<uint64_t>(MSPROF_GE_TENSOR_DATA_SHAPE_LEN), shape.size());
std::copy(shape.begin(), shape.begin() + shape_size, tensor_data.shape);
}
} // namespace ascend
} // namespace device
} // namespace mindspore

105
mindspore/ccsrc/runtime/device/ascend/profiling/profiling_reporter.h Normal file
View File

@ -0,0 +1,105 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_PROFILING_REPORTER_H
#define MINDSPORE_PROFILING_REPORTER_H
#include <string>
#include <utility>
#include <vector>
#include <memory>
#include "securec/include/securec.h"
#include "utils/log_adapter.h"
#include "backend/session/anf_runtime_algorithm.h"
#include "runtime/device/ascend/profiling/prof_common.h"
#include "runtime/device/ascend/profiling/profiling_manager.h"
#include "toolchain/prof_reporter.h"
namespace mindspore {
namespace device {
namespace ascend {
using std::pair;
using std::string;
using std::vector;
class StepPointDesc {
public:
StepPointDesc(string op_name, uint32_t tag) : op_name_(std::move(op_name)), tag_(tag) {}
~StepPointDesc() = default;
string op_name() { return op_name_; }
uint32_t tag() const { return tag_; }
private:
string op_name_;
uint32_t tag_;
};
class ProfilingReporter {
public:
ProfilingReporter(int device_id, uint32_t graph_id, vector<CNodePtr> cnode_list, const vector<uint32_t> &stream_ids,
const vector<uint32_t> &task_ids)
: device_id_(device_id),
graph_id_(graph_id),
cnode_list_(std::move(cnode_list)),
stream_ids_(stream_ids),
task_ids_(task_ids) {}
~ProfilingReporter() = default;
void ReportTasks();
void ReportStepPoint(const vector<std::shared_ptr<StepPointDesc>> &points);
private:
uint32_t device_id_;
uint32_t graph_id_;
vector<CNodePtr> cnode_list_;
vector<uint32_t> stream_ids_;
vector<uint32_t> task_ids_;
map<string, int> node_name_index_map_;
bool CheckStreamTaskValid();
static uint32_t GetBlockDim(const CNodePtr &node);
void ConstructNodeNameIndexMap();
uint32_t GetStreamId(const string &node_name);
uint32_t GetTaskId(const string &node_name);
void ReportData(int32_t device_id, unsigned char *data, size_t data_size, const std::string &tag_name);
void ReportTask(const CNodePtr &node, uint32_t stream_id, uint32_t task_id, KernelType kernel_type);
void ReportNode(const CNodePtr &node, uint32_t stream_id, uint32_t task_id, uint32_t tensor_type);
void BuildProfTensorDataCommon(MsprofGeProfTensorData &tensor_info, uint32_t stream_id, uint32_t task_id);
void BuildTensorData(MsprofGeTensorData &tensor_data, const CNodePtr &node, size_t index, uint32_t tensor_type);
template <typename T>
void SetAlternativeValue(T &property, const size_t property_size, const string &value, const int32_t &device_id) {
if (value.size() < property_size) {
property.type = static_cast<uint8_t>(MSPROF_MIX_DATA_HASH_ID);
const auto ret = strncpy_s(property.data.dataStr, property_size, value.c_str(), value.size());
if (ret != 0) {
MS_LOG(ERROR) << "[Profiling] strncpy_s value " << value.c_str() << " error!";
return;
}
} else {
property.type = static_cast<uint8_t>(MSPROF_MIX_DATA_STRING);
uint64_t hash_id;
(void)ProfilingManager::GetInstance().QueryHashId(device_id, value, hash_id);
property.data.hashId = hash_id;
}
}
};
} // namespace ascend
} // namespace device
} // namespace mindspore
#endif // MINDSPORE_PROFILING_REPORTER_H

36
mindspore/ccsrc/runtime/device/ascend/profiling/profiling_utils.cc
View File

@ -15,16 +15,13 @@
*/
#include <algorithm>
#include "runtime/device/ascend/profiling/reporter/graph_desc_reporter.h"
#include "runtime/device/ascend/profiling/profiling_utils.h"
#include "backend/kernel_compiler/kernel.h"
#include "runtime/device/ascend/profiling/profiling_manager.h"
#include "backend/session/anf_runtime_algorithm.h"
#include "utils/ms_utils.h"
#include "utils/utils.h"
#include "runtime/device/ascend/profiling/reporter/task_desc_reporter.h"
#include "utils/ms_context.h"
#include "runtime/device/ascend/profiling/reporter/point_reporter.h"
#include "nlohmann/json.hpp"
#include "base/core_ops.h"
#include "profiler/device/profiling.h"
@ -271,12 +268,14 @@ NotNull<CNodePtr> ProfilingUtils::CreateProfilingCNode(const ProfilingContent &p
}
void ProfilingUtils::SaveProfilingPoint(uint32_t graph_id, const std::string &node_name, uint32_t point_id) {
std::shared_ptr<ProfDesc> prof_desc_ptr = std::make_shared<PointDesc>(node_name, point_id);
MS_LOG(INFO) << "Save profiling point, graph id" << graph_id << ", node name: " << node_name
<< ", point_id: " << point_id;
std::shared_ptr<StepPointDesc> point_desc_ptr = std::make_shared<StepPointDesc>(node_name, point_id);
auto iter = graph_point_.find(graph_id);
if (iter == graph_point_.end()) {
graph_point_.emplace(graph_id, std::vector<std::shared_ptr<ProfDesc>>{prof_desc_ptr});
graph_point_.emplace(graph_id, std::vector<std::shared_ptr<StepPointDesc>>{point_desc_ptr});
} else {
iter->second.emplace_back(prof_desc_ptr);
iter->second.emplace_back(point_desc_ptr);
}
}
@ -383,7 +382,8 @@ void ProfilingUtils::SetGraphProfilingCNode(uint32_t graph_id, const std::vector
bool ProfilingUtils::ValidComputeGraph(const session::KernelGraph &kernel_graph) {
for (const auto &node : kernel_graph.execution_order()) {
if (AnfAlgo::GetKernelType(node) == TBE_KERNEL || AnfAlgo::GetKernelType(node) == AKG_KERNEL) {
auto kernel_type = AnfAlgo::GetKernelType(node);
if (kernel_type == TBE_KERNEL || kernel_type == AKG_KERNEL || kernel_type == AICPU_KERNEL) {
return true;
}
}
@ -400,32 +400,26 @@ void ProfilingUtils::ReportProfilingData(const std::vector<uint32_t> &task_ids,
uint32_t graph_id) {
auto ret = graph_profiling_cnode_.find(graph_id);
if (ret == graph_profiling_cnode_.end()) {
MS_LOG(ERROR) << "Graph id not found";
MS_LOG(ERROR) << "Graph id not found in graph_profiling_cnode_, graph id is " << graph_id
<< ", will not report this graph profiling data.";
return;
}
auto context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context);
TaskDescReporter task_reporter(context->get_param<uint32_t>(MS_CTX_DEVICE_ID), "vm_task_desc_info", ret->second);
task_reporter.set_task_ids(task_ids);
task_reporter.set_stream_ids(stream_ids);
task_reporter.ReportData();
GraphDescReporter graph_reporter(context->get_param<uint32_t>(MS_CTX_DEVICE_ID), "vm_graph_desc_info", ret->second);
graph_profiling_cnode_.erase(ret);
graph_reporter.ReportData();
auto device_id = context->get_param<uint32_t>(MS_CTX_DEVICE_ID);
ProfilingReporter reporter(device_id, graph_id, ret->second, stream_ids, task_ids);
reporter.ReportTasks();
// Report profiling point
auto point_iter = graph_point_.find(graph_id);
if (point_iter == graph_point_.end()) {
MS_LOG(ERROR) << "Graph id not found in graph_point";
MS_LOG(ERROR) << "Graph id not found in graph_point, will not report this graph step point data, graph id is: "
<< graph_id;
return;
}
PointReporter point_reporter(context->get_param<uint32_t>(MS_CTX_DEVICE_ID), "vm_point");
for (const auto &point : point_iter->second) {
point_reporter.AddReportData(point);
}
point_reporter.ReportData();
reporter.ReportStepPoint(point_iter->second);
}
void ProfilingUtils::SetReportProfilingData(const std::vector<uint32_t> &task_ids,

4
mindspore/ccsrc/runtime/device/ascend/profiling/profiling_utils.h
View File

@ -24,7 +24,7 @@
#include <unordered_map>
#include "backend/session/kernel_graph.h"
#include "utils/contract.h"
#include "runtime/device/ascend/profiling/reporter/profiling_desc.h"
#include "runtime/device/ascend/profiling/profiling_reporter.h"
namespace mindspore {
namespace device {
@ -117,7 +117,7 @@ class ProfilingUtils {
// graph id --> (kernel name list)
inline static std::map<uint32_t, std::vector<CNodePtr>> graph_profiling_cnode_;
inline static std::map<uint32_t, std::vector<std::string>> graph_kernel_name_;
inline static std::map<uint32_t, std::vector<std::shared_ptr<ProfDesc>>> graph_point_;
inline static std::map<uint32_t, std::vector<std::shared_ptr<StepPointDesc>>> graph_point_;
inline static uint32_t custom_node_index_;
inline static std::vector<GraphProfilingData> report_data_;
};

66
mindspore/ccsrc/runtime/device/ascend/profiling/reporter/desc_reporter.cc
View File

@ -1,66 +0,0 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <algorithm>
#include "runtime/device/ascend/profiling/reporter/desc_reporter.h"
#include "runtime/device/ascend/profiling/profiling_manager.h"
#include "utils/log_adapter.h"
namespace {
constexpr size_t kReportMaxLen = 1024;
}
namespace mindspore {
namespace device {
namespace ascend {
DescReporter::~DescReporter() = default;
void DescReporter::ReportByLine(const std::string &data, const std::string &file_name) const {
auto tot_size = data.size();
size_t cur_size = 0;
while (cur_size < tot_size) {
size_t remain_size = tot_size - cur_size;
size_t report_size = std::min(remain_size, kReportMaxLen);
ReporterData report_data{};
report_data.deviceId = device_id_;
report_data.dataLen = report_size;
report_data.data = (unsigned char *)data.c_str() + cur_size;
auto ret = memcpy_s(report_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, file_name.c_str(), file_name.length());
if (ret != 0) {
MS_LOG(EXCEPTION) << "Memcpy_s report data tag failed";
}
auto report_ret = ProfilingManager::GetInstance().CallMsprofReport(NOT_NULL(&report_data));
if (report_ret != 0) {
MS_LOG(EXCEPTION) << "Report data failed";
}
if (report_size == 0) {
MS_LOG(WARNING) << "Report_size is 0";
break;
}
cur_size += report_size;
}
}
void DescReporter::ReportAllLine() {
for (const auto &desc : prof_desc_list_) {
MS_EXCEPTION_IF_NULL(desc);
auto data = desc->ToString();
ReportByLine(data, file_name_);
}
}
} // namespace ascend
} // namespace device
} // namespace mindspore

50
mindspore/ccsrc/runtime/device/ascend/profiling/reporter/desc_reporter.h
View File

@ -1,50 +0,0 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_DESC_REPORTER_H_
#define MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_DESC_REPORTER_H_
#include <string>
#include <utility>
#include <vector>
#include <memory>
#include "toolchain/prof_reporter.h"
#include "runtime/device/ascend/profiling/reporter/profiling_desc.h"
#include "utils/contract.h"
#include "backend/session/kernel_graph.h"
namespace mindspore {
namespace device {
namespace ascend {
class DescReporter {
public:
virtual ~DescReporter() = 0;
DescReporter(int device_id, std::string file_name) : device_id_(device_id), file_name_(std::move(file_name)) {}
virtual void ReportData() = 0;
protected:
void ReportByLine(const std::string &data, const std::string &file_name) const;
void ReportAllLine();
int device_id_;
std::string file_name_;
std::vector<std::shared_ptr<ProfDesc>> prof_desc_list_;
};
} // namespace ascend
} // namespace device
} // namespace mindspore
#endif // MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_DESC_REPORTER_H_

64
mindspore/ccsrc/runtime/device/ascend/profiling/reporter/graph_desc_reporter.cc
View File

@ -1,64 +0,0 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "runtime/device/ascend/profiling/reporter/graph_desc_reporter.h"
#include "backend/session/anf_runtime_algorithm.h"
namespace mindspore {
namespace device {
namespace ascend {
void GraphDescReporter::ReportData() {
for (const auto &node : cnode_list_) {
MS_EXCEPTION_IF_NULL(node);
if (AnfAlgo::GetKernelType(node) != TBE_KERNEL && AnfAlgo::GetKernelType(node) != AKG_KERNEL) {
MS_LOG(INFO) << "Skip non tbe kernel:" << node->fullname_with_scope();
continue;
}
std::vector<DataElement> input_data_list;
std::vector<DataElement> output_data_list;
auto op_name = node->fullname_with_scope();
auto op_type = AnfAlgo::GetCNodeName(node);
auto input_size = AnfAlgo::GetInputTensorNum(node);
for (size_t i = 0; i < input_size; ++i) {
auto input_node_with_index = AnfAlgo::GetPrevNodeOutput(node, i);
auto input_node = input_node_with_index.first;
auto input_index = input_node_with_index.second;
DataElement element{};
element.index_ = i;
element.data_type_ = AnfAlgo::GetOutputDeviceDataType(input_node, input_index);
element.data_format_ = AnfAlgo::GetOutputFormat(input_node, input_index);
element.data_shape_ = AnfAlgo::GetOutputDeviceShape(input_node, input_index);
input_data_list.emplace_back(element);
}
auto output_size = AnfAlgo::GetOutputTensorNum(node);
for (size_t i = 0; i < output_size; ++i) {
DataElement element{};
element.index_ = i;
element.data_type_ = AnfAlgo::GetOutputDeviceDataType(node, i);
element.data_format_ = AnfAlgo::GetOutputFormat(node, i);
element.data_shape_ = AnfAlgo::GetOutputDeviceShape(node, i);
output_data_list.emplace_back(element);
}
auto graph_desc = std::make_shared<GraphDesc>(op_name, op_type, input_data_list, output_data_list);
prof_desc_list_.emplace_back(graph_desc);
}
ReportAllLine();
}
} // namespace ascend
} // namespace device
} // namespace mindspore

41
mindspore/ccsrc/runtime/device/ascend/profiling/reporter/graph_desc_reporter.h
View File

@ -1,41 +0,0 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_GRAPH_DESC_REPORTER_H_
#define MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_GRAPH_DESC_REPORTER_H_
#include <utility>
#include <string>
#include <vector>
#include "runtime/device/ascend/profiling/reporter/desc_reporter.h"
namespace mindspore {
namespace device {
namespace ascend {
class GraphDescReporter : public DescReporter {
public:
GraphDescReporter(uint32_t device_id, const std::string &file_name, std::vector<CNodePtr> cnode_list)
: DescReporter(device_id, file_name), cnode_list_(std::move(cnode_list)) {}
~GraphDescReporter() override = default;
void ReportData() override;
private:
std::vector<CNodePtr> cnode_list_;
};
} // namespace ascend
} // namespace device
} // namespace mindspore
#endif // MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_GRAPH_DESC_REPORTER_H_

50
mindspore/ccsrc/runtime/device/ascend/profiling/reporter/op_name_task_stream_reporter.cc
View File

@ -1,50 +0,0 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "runtime/device/ascend/profiling/reporter/op_name_task_stream_reporter.h"
namespace mindspore {
namespace device {
namespace ascend {
void OpNameTaskStreamReporter::ReportData() {
MS_LOG(INFO) << "ReportData start";
std::map<std::string, std::vector<std::pair<uint32_t, uint32_t>>> op_name_map;
for (auto &iter : stream_id_task_id_op_name_map_) {
auto pair = iter.first;
auto op_name = iter.second;
auto ret = op_name_map.find(op_name);
if (ret == op_name_map.end()) {
auto vect = std::vector<std::pair<uint32_t, uint32_t>>(1, pair);
auto emplace_ret = op_name_map.emplace(op_name, vect);
if (!emplace_ret.second) {
MS_LOG(WARNING) << "Duplicate op_name:" << op_name << " task_id:" << pair.first << " stream_id:" << pair.second;
}
} else {
ret->second.emplace_back(pair);
}
}
for (const auto &iter : op_name_map) {
auto desc_ptr = std::make_shared<TaskStreamOpNameDesc>(iter.first, iter.second);
prof_desc_list_.emplace_back(desc_ptr);
}
ReportAllLine();
MS_LOG(INFO) << "ReportData end";
}
} // namespace ascend
} // namespace device
} // namespace mindspore

42
mindspore/ccsrc/runtime/device/ascend/profiling/reporter/op_name_task_stream_reporter.h
View File

@ -1,42 +0,0 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_OP_NAME_TASK_STREAM_REPORTER_H_
#define MINDSPORE_MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_OP_NAME_TASK_STREAM_REPORTER_H_
#include <utility>
#include <string>
#include <map>
#include "runtime/device/ascend/profiling/reporter/desc_reporter.h"
namespace mindspore {
namespace device {
namespace ascend {
class OpNameTaskStreamReporter : public DescReporter {
public:
OpNameTaskStreamReporter(uint32_t device_id, const std::string &file_name,
std::map<std::pair<uint32_t, uint32_t>, std::string> stream_id_task_id_op_name_map)
: DescReporter(device_id, file_name), stream_id_task_id_op_name_map_(std::move(stream_id_task_id_op_name_map)) {}
~OpNameTaskStreamReporter() override = default;
void ReportData() override;
private:
std::map<std::pair<uint32_t, uint32_t>, std::string> stream_id_task_id_op_name_map_;
};
} // namespace ascend
} // namespace device
} // namespace mindspore
#endif // MINDSPORE_MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_OP_NAME_TASK_STREAM_REPORTER_H_

29
mindspore/ccsrc/runtime/device/ascend/profiling/reporter/point_reporter.cc
View File

@ -1,29 +0,0 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "runtime/device/ascend/profiling/reporter/point_reporter.h"
namespace mindspore {
namespace device {
namespace ascend {
void PointReporter::ReportData() { ReportAllLine(); }
void PointReporter::AddReportData(const std::shared_ptr<ProfDesc> &prof_desc) {
prof_desc_list_.emplace_back(prof_desc);
}
} // namespace ascend
} // namespace device
} // namespace mindspore

37
mindspore/ccsrc/runtime/device/ascend/profiling/reporter/point_reporter.h
View File

@ -1,37 +0,0 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_POINT_REPORTER_H_
#define MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_POINT_REPORTER_H_
#include <memory>
#include <string>
#include "runtime/device/ascend/profiling/reporter/desc_reporter.h"
namespace mindspore {
namespace device {
namespace ascend {
class PointReporter : public DescReporter {
public:
PointReporter(uint32_t device_id, const std::string &file_name) : DescReporter(device_id, file_name) {}
~PointReporter() override = default;
void ReportData() override;
void AddReportData(const std::shared_ptr<ProfDesc> &prof_desc);
};
} // namespace ascend
} // namespace device
} // namespace mindspore
#endif // MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_POINT_REPORTER_H_

99
mindspore/ccsrc/runtime/device/ascend/profiling/reporter/profiling_desc.cc
View File

@ -1,99 +0,0 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "runtime/device/ascend/profiling/reporter/profiling_desc.h"
#include <iterator>
#include <sstream>
namespace mindspore {
namespace device {
namespace ascend {
std::string TaskDesc::ToString() {
std::string out = op_name_;
out.append(" ")
.append(std::to_string(block_dim_))
.append(" ")
.append(std::to_string(task_id_))
.append(" ")
.append(std::to_string(stream_id_))
.append("\n");
return out;
}
std::string GraphDesc::ToString() {
std::string desc;
desc.append("op_name:").append(op_name_).append(" op_type:").append(op_type_);
int input_id = 0;
for (const auto &element : input_data_list_) {
desc.append(" input_id:")
.append(std::to_string(input_id++))
.append(" input_format:")
.append(element.data_format_)
.append(" input_data_type:")
.append(std::to_string(element.data_type_))
.append(" input_shape:")
.append(DataShapeToString(element.data_shape_));
}
input_id = 0;
for (const auto &element : output_data_list_) {
desc.append(" output_id:")
.append(std::to_string(input_id++))
.append(" output_format:")
.append(element.data_format_)
.append(" output_data_type:")
.append(std::to_string(element.data_type_))
.append(" output_shape:")
.append((DataShapeToString(element.data_shape_)));
}
desc.append("\n");
return desc;
}
std::string PointDesc::ToString() {
std::string desc;
desc.append(std::to_string(point_id_)).append(" ").append(op_name_).append("\n");
return desc;
}
std::string GraphDesc::DataShapeToString(const std::vector<size_t> &shape) {
std::ostringstream oss;
oss << "\"";
if (!shape.empty()) {
std::copy(shape.begin(), shape.end() - 1, std::ostream_iterator<size_t>(oss, ","));
oss << shape.back();
}
oss << "\"";
return oss.str();
}
std::string TaskStreamOpNameDesc::ToString() {
std::string desc = op_name_;
// op_name "task_id stream_id" "task_id stream_id"
for (auto pair : stream_id_task_id_pairs_) {
desc.append(" ");
desc.append(std::to_string(pair.first));
desc.append("_");
desc.append(std::to_string(pair.second));
}
desc.append("\n");
return desc;
}
} // namespace ascend
} // namespace device
} // namespace mindspore

100
mindspore/ccsrc/runtime/device/ascend/profiling/reporter/profiling_desc.h
View File

@ -1,100 +0,0 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_PROFILING_DESC_H_
#define MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_PROFILING_DESC_H_
#include <string>
#include <utility>
#include <vector>
namespace mindspore {
namespace device {
namespace ascend {
class ProfDesc {
public:
explicit ProfDesc(std::string op_name) : op_name_(std::move(op_name)) {}
virtual ~ProfDesc() = default;
virtual std::string ToString() = 0;
protected:
std::string op_name_;
};
class TaskDesc : public ProfDesc {
public:
TaskDesc(std::string op_name, uint32_t task_id, uint32_t block_dim, uint32_t stream_id)
: ProfDesc(std::move(op_name)), task_id_(task_id), block_dim_(block_dim), stream_id_(stream_id) {}
~TaskDesc() override = default;
std::string ToString() override;
private:
uint32_t task_id_;
uint32_t block_dim_;
uint32_t stream_id_;
};
struct DataElement {
size_t index_;
std::string data_format_;
int data_type_;
std::vector<size_t> data_shape_;
};
class GraphDesc : public ProfDesc {
public:
GraphDesc(std::string op_name, std::string op_type, std::vector<DataElement> input_data_list,
std::vector<DataElement> output_data_list)
: ProfDesc(std::move(op_name)),
op_type_(std::move(op_type)),
input_data_list_(std::move(input_data_list)),
output_data_list_(std::move(output_data_list)) {}
~GraphDesc() override = default;
std::string ToString() override;
private:
std::string op_type_;
std::vector<DataElement> input_data_list_;
std::vector<DataElement> output_data_list_;
[[nodiscard]] static std::string DataShapeToString(const std::vector<size_t> &shape);
};
class PointDesc : public ProfDesc {
public:
PointDesc(std::string op_name, uint32_t point_id) : ProfDesc(std::move(op_name)), point_id_(point_id) {}
~PointDesc() override = default;
std::string ToString() override;
private:
uint32_t point_id_;
};
class TaskStreamOpNameDesc : public ProfDesc {
public:
TaskStreamOpNameDesc(std::string op_name, std::vector<std::pair<uint32_t, uint32_t>> stream_id_task_id_pairs)
: ProfDesc(std::move(op_name)), stream_id_task_id_pairs_(std::move(stream_id_task_id_pairs)) {}
~TaskStreamOpNameDesc() override = default;
std::string ToString() override;
private:
std::vector<std::pair<uint32_t, uint32_t>> stream_id_task_id_pairs_;
};
} // namespace ascend
} // namespace device
} // namespace mindspore
#endif // MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_PROFILING_DESC_H_

60
mindspore/ccsrc/runtime/device/ascend/profiling/reporter/task_desc_reporter.cc
View File

@ -1,60 +0,0 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "runtime/device/ascend/profiling/reporter/task_desc_reporter.h"
#include "backend/session/anf_runtime_algorithm.h"
#include "backend/kernel_compiler/ascend_kernel_mod.h"
namespace mindspore {
namespace device {
namespace ascend {
void TaskDescReporter::ReportData() {
MS_LOG(INFO) << "cnode_list.size()=" << cnode_list_.size() << " task_ids_.size()=" << task_ids_.size();
if (cnode_list_.size() != task_ids_.size()) {
MS_LOG(ERROR) << "cnode list size not equal task ids size";
return;
}
size_t task_index = 0;
for (const auto &node : cnode_list_) {
MS_EXCEPTION_IF_NULL(node);
if (AnfAlgo::GetKernelType(node) != TBE_KERNEL && AnfAlgo::GetKernelType(node) != AKG_KERNEL) {
MS_LOG(INFO) << "Skip non tbe kernel:" << node->fullname_with_scope();
++task_index;
continue;
}
auto kernel_mod = AnfAlgo::GetKernelMod(node);
auto ascend_kernel_mod = dynamic_cast<kernel::AscendKernelMod *>(kernel_mod);
MS_EXCEPTION_IF_NULL(ascend_kernel_mod);
// Check task_id and stream_id valid
CheckStreamTaskValid(task_index, task_index);
auto desc_ptr = std::make_shared<TaskDesc>(node->fullname_with_scope(), task_ids_[task_index],
ascend_kernel_mod->block_dim(), stream_ids_[task_index]);
prof_desc_list_.emplace_back(desc_ptr);
++task_index;
}
ReportAllLine();
}
void TaskDescReporter::CheckStreamTaskValid(size_t task_id, size_t stream_id) const {
if (task_id >= task_ids_.size() || stream_id >= stream_ids_.size()) {
MS_LOG(EXCEPTION) << "Index invalid. task_id:" << task_id << ", task_ids.size:" << task_ids_.size()
<< ", stream_id:" << stream_id << ", stream_ids.size:" << stream_ids_.size();
}
}
} // namespace ascend
} // namespace device
} // namespace mindspore

46
mindspore/ccsrc/runtime/device/ascend/profiling/reporter/task_desc_reporter.h
View File

@ -1,46 +0,0 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_TASK_DESC_REPORTER_H_
#define MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_TASK_DESC_REPORTER_H_
#include <utility>
#include <string>
#include <vector>
#include "runtime/device/ascend/profiling/reporter/desc_reporter.h"
namespace mindspore {
namespace device {
namespace ascend {
class TaskDescReporter : public DescReporter {
public:
TaskDescReporter(int device_id, const std::string &file_name, std::vector<CNodePtr> cnode_list)
: DescReporter(device_id, file_name), cnode_list_(std::move(cnode_list)) {}
~TaskDescReporter() override = default;
void ReportData() override;
void set_task_ids(const std::vector<uint32_t> &task_ids) { task_ids_ = task_ids; }
void set_stream_ids(const std::vector<uint32_t> &stream_ids) { stream_ids_ = stream_ids; }
private:
std::vector<uint32_t> task_ids_;
std::vector<uint32_t> stream_ids_;
void CheckStreamTaskValid(size_t task_id, size_t stream_id) const;
std::vector<CNodePtr> cnode_list_;
};
} // namespace ascend
} // namespace device
} // namespace mindspore
#endif // MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_TASK_DESC_REPORTER_H_

66
mindspore/python/mindspore/profiler/common/struct_type.py Normal file
View File

@ -0,0 +1,66 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Define a cpp type map python struct type."""
from enum import Enum
class StructType(Enum):
"""Define cpp struct type, value is python struct type."""
CHAR = 'c'
UCHAR = 'B'
UINT16 = 'H'
UINT32 = 'I'
UINT64 = 'Q'
@classmethod
def format(cls, cpp_types):
"""
Given a Cpp type list, and return a python struct format string.
Args:
cpp_types (list): The cpp type list that should be a member of StructType.
Returns:
str, a python struct format string.
Example:
>>> cpp_typs = [StructType.UINT16, StructType.UINT32, StructType.UINT32]
>>> ret = StructType.format(cpp_typs)
>>> print(ret)
... 'HII'
"""
return ''.join([member.value for member in cpp_types])
@classmethod
def sizeof(cls, cpp_type):
"""Given a Cpp type list or a StructType value, and return a python struct format size."""
size_map = dict(
CHAR=1,
UCHAR=1,
UINT16=2,
UINT32=4,
UINT64=8
)
if isinstance(cpp_type, StructType):
return size_map[cpp_type.name]
size = 0
for member in cpp_type:
if isinstance(member, list):
size += cls.sizeof(member)
else:
size += size_map[member.name]
return size

105
mindspore/python/mindspore/profiler/parser/framework_enum.py Normal file
View File

@ -0,0 +1,105 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Define framework enum data."""
from enum import Enum, IntEnum
class FileDataType(Enum):
"""Define framework file data type."""
STEP_INFO = 'step_info'
HASH_DIC = 'hash_dic'
TENSOR_DATA_INFO = 'tensor_data_info'
TASK_DESC_INFO = 'task_desc_info'
@classmethod
def members(cls):
return {member.value for member in cls}
class VmDataType(IntEnum):
"""Definition of vm data type."""
NUMBER_TYPE_BEGIN = 30
BOOL = 31
INT = 32
INT8 = 33
INT16 = 34
INT32 = 35
INT64 = 36
UINT = 37
UINT8 = 38
UINT16 = 39
UINT32 = 40
UINT64 = 41
FLOAT = 42
FLOAT16 = 43
FLOAT32 = 44
FLOAT64 = 45
COMPLEX = 46
NUMBER_TYPE_END = 47
@classmethod
def get_data_type_name(cls, num):
"""
Get the name of data type by enum number.
Args:
num (int): Enum number.
Returns:
str, the name of data type.
"""
data_type = cls._value2member_map_.get(num)
return 'UNKNOWN' if data_type is None else data_type.name
class VmFormat(IntEnum):
"""Define mindspore data type."""
UNKNOWN = 0
DEFAULT = 1
NC1KHKWHWC0 = 2
ND = 3
NCHW = 4
NHWC = 5
HWCN = 6
NC1HWC0 = 7
FRAC_Z = 8
C1HWNCOC0 = 9
FRAC_NZ = 10
NC1HWC0_C04 = 11
FRACTAL_Z_C04 = 12
NDHWC = 13
FRACTAL_ZN_LSTM = 14
FRACTAL_ZN_RNN = 15
ND_RNN_BIAS = 16
NDC1HWC0 = 17
NCDHW = 18
FRACTAL_Z_3D = 19
DHWNC = 20
DHWCN = 21
@classmethod
def get_format_name(cls, num):
"""
Get the name of format by enum number.
Args:
num (int): Enum number.
Returns:
str, the name of data type.
"""
format_name = cls._value2member_map_.get(num)
return 'UNKNOWN' if format_name is None else format_name.name

790
mindspore/python/mindspore/profiler/parser/framework_parser.py
View File

@ -14,156 +14,36 @@
# ============================================================================
"""Thr parser for parsing framework files."""
import csv
import enum
import json
import os
import re
import stat
import struct
import json
from pathlib import Path
from typing import List
from collections import defaultdict
from collections import namedtuple
from mindspore.profiler.common.exceptions.exceptions import \
ProfilerPathErrorException, ProfilerDirNotFoundException, \
ProfilerFileNotFoundException, ProfilerDeviceIdMismatchException, \
ProfilerRawFileException, ProfilerParamValueErrorException
from mindspore.profiler.common.validator.validate_path import \
validate_and_normalize_path
from mindspore import log as logger
from mindspore.profiler.parser.framework_struct import TASK_DESC_STRUCT
from mindspore.profiler.parser.framework_struct import TENSOR_DATA_STRUCT
from mindspore.profiler.parser.framework_struct import STEP_INFO_STRUCT
from mindspore.profiler.parser.framework_enum import VmDataType, VmFormat, FileDataType
from mindspore.profiler.common.struct_type import StructType
from mindspore.profiler.common.exceptions.exceptions import ProfilerDirNotFoundException
from mindspore.profiler.common.exceptions.exceptions import ProfilerFileNotFoundException
from mindspore.profiler.common.exceptions.exceptions import ProfilerParamValueErrorException
class VmDataType(enum.IntEnum):
"""Definition of vm data type."""
NUMBER_TYPE_BEGIN = 30
NUMBER_TYPE_BOOL = 31
NUMBER_TYPE_INT = 32
NUMBER_TYPE_INT8 = 33
NUMBER_TYPE_INT16 = 34
NUMBER_TYPE_INT32 = 35
NUMBER_TYPE_INT64 = 36
NUMBER_TYPE_UINT = 37
NUMBER_TYPE_UINT8 = 38
NUMBER_TYPE_UINT16 = 39
NUMBER_TYPE_UINT32 = 40
NUMBER_TYPE_UINT64 = 41
NUMBER_TYPE_FLOAT = 42
NUMBER_TYPE_FLOAT16 = 43
NUMBER_TYPE_FLOAT32 = 44
NUMBER_TYPE_FLOAT64 = 45
NUMBER_TYPE_COMPLEX = 46
NUMBER_TYPE_END = 47
@classmethod
def get_data_type_name(cls, num):
"""
Get the name of data type by enum number.
Args:
num (int): Enum number.
Returns:
str, the name of data type.
"""
data_type = cls._value2member_map_.get(num)
return 'UNKNOWN' if data_type is None else data_type.name
FILE_DATA_STRUCT_DICT = {
FileDataType.STEP_INFO.value: STEP_INFO_STRUCT,
FileDataType.TENSOR_DATA_INFO.value: TENSOR_DATA_STRUCT,
FileDataType.TASK_DESC_INFO.value: TASK_DESC_STRUCT
}
class GeDataType(enum.IntEnum):
"""Definition of ge data type."""
DT_FLOAT = 0
DT_FLOAT16 = 1
DT_INT8 = 2
DT_INT16 = 6
DT_UINT16 = 7
DT_UINT8 = 4
DT_INT32 = 3
DT_INT64 = 9
DT_UINT32 = 8
DT_UINT64 = 10
DT_BOOL = 12
DT_DOUBLE = 11
DT_STRING = 13
DT_DUAL_SUB_INT8 = 14
DT_DUAL_SUB_UINT8 = 15
DT_COMPLEX64 = 16
DT_COMPLEX128 = 17
DT_QINT8 = 18
DT_QINT16 = 19
DT_QINT32 = 20
DT_QUINT8 = 21
DT_QUINT16 = 22
DT_RESOURCE = 23
DT_STRING_REF = 24
DT_DUAL = 25
DT_UNDEFINED = 26
@classmethod
def get_data_type_name(cls, num):
"""
Get the name of data type by enum number.
Args:
num (int): Enum number.
Returns:
str, the name of data type.
"""
data_type = cls._value2member_map_.get(num)
return 'UNKNOWN' if data_type is None else data_type.name
class GeFormat(enum.IntEnum):
"""Definition of ge format type."""
FORMAT_NCHW = 0
FORMAT_NHWC = 1
FORMAT_ND = 2
FORMAT_NC1HWC0 = 3
FORMAT_FRACTAL_Z = 4
FORMAT_NC1C0HWPAD = 5
FORMAT_NHWC1C0 = 6
FORMAT_FSR_NCHW = 7
FORMAT_FRACTAL_DECONV = 8
FORMAT_C1HWNC0 = 9
FORMAT_FRACTAL_DECONV_TRANSPOSE = 10
FORMAT_FRACTAL_DECONV_SP_STRIDE_TRANS = 11
FORMAT_NC1HWC0_C04 = 12
FORMAT_FRACTAL_Z_C04 = 13
FORMAT_CHWN = 14
FORMAT_FRACTAL_DECONV_SP_STRIDE8_TRANS = 15
FORMAT_HWCN = 16
FORMAT_NC1KHKWHWC0 = 17
FORMAT_BN_WEIGHT = 18
FORMAT_FILTER_HWCK = 19
FORMAT_HASHTABLE_LOOKUP_LOOKUPS = 20
FORMAT_HASHTABLE_LOOKUP_KEYS = 21
FORMAT_HASHTABLE_LOOKUP_VALUE = 22
FORMAT_HASHTABLE_LOOKUP_OUTPUT = 23
FORMAT_HASHTABLE_LOOKUP_HITS = 24
FORMAT_C1HWNCOC0 = 25
FORMAT_MD = 26
FORMAT_NDHWC = 27
FORMAT_FRACTAL_ZZ = 28
FORMAT_FRACTAL_NZ = 29
FORMAT_NCDHW = 30
FORMAT_DHWCN = 31
FORMAT_NDC1HWC0 = 32
FORMAT_FRACTAL_Z_3D = 33
FORMAT_CN = 34
FORMAT_NC = 35
FORMAT_DHWNC = 36
FORMAT_FRACTAL_Z_3D_TRANSPOSE = 37
FORMAT_RESERVED = 38
FORMAT_ALL = 39
@classmethod
def get_format_name(cls, num):
"""
Get the name of format type by enum number.
Args:
num (int): Enum number.
Returns:
str, the name of format type.
"""
format_type = cls._value2member_map_.get(num)
return 'UNKNOWN' if format_type is None else format_type.name
COL_NAMES = ['task_id', 'stream_id', 'block_dim', 'full_op_name', 'op_name', 'op_type', 'subgraph', 'op_info']
OpData = namedtuple('OpData', field_names=COL_NAMES)
class FrameworkParser:
@ -171,41 +51,30 @@ class FrameworkParser:
Thr parser for parsing framework files.
Args:
profiling_id (str): The profiling ID.
device_id (str): The device ID.
profiling_path (str): The profiling path which should be contain CANN profiling data.
rank_id (str): The rank ID.
output_path (str): The directory of the parsed file. Default: `./`.
"""
_regex_framework = r'Framework\.(?P<data_type>.+)\.(?P<device_id>\d).+'
_regex_framework_in_data = r'Framework\.(?P<data_type>.+)\.' \
r'(?P<device_id>\d)\.(?P<profiling_id>[a-zA-Z0-9]+).+'
_col_names = [
'task_id', 'stream_id', 'block_dim', 'full_op_name', 'op_name',
'op_type', 'subgraph', 'op_info'
]
_graph_attr_name = [
'input_format', 'input_data_type', 'input_shape', 'output_format',
'output_data_type', 'output_shape'
]
output_file_format = 'framework_raw_{rank_id}.csv'
# if the task id is less than the task id threshold, The combination of
# task id and Stream id represents one operator, else the task id represents
# one operator
_task_id_threshold = 25000
def __init__(self, profiling_path, rank_id, output_path='./'):
self._profiling_path = profiling_path
self._output_path = output_path
self._rank_id = rank_id
def __init__(self, profiling_id, device_id, rank_id, output_path='./'):
self._raw_data_dir = output_path
self._profiling_path = self._get_raw_profiling_path(profiling_id)
self._backend_type = None
self._framework_path = {'graph': [], 'task': [], 'point': []}
self._search_file(profiling_id, device_id)
self._device_id = device_id
self._save_path = self._get_save_path(rank_id, output_path)
self._hash_dict = {}
self._task_id_full_op_name_dict = {}
self._task_cache = {}
self._point_info = {}
self._parse_task_files()
self._parse_point_files()
@staticmethod
def _check_output_path(path):
if not os.path.exists(path) or not os.path.isdir(path):
raise ProfilerDirNotFoundException(path)
@property
def save_path(self):
@ -215,7 +84,7 @@ class FrameworkParser:
Returns:
str, the save path.
"""
return self._save_path
return os.path.realpath(os.path.join(self._output_path, self.output_file_format.format(rank_id=self._rank_id)))
@property
def point_info(self):
@ -223,24 +92,11 @@ class FrameworkParser:
The property of the framework point information.
Returns:
dict, the framework point information.
dict, the framework point information, key is tag, value is op name.
"""
# Note: In the multi-subgraph or multi-tag scenario, op name is overwritten.
return self._point_info
def to_task_id_full_op_name_dict(self):
"""
Get the task id and full operator name dict.
Returns:
dict, the task id and full operator name dict.
"""
return self._task_id_full_op_name_dict
def parse(self):
"""Parse the framework files."""
self._parse_graph_files_and_save(self._task_cache)
del self._task_cache
def check_op_name(self, op_name, is_prefix=True):
"""
Check whether the operator name exists.
@ -264,318 +120,281 @@ class FrameworkParser:
return True
return False
def _get_raw_profiling_path(self, profiling_id):
def to_task_id_full_op_name_dict(self):
"""
Get raw profiling path.
Args:
profiling_id (str): The profiling ID.
Get the task id and full operator name dict.
Returns:
str, the raw profiling path.
Raises:
ProfilerPathErrorException: If the profiling path is invalid.
ProfilerDirNotFoundException: If the profiling dir is not found.
dict, the task id and full operator name dict.
"""
profiling_path = os.path.join(self._raw_data_dir, profiling_id)
try:
profiling_path = validate_and_normalize_path(profiling_path)
except RuntimeError:
raise ProfilerPathErrorException('Profiling path is invalid.')
if not os.path.isdir(profiling_path):
raise ProfilerDirNotFoundException(profiling_path)
return profiling_path
return self._task_id_full_op_name_dict
def _search_file(self, profiling_id, device_id):
def parse(self):
"""Parse the framework files."""
framework_path_dict = self._search_file(self._profiling_path)
self._hash_dict = self._parse_hash_dic(framework_path_dict)
all_file_data = self._parse_binary_data(framework_path_dict)
task_id_full_op_name_dict = self._construct_task_id_full_op_name_dict(
all_file_data[FileDataType.TASK_DESC_INFO.value])
point_info = self._construct_point_info(task_id_full_op_name_dict, all_file_data[FileDataType.STEP_INFO.value])
task_id_op_attr_dict = self._construct_task_id_op_attr_dict(all_file_data[FileDataType.TENSOR_DATA_INFO.value])
self._point_info = point_info
self._task_id_full_op_name_dict = task_id_full_op_name_dict
all_op_data = self._construct_op_data_to_file(all_file_data[FileDataType.TASK_DESC_INFO.value],
task_id_op_attr_dict)
self._write_framework_to_file(all_op_data, output_file=self.save_path)
def _search_file(self, profiling_path):
"""
Search all framework files in raw profiling path.
Args:
profiling_id (str): The profiling ID.
device_id (str): The device ID.
profiling_path (str): This profiling path should contain data dir.
Return:
dict, return a dict container all framework file paths. Format is {FileDataType: [file paths]}.
Raises:
ProfilerFileNotFoundException: If the framework files are not found.
"""
# first search in the JOB dir, and if not, search in the sub directory
# in the JOB
self._search_file_from_job_path(device_id, search_in_sub_path=False)
if self._backend_type is None:
self._search_file_from_job_path(device_id, search_in_sub_path=True)
self._search_file_from_data_path(profiling_id, device_id)
data_dir = os.path.join(profiling_path, 'data')
if not os.path.isdir(data_dir):
raise ProfilerDirNotFoundException(data_dir)
if self._backend_type is None:
framework_path_dict = defaultdict(list)
for file in Path(data_dir).glob(r'Framework*[0-9]'):
file_name = file.name
match = re.search(self._regex_framework, file_name)
if match is None:
logger.warning("Profiler does not support to analyse file(%s), this file name format is not %s, "
"skip this file.", file.resolve(), self._regex_framework)
continue
if match['data_type'] not in FileDataType.members():
logger.warning("Profiler does not support to analyse file(%s), this file data type is %s, "
"skip this file.", file.resolve(), match['data_type'])
continue
framework_path_dict[match['data_type']].append(file.resolve())
empty_files = [data_type for data_type, files in framework_path_dict.items() if not files]
if not framework_path_dict or empty_files:
if empty_files:
logger.error("Can not find %s files when parse profiler framework file.", ','.join(empty_files))
raise ProfilerFileNotFoundException('Framework')
self._framework_path['graph'].sort()
self._framework_path['task'].sort()
def _search_file_from_job_path(self, device_id, search_in_sub_path=False):
"""
Search framework files from job path.
Args:
device_id (str): The device ID.
search_in_sub_path (bool): `True` if search file in profiling dir,
else search in profiling sub dir. Default: False.
Raises:
ProfilerRawFileException: If the framework file type is inconsistent.
ProfilerDeviceIdMismatchException: If the device id is mismatch
with framework in the raw dir.
"""
profiling_dir = os.path.join(self._profiling_path, 'data') \
if search_in_sub_path else self._profiling_path
if not os.path.isdir(profiling_dir):
return
files = os.listdir(profiling_dir)
for file in files:
pattern = re.search(self._regex_framework, file)
if not pattern or file.endswith('.done'):
for data_type in FileDataType.members():
if data_type not in framework_path_dict:
logger.warning("Can not find %s file when parse profiler framework file.", data_type)
continue
attrs = pattern.groupdict()
framework_path_dict[data_type].sort()
device_id_in_path = attrs.get('device_id')
if device_id_in_path != device_id:
raise ProfilerDeviceIdMismatchException()
return framework_path_dict
data_type = attrs.get('data_type')
data_type = data_type.replace("host.", "")
if data_type.startswith('vm_'):
if self._backend_type and self._backend_type != 'vm':
raise ProfilerRawFileException('Backend type is inconsistent.')
self._backend_type = 'vm'
_, data_type = data_type.split('_', 1)
else:
if self._backend_type and self._backend_type != 'ge':
raise ProfilerRawFileException('Backend type is inconsistent.')
self._backend_type = 'ge'
if data_type.startswith('graph_desc_info'):
self._framework_path['graph'].append(
os.path.join(profiling_dir, file)
)
elif data_type.startswith('task_desc_info'):
self._framework_path['task'].append(
os.path.join(profiling_dir, file)
)
elif data_type.startswith('point'):
self._framework_path['point'].append(
os.path.join(profiling_dir, file)
)
def _search_file_from_data_path(self, profiling_id, device_id):
"""
Search framework files from data path.
Args:
profiling_id (str): The profiling ID.
device_id (str): The device ID.
Raises:
ProfilerRawFileException: If the framework file type is inconsistent.
ProfilerDeviceIdMismatchException: If the device id is mismatch
with framework in the raw dir.
"""
profiling_data_path = os.path.join(
self._raw_data_dir, 'container', device_id, 'data'
)
if not os.path.isdir(profiling_data_path):
return
files = os.listdir(profiling_data_path)
for file in files:
pattern = re.search(self._regex_framework_in_data, file)
if not pattern or file.endswith('.done') or file.endswith('.zip'):
continue
attrs = pattern.groupdict()
profiling_id_in_path = attrs.get('profiling_id')
if profiling_id_in_path != profiling_id:
continue
device_id_in_path = attrs.get('device_id')
if device_id_in_path != device_id:
raise ProfilerDeviceIdMismatchException()
data_type = attrs.get('data_type')
data_type = data_type.replace("host.", "")
if data_type.startswith('vm_'):
if self._backend_type and self._backend_type != 'vm':
raise ProfilerRawFileException('Backend type is inconsistent.')
self._backend_type = 'vm'
_, data_type = data_type.split('_', 1)
else:
if self._backend_type and self._backend_type != 'ge':
raise ProfilerRawFileException('Backend type is inconsistent.')
self._backend_type = 'ge'
if data_type.startswith('graph_desc_info'):
self._framework_path['graph'].append(
os.path.join(profiling_data_path, file)
)
elif data_type.startswith('task_desc_info'):
self._framework_path['task'].append(
os.path.join(profiling_data_path, file)
)
elif data_type.startswith('point'):
self._framework_path['point'].append(
os.path.join(profiling_data_path, file)
)
def _get_save_path(self, rank_id, output_path):
"""
Get the save path.
Args:
rank_id (str): The rank ID.
output_path (str): The output dir.
Returns:
str, the save path.
Raises:
ProfilerPathErrorException: If the output path is invalid.
ProfilerDirNotFoundException: If the output dir is not found.
"""
try:
output_dir = validate_and_normalize_path(output_path)
except RuntimeError:
raise ProfilerPathErrorException('Output path is invalid.')
if not os.path.isdir(output_dir):
raise ProfilerDirNotFoundException(output_dir)
return os.path.join(
output_dir, '_'.join(['framework', 'raw', rank_id]) + '.csv'
)
def _parse_task_files(self):
"""Parse the framework task files."""
for path in self._framework_path['task']:
path = validate_and_normalize_path(path)
@staticmethod
def _parse_hash_dic(framework_path_dict):
"""Parse the hash dic files, and return a hash value map op name dict."""
hash_op_dict = {}
for path in framework_path_dict[FileDataType.HASH_DIC.value]:
with open(path, 'r') as file:
for task_info in file:
infos = task_info.strip('\n').split(' ')
infos = infos[1:] if len(infos) == 5 else infos
# key is op name, values is task id, stream id, block_dim
self._task_cache[infos[0]] = [infos[2], infos[3], infos[1]]
for hash_str in file:
hash_value, op_name = hash_str.strip().split(':')
hash_op_dict[hash_value] = op_name
return hash_op_dict
# if the task id is less than the task id threshold, the
# stream id and task id correspond to an operator
task_id = infos[2]
if int(task_id) < self._task_id_threshold:
task_id = '_'.join([infos[3], task_id])
self._task_id_full_op_name_dict[task_id] = infos[0]
def _parse_binary_data(self, framework_path_dict):
"""Parse binary data in the FILE_DATA_STRUCT_DICT from given files, such as task data, step point data"""
all_file_data = defaultdict(list)
for file_data_type, data_struct in FILE_DATA_STRUCT_DICT.items():
line_size = StructType.sizeof(data_struct.values())
for path in framework_path_dict[file_data_type]:
with open(path, 'rb') as file_handler:
while True:
line = file_handler.read(line_size)
if len(line) < line_size:
break
line_data = self._transform_binary_data(data_struct, line)
all_file_data[file_data_type].append(line_data)
return all_file_data
def _parse_graph_files_and_save(self, task_cache):
def _transform_binary_data(self, data_struct, binary_data):
"""
Parse the framework graph files and save the framework information.
Parse the binary data to get the actual data
Args:
task_cache (dict): The task information cache.
"""
with open(self._save_path, 'w') as save_file:
csv_writer = csv.writer(save_file)
csv_writer.writerow(self._col_names)
pre_graph_info = None
for path in self._framework_path['graph']:
first_row = True
with open(path, 'r') as graph_file:
for graph_info in graph_file:
if first_row is True:
first_row = False
# The last row of the previous file and the first row of the current file may need
# to be combined to one row
if graph_info.startswith("op_name:") is False:
pre_graph_info = pre_graph_info + graph_info
continue
if pre_graph_info is not None:
self._parse_graph_row_and_save(task_cache, csv_writer, pre_graph_info)
pre_graph_info = graph_info
if pre_graph_info is not None:
self._parse_graph_row_and_save(task_cache, csv_writer, pre_graph_info)
none_list = [None, None, None, None]
for key, value in task_cache.items():
value.append(key)
value.extend(none_list)
csv_writer.writerow(value)
os.chmod(self._save_path, stat.S_IREAD | stat.S_IWRITE)
def _parse_graph_row_and_save(self, task_cache, csv_writer, graph_info):
"""
Parse the framework graph row and save the framework information.
Args:
task_cache (dict): The task information cache.
csv_writer (csv): Csv writer.
graph_info (str): Row info of graph.
"""
result = self._parse_one_row_graph_info(graph_info)
task_info = task_cache.get(result[0])
if task_info:
task_info.extend(result)
csv_writer.writerow(task_info)
del task_cache[result[0]]
else:
save_info = [None, None, None]
save_info.extend(result)
csv_writer.writerow(save_info)
def _parse_one_row_graph_info(self, row_info):
"""
Parse the graph information in one row.
Args:
row_info (str): One row graph information.
Args：
data_struct (dict): Key is the data name, value is StructType.
binary_data (str): This value should be a binary string.
Returns:
list[str], the parsed graph information.
dict, key is data name, value is a actual value.
Example:
>>> ret = self._transform_binary_data({'op_name': StructType.UINT32}, b'1101')
>>> print(ret)
... {'op_name': (825241905,)}
"""
full_op_name = None
op_name = None
subgraph_name = None
op_type = None
op_info = dict()
cur_op_info_key = None
infos = row_info.strip('\n').split(' ')
for info in infos:
attr_name, attr_value = info.split(':', 1)
if attr_name == 'op_name':
full_op_name = attr_value
subgraph_name = self._get_subgraph_name(full_op_name)
op_name = self._get_op_name(full_op_name, subgraph_name)
elif attr_name == 'op_type':
op_type = attr_value
elif attr_name in ['input_id', 'output_id']:
cur_op_info_key = '{}_{}'.format(
attr_name.split('_')[0], attr_value
)
op_info[cur_op_info_key] = dict()
elif attr_name in self._graph_attr_name:
op_attr = attr_name.split('_', 1)[1]
if op_attr == 'shape':
attr_value = attr_value.strip('"')
if self._backend_type == 'vm':
if op_attr == 'data_type':
attr_value = VmDataType.get_data_type_name(
int(attr_value)
)
actual_data = {}
cursor = 0
for name, data_type in data_struct.items():
data_size = StructType.sizeof(data_type)
if isinstance(data_type, list):
if name in ('opName', 'opType'):
unpack_data = self._special_process_mixed_data(binary_data, cursor, data_size)
elif name == 'tensorData':
tensor_num = actual_data['tensorNum']
unpack_data = self._special_process_tensor_data(cursor, data_type, binary_data, tensor_num)
elif name == 'tensorNum':
unpack_data = self._speceal_process_tensor_num(data_type, binary_data, cursor)
else:
if op_attr == 'data_type':
attr_value = GeDataType.get_data_type_name(
int(attr_value)
)
elif op_attr == 'format':
attr_value = GeFormat.get_format_name(int(attr_value))
# skip reserve data
unpack_data = None
op_info[cur_op_info_key][op_attr] = attr_value
actual_data[name] = unpack_data
cursor += data_size
continue
# the list info are full_op_name, op_name, op_type, subgraph, op_info
return [full_op_name, op_name, op_type, subgraph_name,
json.dumps(op_info)]
unpack_data = struct.unpack(data_type.value, binary_data[cursor: cursor+data_size])[0]
cursor += data_size
actual_data[name] = unpack_data
return actual_data
def _get_subgraph_name(self, full_op_name):
def _special_process_mixed_data(self, binary_data, cursor, data_size):
"""Specially processes mixed data, for example, opName and opType"""
# The first byte is type flag, 0 means data is string, 1 means data is hash value
flag = struct.unpack(StructType.UCHAR.value, binary_data[cursor:cursor + 1])[0]
# skip rsv data, rsv has 7 bytes
skip_size = 8
remain_size = data_size - skip_size
if flag == 0:
unpack_data = struct.unpack(StructType.CHAR.value * remain_size,
binary_data[cursor + skip_size:cursor + data_size])
try:
unpack_data = ''.join(list(map(lambda c: c.decode(),
filter(lambda c: c != b'\x00', unpack_data))))
except Exception as exc:
raise exc
else:
size = StructType.sizeof(StructType.UINT64) + skip_size
hash_value = struct.unpack(StructType.UINT64.value,
binary_data[cursor + skip_size:cursor + size])[0]
unpack_data = self._hash_dict[str(hash_value)]
return unpack_data
@staticmethod
def _special_process_tensor_data(cursor, data_type, binary_data, tensor_num):
"""The tensor data depends tensor num, so need to special process."""
start = cursor
op_attr_struct = data_type[0]
op_attr_size = StructType.sizeof(op_attr_struct)
unpack_data = []
for _ in range(tensor_num):
buffer = binary_data[start:start + op_attr_size]
values = struct.unpack(StructType.format(op_attr_struct), buffer)
one_data = dict(
tensorType=values[0],
format=values[1],
dataType=values[2],
shape=list(filter(lambda x: x != 0, values[3:]))
)
unpack_data.append(one_data)
start += op_attr_size
return unpack_data
@staticmethod
def _speceal_process_tensor_num(data_type, binary_data, cursor):
"""The memory of tensorNum is aligned, so here need to special process"""
tensor_num_struct = data_type[0]
size = StructType.sizeof(tensor_num_struct)
unpack_data = struct.unpack(tensor_num_struct.value, binary_data[cursor:cursor + size])[0]
return unpack_data
def _construct_task_id_full_op_name_dict(self, task_desc_info):
"""The task desc info is a list[task_desc], task_desc is a dict, key is same as TASK_DESC_STRUCT."""
task_id_full_op_name = {}
for task_desc in task_desc_info:
task_id = self._combine_stream_task_id(task_desc['streamId'], task_desc['taskId'])
task_id_full_op_name[task_id] = task_desc['opName']
return task_id_full_op_name
def _construct_point_info(self, task_id_full_op_name_dict, step_point_data):
"""step_point_data is a list[step_data], step data is a dict, key is same as STEP_INFO_STRUCT."""
point_info = {}
for step_point in step_point_data:
task_id = self._combine_stream_task_id(step_point['streamId'], step_point['taskId'])
tag = step_point['tag']
full_op_name = task_id_full_op_name_dict[task_id]
point_info[tag] = full_op_name
return point_info
def _construct_task_id_op_attr_dict(self, prof_tensor_data):
"""prof_tensor_data is a list[tensor_data], tensor_data is a dict, key is same as TENSOR_DATA_STRUCT."""
task_id_op_attr_dict = defaultdict(list)
for tensor_data in prof_tensor_data:
task_id = self._combine_stream_task_id(tensor_data['streamId'], tensor_data['taskId'])
for tensor_attr in tensor_data['tensorData']:
tensor_type = 'input' if tensor_attr['tensorType'] == 0 else 'output'
tensor_format = VmFormat.get_format_name(tensor_attr['format'])
op_attr = dict(
tensor_type=tensor_type,
format=tensor_format,
data_type=VmDataType.get_data_type_name(tensor_attr['dataType']),
shape=tensor_attr['shape']
)
task_id_op_attr_dict[task_id].append(op_attr)
for task_id, op_attrs in task_id_op_attr_dict.items():
input_count = 0
output_count = 0
new_op_attr = {}
for op_attr in op_attrs:
if op_attr['tensor_type'] == 'input':
op_attr.pop('tensor_type')
new_op_attr[f'input_{input_count}'] = op_attr
input_count += 1
else:
op_attr.pop('tensor_type')
new_op_attr[f'output_{output_count}'] = op_attr
output_count += 1
task_id_op_attr_dict[task_id] = new_op_attr
return task_id_op_attr_dict
def _construct_op_data_to_file(self, task_desc_info, task_id_op_attr_dict):
"""Build data written to a file."""
all_op_data = []
for task_desc in task_desc_info:
task_id = task_desc['taskId']
full_op_name = task_desc['opName']
subgraph = self._get_subgraph_name(full_op_name)
combined_task_id = self._combine_stream_task_id(task_desc['streamId'], task_id)
op_data = OpData(task_id=task_id,
stream_id=task_desc['streamId'],
block_dim=task_desc['blockDims'],
full_op_name=full_op_name,
op_name=full_op_name.split('/')[-1],
op_type=task_desc['opType'],
subgraph=subgraph,
op_info=json.dumps(task_id_op_attr_dict[combined_task_id]))
all_op_data.append(op_data)
return all_op_data
@staticmethod
def _write_framework_to_file(all_op_data: List[OpData], output_file):
with open(output_file, 'w') as file_handler:
csv_writer = csv.writer(file_handler)
csv_writer.writerow(COL_NAMES)
csv_writer.writerows(all_op_data)
@staticmethod
def _get_subgraph_name(full_op_name):
"""
Get subgraph name.
@ -590,39 +409,16 @@ class FrameworkParser:
return subgraph_name
return None
def _get_op_name(self, full_op_name, subgraph_name):
@staticmethod
def _combine_stream_task_id(stream_id, task_id):
"""
Get operator name.
Args:
full_op_name (str): The full operator name.
subgraph_name (str): The subgraph name.
Returns:
str, the operator name.
When the Task ID is less than the threshold, it will be reuse in different streams,
only the stream ID and task ID combined are unique values.
"""
if subgraph_name is None:
return full_op_name
if self._backend_type == 'vm':
return full_op_name.split('/')[-1]
strs = full_op_name.split(subgraph_name + '/')
op_name = None
for name_str in strs:
if not name_str:
continue
if op_name is None:
op_name = name_str.split('/')[-1]
else:
op_name = '+'.join([op_name, name_str.split('/')[-1]])
return op_name
def _parse_point_files(self):
"""Parse the framework point files."""
for path in self._framework_path['point']:
path = validate_and_normalize_path(path)
with open(path, 'r') as file:
for point_info in file:
infos = point_info.strip('\n').split(' ')
self._point_info[int(infos[0])] = infos[1]
# if the task id is less than the task id threshold, The combination of
# task id and Stream id represents one operator, else the task id represents
# one operator
task_id_threshold = 25000
if task_id < task_id_threshold:
return f'{stream_id}_{task_id}'
return str(task_id)

61
mindspore/python/mindspore/profiler/parser/framework_struct.py Normal file
View File

@ -0,0 +1,61 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Thr parser for parsing framework files."""
from mindspore.profiler.common.struct_type import StructType
# Note: All keys should be named with lower camel case, which are the same as those in C++.
TASK_DESC_STRUCT = dict(
magicNumber=StructType.UINT16,
dataTag=StructType.UINT16,
taskType=StructType.UINT32,
opName=[StructType.UINT64] * 16, # opName is a mix data
opType=[StructType.UINT64] * 8, # opType is a mix data
curIterNum=StructType.UINT64,
timeStamp=StructType.UINT64,
shapeType=StructType.UINT32,
blockDims=StructType.UINT32,
modelId=StructType.UINT32,
streamId=StructType.UINT32,
taskId=StructType.UINT32,
threadId=StructType.UINT32,
reserve=[StructType.UCHAR] * 16
)
STEP_INFO_STRUCT = dict(
magicNumber=StructType.UINT16,
dataTag=StructType.UINT16,
modelId=StructType.UINT32,
streamId=StructType.UINT32,
taskId=StructType.UINT32,
timeStamp=StructType.UINT64,
curIterNum=StructType.UINT64,
threadId=StructType.UINT32,
tag=StructType.UCHAR,
reserve=[StructType.UCHAR] * 27
)
TENSOR_DATA_STRUCT = dict(
magicNumber=StructType.UINT16,
dataTag=StructType.UINT16,
modelId=StructType.UINT32,
curIterNum=StructType.UINT64,
streamId=StructType.UINT32,
taskId=StructType.UINT32,
tensorNum=[StructType.UCHAR] * 4, # Note: Here the memory is aligned. The actual memory usage is 1, 3 is padding.
tensorData=[[StructType.UINT32] * 11] * 5,
reserve=[StructType.UCHAR] * 8 # Note: Here the memory is aligned. The actual memory usage is 4, 4 is padding.
)

5
mindspore/python/mindspore/profiler/parser/integrator.py
View File

@ -1434,6 +1434,11 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
if start_time > step_end_time:
per_step_time_list.append(cur_step_metric_time)
step += 1
if step >= len(step_time_list):
logger.warning("Compute profiler compute_time_inside_step time, "
"find the data length is more than step count, "
"maybe current graph has multi sub graph, skip the last data.")
break
step_end_time = step_time_list[step][self._start_time_idx] + \
step_time_list[step][self._duration_idx]
cur_step_metric_time = 0

20
mindspore/python/mindspore/profiler/profiling.py
View File

@ -25,6 +25,8 @@ import mindspore._c_expression as c_expression
import mindspore._c_dataengine as cde
from mindspore.profiler.common.exceptions.exceptions import ProfilerFileNotFoundException, \
ProfilerIOException, ProfilerException, ProfilerRawFileException
from mindspore.profiler.common.exceptions.exceptions import ProfilerPathErrorException
from mindspore.profiler.common.exceptions.exceptions import ProfilerDirNotFoundException
from mindspore.profiler.common.util import get_file_path, fwrite_format
from mindspore.profiler.common.validator.validate_path import \
validate_and_normalize_path
@ -299,6 +301,8 @@ class Profiler:
job_id = self._get_profiling_job_id()
logger.info("Profiling: job id is %s ", job_id)
self._check_output_path(output_path=self._output_path)
source_path = os.path.join(self._output_path, job_id)
# parse hwts.log.data.45.dev file, and get task profiling data
hwts_output_filename = self._hwts_output_filename_target + self._rank_id + ".txt"
@ -310,7 +314,7 @@ class Profiler:
hwtslog_parser.execute()
# parse Framework file, and get the relation of op and tasks
framework_parser = FrameworkParser(job_id, self._dev_id, self._rank_id, self._output_path)
framework_parser = FrameworkParser(source_path, self._rank_id, self._output_path)
logger.info("Profiling: analyzing framework data.")
framework_parser.parse()
op_task_dict = framework_parser.to_task_id_full_op_name_dict()
@ -403,6 +407,16 @@ class Profiler:
logger.info("Profiling: analyzing the operation FLOPs.")
flops_parser.execute()
@staticmethod
def _check_output_path(output_path):
try:
output_path = validate_and_normalize_path(output_path)
except RuntimeError:
raise ProfilerPathErrorException('Output path is invalid.')
if not os.path.isdir(output_path):
raise ProfilerDirNotFoundException(output_path)
return output_path
def start(self):
"""
Used for Ascend, GPU, start profiling. Profiling can be turned on based on step and epoch.
@ -669,7 +683,6 @@ class Profiler:
Returns:
str, profiling job id.
"""
job_id = ""
job_dirs = filter(lambda item: item.startswith('JOB') or item.startswith('PROF') and \
os.path.isdir(os.path.join(self._output_path, item)),
@ -699,6 +712,9 @@ class Profiler:
"profiler will ignore this job dir.", job_dir, self._dev_id, training_device_id)
continue
if not os.listdir(os.path.join(job_dir, 'data')):
continue
job_start_time = self._parse_host_start_log(host_start_file_path)
if not job_start_time:
logger.warning("Find profiling job path %s, but fail to get job start info, "

4
tests/ut/data/profiler_data/JOB1/Framework.host.vm_graph_desc_info.0.slice_0
View File

@ -1,4 +0,0 @@
op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224"
op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16"
op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16"
op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16"

0
tests/ut/data/profiler_data/JOB1/Framework.host.vm_graph_desc_info.0.slice_0.done
View File

4
tests/ut/data/profiler_data/JOB1/Framework.host.vm_task_desc_info.0.slice_0
View File

@ -1,4 +0,0 @@
Default/Cast-op6 32 51517 0
Default/TransData-op7 32 51518 0
Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 51519 0
Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 51522 0

4
tests/ut/data/profiler_data/JOB4/data/Framework.host.vm_graph_desc_info.0.slice_0
View File

@ -1,4 +0,0 @@
op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224"
op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16"
op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16"
op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16"

2
tests/ut/data/profiler_data/JOB4/data/Framework.host.vm_point.0.slice_0
View File

@ -1,2 +0,0 @@
1 Default/Cast-op6
2 Default/TransData-op7

4
tests/ut/data/profiler_data/JOB4/data/Framework.host.vm_task_desc_info.0.slice_0
View File

@ -1,4 +0,0 @@
Default/Cast-op6 32 1 0
Default/TransData-op7 32 2 0
Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 3 0
Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 4 0

6
tests/ut/data/profiler_data/framework/data/Framework.hash_dic.6.slice_0 Normal file
View File

@ -0,0 +1,6 @@
221281922719854948:Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/OneHot-op183
11321138686081159177:Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/SoftmaxCrossEntropyWithLogits-op184
9987411238351573275:Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/ReduceMean-op186
18188948999212452788:Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/Tile-op188
4352827225462011144:Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/RealDiv-op189
12251689346898190624:Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/Mul-op191

BIN
tests/ut/data/profiler_data/framework/data/Framework.step_info.6.slice_0 Normal file
View File

Binary file not shown.

BIN
tests/ut/data/profiler_data/framework/data/Framework.task_desc_info.6.slice_0 Normal file
View File

Binary file not shown.

BIN
tests/ut/data/profiler_data/framework/data/Framework.tensor_data_info.6.slice_0 Normal file
View File

Binary file not shown.

1
tests/ut/data/profiler_data/framework/expected/task_id_full_op_name.json Normal file
View File

File diff suppressed because one or more lines are too long

1
tests/ut/python/profiler/__init__.py
View File

@ -18,4 +18,5 @@ import os
RAW_DATA_BASE = os.path.realpath(os.path.join(os.path.dirname(__file__), '../../data/profiler_data'))
RAW_DATA = os.path.realpath(os.path.join(RAW_DATA_BASE, 'JOB1'))
RAW_DATA_JOB2 = os.path.realpath(os.path.join(RAW_DATA_BASE, 'JOB2'))
FRAMEWORK_RAW_DATA = os.path.realpath(os.path.join(RAW_DATA_BASE, 'framework'))
PROFILER_DIR = os.path.realpath(os.path.join(RAW_DATA_BASE, 'profiler'))

106
tests/ut/python/profiler/parser/test_framework_parser.py
View File

@ -15,16 +15,10 @@
"""Test the framework parser module."""
import csv
import os
import shutil
import json
import tempfile
from unittest import mock
import pytest
from mindspore.profiler.common.exceptions.exceptions import \
ProfilerFileNotFoundException
from mindspore.profiler.parser.framework_parser import FrameworkParser
from tests.ut.python.profiler import PROFILER_DIR, RAW_DATA_BASE
from tests.ut.python.profiler import FRAMEWORK_RAW_DATA
def get_framework_result(file_path):
@ -47,84 +41,36 @@ def get_framework_result(file_path):
class TestFrameworkParser:
"""Test the class of `FrameworkParser`."""
def setup_method(self):
def setup_class(self):
"""Initialization before test case execution."""
self._output_path_1 = tempfile.NamedTemporaryFile(prefix='test_framework_parser_').name
shutil.copytree(RAW_DATA_BASE, self._output_path_1)
self._parser_1 = FrameworkParser('JOB1', '0', '0', self._output_path_1)
self._output_path_2 = tempfile.NamedTemporaryFile(prefix='test_framework_parser_').name
shutil.copytree(RAW_DATA_BASE, self._output_path_2)
self._parser_2 = FrameworkParser('JOB2', '0', '0', self._output_path_2)
self._output_path_4 = tempfile.NamedTemporaryFile(prefix='test_framework_parser_').name
shutil.copytree(RAW_DATA_BASE, self._output_path_4)
self._parser_4 = FrameworkParser('JOB4', '0', '0', self._output_path_4)
self._profiling_path = os.path.realpath(FRAMEWORK_RAW_DATA)
def teardown_method(self) -> None:
"""Clear up after test case execution."""
shutil.rmtree(self._output_path_1)
shutil.rmtree(self._output_path_2)
shutil.rmtree(self._output_path_4)
def test_save_path(self):
def test_format_save_path(self):
"""Test the querying save path function."""
expect_result = os.path.join(self._output_path_1, 'framework_raw_0.csv')
assert expect_result == self._parser_1.save_path
expect_result = os.path.join(self._output_path_2, 'framework_raw_0.csv')
assert expect_result == self._parser_2.save_path
def test_point_info(self):
"""Test the querying point info function."""
expect_result = {
1: 'Default/Cast-op6',
2: 'Default/TransData-op7'
}
assert expect_result == self._parser_4.point_info
def test_to_task_id_full_op_name_dict(self):
"""Test the querying task id and full operator name dict function."""
expect_result = {
'51517': 'Default/Cast-op6',
'51518': 'Default/TransData-op7',
'51519': 'Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5',
'51522': 'Default/network-WithLossCell/_backbone-ResNet/'
'layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28'
}
assert expect_result == self._parser_1.to_task_id_full_op_name_dict()
assert expect_result == self._parser_2.to_task_id_full_op_name_dict()
expect_result = {
'0_1': 'Default/Cast-op6',
'0_2': 'Default/TransData-op7',
'0_3': 'Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5',
'0_4': 'Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/'
'0-ResidualBlock/conv1-Conv2d/Cast-op28'
}
assert expect_result == self._parser_4.to_task_id_full_op_name_dict()
output_path = os.path.realpath('./')
rank_id = '2'
parser = FrameworkParser(self._profiling_path, rank_id, output_path)
expect_result = os.path.join(output_path, FrameworkParser.output_file_format.format(rank_id=rank_id))
assert parser.save_path == expect_result
def test_parse(self):
"""Test the parse function."""
expect_framework_file = os.path.join(PROFILER_DIR, 'framework_raw_0.csv')
expect_framework_file = os.path.realpath(expect_framework_file)
expect_result = get_framework_result(expect_framework_file)
with tempfile.TemporaryDirectory() as output_path:
parser = FrameworkParser(self._profiling_path, '1', output_path)
parser.parse()
assert os.path.exists(parser.save_path)
self._parser_1.parse()
framework_file = os.path.join(self._output_path_1, 'framework_raw_0.csv')
result = get_framework_result(framework_file)
assert expect_result == result
expected_point = {2: "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Add-op123",
3: "Default/Assign-op245",
4: "Default/Assign-op245"}
assert parser.point_info == expected_point
self._parser_2.parse()
framework_file = os.path.join(self._output_path_2, 'framework_raw_0.csv')
result = get_framework_result(framework_file)
assert expect_result == result
task_id_full_op_name_file = os.path.join(self._profiling_path, 'expected', 'task_id_full_op_name.json')
self._compare_json(task_id_full_op_name_file, parser.to_task_id_full_op_name_dict())
@mock.patch('os.listdir')
@mock.patch('os.path.isdir')
def test_create_framework_parser_fail_1(self, *args):
"""Test the function of fail to create framework parser."""
args[0].return_value = True
args[1].return_value = []
with pytest.raises(ProfilerFileNotFoundException) as exc_info:
FrameworkParser('JOB1', '0', '0')
assert exc_info.value.error_code == '50546084'
assert exc_info.value.message == 'The file <Framework> not found.'
@staticmethod
def _compare_json(expected_file, result_dict):
with open(expected_file, 'r') as file_handler:
expected_data = json.load(file_handler)
assert expected_data == result_dict