Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!647 change profiling iter_end point after last tbe kernel 

Merge pull request !647 from caifubi/profiling_iter_end_point
This commit is contained in:
mindspore-ci-bot 2020-04-26 11:24:00 +08:00 committed by Gitee
parent ad2a5dcad2 3f69b5b518
commit 2a31fce599
2 changed files with 20 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc
View File

@ -33,7 +33,7 @@ constexpr char kIterEndNode[] = "PROFILING_ITER_END";
std::unordered_map<uint32_t, std::vector<std::string>> ProfilingUtils::graph_kernel_name_;
uint32_t ProfilingUtils::custom_node_index_ = 1;
ProfilingTraceInfo ProfilingUtils::GetProfilingTraceFromEnv(const NotNull<session::KernelGraph *> graph_ptr) {
ProfilingTraceInfo ProfilingUtils::GetProfilingTraceFromEnv(NotNull<const session::KernelGraph *> graph_ptr) {
MS_LOG(INFO) << "get env start";
custom_node_index_ = 1;
auto &cnode_exec_order = graph_ptr->execution_order();
@ -148,18 +148,29 @@ std::string ProfilingUtils::GetTraceBpEnd(const std::vector<CNodePtr> &cnode_exe
}
if (bp_end_str.empty()) {
auto last_cnode = cnode_exec_order.back();
MS_EXCEPTION_IF_NULL(last_cnode);
bp_end_str = last_cnode->fullname_with_scope();
bp_end_str = GetGraphLastTbeKernelName(cnode_exec_order);
}
return bp_end_str;
}
std::string ProfilingUtils::GetGraphLastTbeKernelName(const std::vector<CNodePtr> &cnode_exec_order) {
std::string last_tbe_kernel_name = "";
// find last tbe_kernel
for (auto iter = cnode_exec_order.rbegin(); iter != cnode_exec_order.rend(); ++iter) {
if (AnfAlgo::GetKernelType(*iter) == TBE_KERNEL) {
last_tbe_kernel_name = (*iter)->fullname_with_scope();
break;
}
}
if (last_tbe_kernel_name.empty()) {
MS_LOG(WARNING) << "tbe kernel not found in graph";
}
return last_tbe_kernel_name;
}
std::string ProfilingUtils::GetTraceNetoutput(const std::vector<CNodePtr> &cnode_exec_order) {
const char *trace_netoutput = std::getenv(kIterEndNode);
auto &last_cnode = cnode_exec_order.back();
MS_EXCEPTION_IF_NULL(last_cnode);
return trace_netoutput == nullptr ? last_cnode->fullname_with_scope() : std::string(trace_netoutput);
return trace_netoutput == nullptr ? GetGraphLastTbeKernelName(cnode_exec_order) : std::string(trace_netoutput);
}
NotNull<CNodePtr> ProfilingUtils::CreateProfilingCNode(const ProfilingContent &profiling_content,

3
mindspore/ccsrc/device/ascend/profiling/profiling_utils.h
View File

@ -94,7 +94,7 @@ class ProfilingUtils {
// And other cnode, like AllReduce, export PROFILING_CUSTOM_1='full name of AllReduce cnode'
// GetNext, export PROFIFLING_CUSTOM_2='full name fo GetNext cnode'
// The variable i in PROFILING_CUSTOM_i should start from 1 without interruption.
static ProfilingTraceInfo GetProfilingTraceFromEnv(const NotNull<session::KernelGraph *> graph_ptr);
static ProfilingTraceInfo GetProfilingTraceFromEnv(NotNull<const session::KernelGraph *> graph_ptr);
// Insert two profiling trace points, one in front and one behind
static void ProfilingCustomOp(const mindspore::AnfNodePtr &anf_node, const ProfilingTraceInfo &profiling_trace_info,
@ -114,6 +114,7 @@ class ProfilingUtils {
static std::string GetTraceBegin(const std::vector<CNodePtr> &cnode_exec_order);
static std::string GetTraceBpEnd(const std::vector<CNodePtr> &cnode_exec_order);
static std::string GetTraceNetoutput(const std::vector<CNodePtr> &cnode_exec_order);
static std::string GetGraphLastTbeKernelName(const std::vector<CNodePtr> &cnode_exec_order);
static void GetTraceHccl(const std::vector<CNodePtr> &cnode_exec_order,
NotNull<ProfilingTraceInfo *> profiling_trace);
static void GetCNodeOutputRealNode(const std::string &node_name, const std::vector<CNodePtr> &cnode_exec_order,