mindspore-Ecosystem
/
mindspore
21
0
Fork 27
Code Issues Pull Requests 6 Projects Releases Wiki Activity

clean code

This commit is contained in:
huangbingjian 2022-08-26 11:12:16 +08:00
parent 398aa7ac60
commit d0dd47ef3b
20 changed files with 46 additions and 413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ccsrc/frontend/operator/ops_front_infer_function.cc
View File

@ -1030,7 +1030,7 @@ AbstractBasePtr InferImplStringGetItem(const AnalysisEnginePtr &, const Primitiv
num += len;
}
std::string res;
res.append(1, str.at(num));
(void)res.append(1, str.at(num));
return std::make_shared<AbstractScalar>(res);
}

3
mindspore/ccsrc/frontend/optimizer/environ_conversion.cc
View File

@ -97,7 +97,8 @@ void TransformNodeAbstractIfEnvType(const AnfNodePtr &node, const abstract::Abst
TypeId GetValueType(const CNodePtr &cnode) {
// (EnvironSet/EnvironGet, environ, key, value/default)
if (cnode->inputs().size() != 4) {
constexpr size_t environ_input_size = 4;
if (cnode->size() != environ_input_size) {
MS_LOG(EXCEPTION) << "EnvrionSet/EnvironGet cnode should have 4 inputs, but: " << cnode->DebugString();
}
const auto &value_abstract = cnode->input(3)->abstract();

6
mindspore/ccsrc/frontend/optimizer/irpass/branch_culling.cc
View File

@ -160,8 +160,8 @@ bool HasDependencyOnSubGraph(const FuncGraphPtr &graph, const AnfNodePtr &state)
}
auto cur_cnode = cur_node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(cur_cnode);
for (size_t i = 1; i < cur_cnode->inputs().size(); i++) {
nodes.push(cur_cnode->inputs()[i]);
for (size_t i = 1; i < cur_cnode->size(); i++) {
nodes.push(cur_cnode->input(i));
}
}
return false;
@ -232,7 +232,7 @@ FuncGraphPtr TransformGraphCondBranchNodes(
if (should_replace) {
auto new_node = graph->NewCNode({});
repl_node[node] = new_node;
nodes_changed.emplace_back(node->cast<CNodePtr>(), new_node);
(void)nodes_changed.emplace_back(node->cast<CNodePtr>(), new_node);
}
}
RunSwitchNodeReplace(manager, nodes_changed, repl_node, repl_node_inputs, graph);

5
mindspore/ccsrc/frontend/optimizer/irpass/environ_eliminate.h
View File

@ -92,6 +92,7 @@ class EnvironGetAddEliminater : public AnfVisitor {
// {prim::kPrimEnvironGet, {...}, C, Z}
auto cnode = node->cast<CNodePtr>();
auto inp1 = cnode->input(1)->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(inp1);
auto c = cnode->input(2);
auto z = cnode->input(3);
@ -140,6 +141,7 @@ class EnvironGetSetEliminater : public AnfVisitor {
// {prim::kPrimEnvironGet, {...}, C2, Z}
auto cnode = node->cast<CNodePtr>();
auto inp1 = cnode->input(1)->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(inp1);
auto key2 = cnode->input(2);
auto c2 = GetValueNode<SymbolicKeyInstancePtr>(key2);
auto default_v = cnode->input(3);
@ -227,7 +229,8 @@ class SplitEnvironGetSetWithTupleValue : public AnfVisitor {
~SplitEnvironGetSetWithTupleValue() override = default;
AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override {
if (!(IsPrimitiveCNode(node, prim::kPrimEnvironSet) || IsPrimitiveCNode(node, prim::kPrimEnvironGet))) {
if (!(IsPrimitiveCNode(node, prim::kPrimEnvironSet) || IsPrimitiveCNode(node, prim::kPrimEnvironGet)) ||
node->func_graph() == nullptr) {
return nullptr;
}
// {prim::kPrimEnvironSet, E, key, node_with_abstract_is_tuple} or

2
mindspore/ccsrc/frontend/optimizer/irpass/ge/dropout_for_ge.cc
View File

@ -137,8 +137,10 @@ AnfNodePtr DropoutForGE::operator()(const OptimizerPtr &opt, const AnfNodePtr &n
AnfNodePtr DropoutGradForGE::operator()(const OptimizerPtr &opt, const AnfNodePtr &node) {
Reset();
MS_EXCEPTION_IF_NULL(node->func_graph());
AnfVisitor::Match(prim::kPrimDropoutGrad, {IsNode, IsNode})(node);
auto dropout_grad_node = node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(dropout_grad_node);
auto origin_prim = GetValueNode<PrimitivePtr>(dropout_grad_node->input(0));
auto keep_prob = origin_prim->GetAttr(kKeepProbAttrName);

2
mindspore/ccsrc/frontend/optimizer/irpass/ge/ge_specialized_prepare.cc
View File

@ -71,7 +71,9 @@ void GeTensorArrayPrepare::InsertFlowOutputToTA(const std::vector<AnfNodePtr> &a
}
void GeTensorArrayPrepare::TransformTASizeFromAttrToInput(const AnfNodePtr &node) {
MS_EXCEPTION_IF_NULL(node);
auto ta_node = node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(ta_node);
int32_t res_size = 0;
PrimitivePtr prim = GetValueNode<PrimitivePtr>(ta_node->input(0));
// get size attr

1
mindspore/ccsrc/frontend/optimizer/irpass/gradient_eliminate.cc
View File

@ -61,6 +61,7 @@ bool ExpandJPrim::operator()(const FuncGraphPtr &func_graph, const OptimizerPtr
// Expand j nodes that don't have embed j nodes.
bool change = false;
auto manager = optimizer->manager();
MS_EXCEPTION_IF_NULL(manager);
FuncGraphVector func_graphs;
HashMap<AnfNodePtr, size_t> j_node_to_index_map;
size_t index = 0;

2
mindspore/ccsrc/frontend/optimizer/irpass/inline.h
View File

@ -257,7 +257,7 @@ class InlinerBase : public AnfVisitor {
bool should_simplify = false;
for (size_t i = 0; i < fg_params.size(); i++) {
if (mng->node_users()[fg_params[i]].size() != 0) {
used_param_index.emplace_back(i);
(void)used_param_index.emplace_back(i);
} else {
MS_LOG(DEBUG) << "Not used parameter " << fg_params[i]->DebugString() << " for calling " << fg->ToString();
should_simplify = true;

2
mindspore/ccsrc/frontend/optimizer/irpass/item_tuple_or_list_eliminate.h
View File

@ -268,7 +268,7 @@ class TupleListSetitemEliminator : public AnfVisitor {
for (size_t i = 1; i < args_.size(); ++i) {
auto abs = args_[i]->abstract();
MS_EXCEPTION_IF_NULL(abs);
abs_list.emplace_back(abs->Broaden());
(void)abs_list.emplace_back(abs->Broaden());
}
make_tuple->set_abstract(std::make_shared<abstract::AbstractTuple>(abs_list));
}

1
mindspore/ccsrc/frontend/optimizer/irpass/meta_fg_var_prepare.cc
View File

@ -135,6 +135,7 @@ AnfNodePtr MetaFgVarPrepare::operator()(const OptimizerPtr &, const AnfNodePtr &
auto meta_func = value->cast<MetaFuncGraphPtr>();
MS_EXCEPTION_IF_NULL(meta_func);
auto grad_op_ptr = meta_func->cast<prim::GradOperationPtr>();
MS_EXCEPTION_IF_NULL(grad_op_ptr);
sens_param = grad_op_ptr->sens_param();
// Remove the tuple/list inputs from order list for grad(UnpackGraph(..), list/tuple)(..)

1
mindspore/ccsrc/frontend/optimizer/irpass/minmax_grad.h
View File

@ -40,6 +40,7 @@ class MinMaximumGrad : public AnfVisitor {
// check single use
auto mng = optimizer->manager();
MS_EXCEPTION_IF_NULL(mng);
auto &users = mng->node_users();
if (users.find(grad_) == users.end() || users[grad_].size() != 1) {
return nullptr;

12
mindspore/ccsrc/frontend/optimizer/irpass/parameter_eliminate.h
View File

@ -32,6 +32,7 @@ namespace mindspore {
namespace opt {
namespace irpass {
static inline std::vector<CNodePtr> GetCallers(const FuncGraphPtr &fg) {
MS_EXCEPTION_IF_NULL(fg);
const auto &fg_caller_and_indexes = fg->func_graph_cnodes_index();
std::vector<CNodePtr> caller_cnodes = {};
// Find all caller of fg.
@ -90,6 +91,7 @@ static inline std::pair<FuncGraphPtr, std::vector<CNodePtr>> SearchFuncGraphCall
static inline std::pair<mindspore::HashSet<size_t>, mindspore::HashMap<size_t, size_t>> EraseUnusedParameters(
const FuncGraphPtr &fg, bool eliminate_only_returned_parameter) {
MS_EXCEPTION_IF_NULL(fg);
const FuncGraphManagerPtr &manager = fg->manager();
MS_EXCEPTION_IF_NULL(manager);
const auto &manager_node_users = manager->node_users();
@ -160,14 +162,15 @@ static inline std::pair<mindspore::HashSet<size_t>, mindspore::HashMap<size_t, s
// Adjust the call arguments of func graph whose parameter's eliminated.
static inline void AdjustCallerArgs(const FuncGraphPtr &called, const CNodePtr &caller,
const mindspore::HashSet<size_t> &unused_parameter_indexes) {
MS_EXCEPTION_IF_NULL(caller->func_graph());
const FuncGraphManagerPtr &manager = caller->func_graph()->manager();
MS_EXCEPTION_IF_NULL(manager);
std::vector<AnfNodePtr> new_args = {caller->input(0)};
for (size_t i = 0; i < caller->inputs().size() - 1; i++) {
for (size_t i = 0; i < caller->size() - 1; i++) {
if (unused_parameter_indexes.find(i) == unused_parameter_indexes.end()) {
(void)new_args.emplace_back(caller->inputs()[i + 1]);
(void)new_args.emplace_back(caller->input(i + 1));
} else {
MS_LOG(DEBUG) << "Erase arg:" << caller->inputs()[i + 1]->DebugString() << ",index:" << i;
MS_LOG(DEBUG) << "Erase arg:" << caller->input(i + 1)->DebugString() << ",index:" << i;
}
}
// Remove any Args which may be packed into VarArgs if VarArgs is not used in called FuncGraph;
@ -176,7 +179,7 @@ static inline void AdjustCallerArgs(const FuncGraphPtr &called, const CNodePtr &
// 2. The arguments in caller may be less than the formal parameters in called as some parameters can have
// default value.
if (!called->has_vararg() &&
caller->inputs().size() > (1 + IntToSize(called->GetPositionalArgsCount()) + called->fv_param_count())) {
caller->size() > (1 + IntToSize(called->GetPositionalArgsCount()) + called->fv_param_count())) {
size_t start_offset = IntToSize(called->GetPositionalArgsCount()) + 1;
size_t end_offset = called->fv_param_count();
(void)new_args.erase(new_args.cbegin() + SizeToLong(start_offset), new_args.cend() - SizeToLong(end_offset));
@ -196,6 +199,7 @@ static inline void AdjustCallerArgs(const FuncGraphPtr &called, const CNodePtr &
// we should convert getitem(returned_tuple, x) into the eliminating argument itself.
static inline void AdjustGetItemCall(const CNodePtr &caller,
const mindspore::HashMap<size_t, size_t> &only_return_parameter_indexes) {
MS_EXCEPTION_IF_NULL(caller->func_graph());
const FuncGraphManagerPtr &manager = caller->func_graph()->manager();
MS_EXCEPTION_IF_NULL(manager);
if (only_return_parameter_indexes.empty()) {

1
mindspore/ccsrc/frontend/optimizer/irpass/shard_eliminate.cc
View File

@ -31,6 +31,7 @@ AnfNodePtr ExpandShard(const CNodePtr &node) {
bool ExpandShardPrim::operator()(const FuncGraphPtr &func_graph, const OptimizerPtr &optimizer) {
bool change = false;
auto manager = optimizer->manager();
MS_EXCEPTION_IF_NULL(manager);
for (auto &shard_node : prim_nodes_) {
auto expanded_shard = internal::ExpandShard(shard_node);
(void)manager->Replace(shard_node, expanded_shard);

1
mindspore/ccsrc/frontend/optimizer/irpass/taylor_eliminate.cc
View File

@ -136,6 +136,7 @@ bool ExpandTaylorPrim::operator()(const FuncGraphPtr &, const OptimizerPtr &opti
// Search all taylor nodes.
bool change = false;
auto manager = optimizer->manager();
MS_EXCEPTION_IF_NULL(manager);
for (auto &taylor_node : prim_nodes_) {
auto taylor_fg_node = taylor_node->input(1);
auto taylor_fg = GetValueNode<FuncGraphPtr>(taylor_fg_node);

17
mindspore/ccsrc/frontend/optimizer/irpass/updatestate_eliminate.cc
View File

@ -313,8 +313,8 @@ void GetLoadsFromUpdateState(const CNodePtr &update_state, std::vector<CNodePtr>
void GetLoadsFollowLoad(const CNodePtr &update_state, const CNodePtr &load, std::vector<CNodePtr> *update_states,
std::vector<CNodePtr> *loads) {
update_states->emplace_back(update_state);
loads->emplace_back(load);
(void)update_states->emplace_back(update_state);
(void)loads->emplace_back(load);
auto &load_attach = load->input(kAttachIndex);
if (IsPrimitiveCNode(load_attach, prim::kPrimUpdateState)) {
GetLoadsFromUpdateState(load_attach->cast<CNodePtr>(), update_states, loads);
@ -338,10 +338,10 @@ void GetLoadsFollowTuple(const CNodePtr &update_state, const CNodePtr &make_tupl
return;
}
// Add update_state and load nodes.
update_states->emplace_back(update_state);
(void)update_states->emplace_back(update_state);
for (size_t i = 1; i < inputs.size(); ++i) {
auto &element = inputs.at(i);
loads->emplace_back(element->cast<CNodePtr>());
(void)loads->emplace_back(element->cast<CNodePtr>());
}
// Follow prev update state if found.
auto prev_node = update_state->input(kInputIndex);
@ -398,7 +398,7 @@ void EliminateUselessNodesForUpdateStates(const std::vector<CNodePtr> &update_st
std::vector<AnfNodePtr> depend_nodes;
for (auto &user : us_users) {
if (IsPrimitiveCNode(user.first, prim::kPrimDepend) && user.second == kAttachIndex) {
depend_nodes.emplace_back(user.first);
(void)depend_nodes.emplace_back(user.first);
}
}
if (depend_nodes.empty()) {
@ -450,7 +450,7 @@ AnfNodePtr EliminateUpdateStateForLoads(const CNodePtr &old_update_state, const
AnfNodePtrList make_tuple_inputs;
std::set<AnfNodePtr> loaded_para_set;
make_tuple_inputs.reserve(loads.size() + 1);
make_tuple_inputs.emplace_back(NewValueNode(prim::kPrimMakeTuple));
(void)make_tuple_inputs.emplace_back(NewValueNode(prim::kPrimMakeTuple));
auto input_monad = loads.back()->input(kAttachIndex);
for (auto iter = loads.rbegin(); iter != loads.rend(); ++iter) {
auto &load = *iter;
@ -458,7 +458,7 @@ AnfNodePtr EliminateUpdateStateForLoads(const CNodePtr &old_update_state, const
const bool is_new_load = result.second;
if (is_new_load) {
// Put Load node as a tuple element, if the parameter is not loaded by other Load.
make_tuple_inputs.emplace_back(load);
(void)make_tuple_inputs.emplace_back(load);
}
auto load_attach = load->input(kAttachIndex);
if (load_attach != input_monad) {
@ -818,6 +818,7 @@ bool UpdatestateDependEliminater::operator()(const FuncGraphPtr &func_graph, con
bool change = false;
auto manager = optimizer->manager();
MS_EXCEPTION_IF_NULL(manager);
auto &all_nodes = manager->all_nodes();
std::vector<AnfNodePtr> todo = DeepScopedGraphSearchWithFilter(func_graph->get_return(), AlwaysInclude, filter);
for (auto &node : todo) {
@ -852,6 +853,7 @@ bool UpdatestateAssignEliminater::operator()(const FuncGraphPtr &func_graph, con
bool change = false;
auto manager = optimizer->manager();
MS_EXCEPTION_IF_NULL(manager);
auto &all_nodes = manager->all_nodes();
std::vector<AnfNodePtr> todo = DeepScopedGraphSearchWithFilter(func_graph->get_return(), AlwaysInclude, filter);
for (auto &node : todo) {
@ -888,6 +890,7 @@ bool UpdatestateLoadsEliminater::operator()(const FuncGraphPtr &func_graph, cons
bool change = false;
auto manager = optimizer->manager();
MS_EXCEPTION_IF_NULL(manager);
auto &all_nodes = manager->all_nodes();
std::vector<AnfNodePtr> todo = DeepScopedGraphSearchWithFilter(func_graph->get_return(), AlwaysInclude, filter);
for (auto &node : todo) {

2
mindspore/ccsrc/frontend/optimizer/irpass/vmap_eliminate.cc
View File

@ -385,6 +385,7 @@ void ExpandVmapValueNode(const FuncGraphPtr &vmap_fg, const pipeline::ResourceBa
int axis_size) {
// Map ValueNode.
auto manager = resource->manager();
MS_EXCEPTION_IF_NULL(manager);
auto value_nodes = vmap_fg->value_nodes();
for (const auto &value_pair : value_nodes) {
auto node = value_pair.first;
@ -498,6 +499,7 @@ bool ExpandVmapPrim::operator()(const FuncGraphPtr &, const OptimizerPtr &optimi
// Expand vmap nodes that don't have embed j or vmap nodes.
bool change = false;
auto manager = optimizer->manager();
MS_EXCEPTION_IF_NULL(manager);
for (auto &vmap_node : prim_nodes_) {
auto VmapPrim = GetValueNode<PrimitivePtr>(vmap_node->input(0));
MS_EXCEPTION_IF_NULL(VmapPrim);

6
mindspore/ccsrc/frontend/optimizer/opt.cc
View File

@ -100,6 +100,7 @@ static inline bool isTraversable(const AnfNodePtr &node) {
static AnfNodePtr DoTransform(const OptimizerPtr &optimizer, const AnfNodePtr &node,
const SubstitutionPtr &substitution) {
auto manager = optimizer->manager();
MS_EXCEPTION_IF_NULL(manager);
bool is_match = substitution->predicate_(node);
if (is_match) {
TraceGuard trace_guard(std::make_shared<TraceOpt>(node->debug_info()));
@ -164,6 +165,7 @@ static void UpdateTransformingListWithUserNodes(const OptimizerPtr &optimizer, c
return;
}
auto manager = optimizer->manager();
MS_EXCEPTION_IF_NULL(manager);
auto &node_users = manager->node_users();
auto users_iterator = node_users.find(node);
if (users_iterator == node_users.end()) {
@ -226,9 +228,10 @@ bool SubstitutionList::ApplySubstitutionToIR(const OptimizerPtr &optimizer, cons
double start = GetTime();
#endif
FuncGraphManagerPtr manager = optimizer->manager();
MS_EXCEPTION_IF_NULL(manager);
auto seen = NewSeenGeneration();
std::deque<AnfNodePtr> todo;
todo.emplace_back(func_graph->output());
(void)todo.emplace_back(func_graph->output());
bool changes = false;
auto &all_nodes = manager->all_nodes();
@ -330,6 +333,7 @@ bool SubstitutionList::operator()(const FuncGraphPtr &func_graph, const Optimize
MS_EXCEPTION_IF_NULL(optimizer);
MS_EXCEPTION_IF_NULL(func_graph);
FuncGraphManagerPtr manager = optimizer->manager();
MS_EXCEPTION_IF_NULL(manager);
manager->AddFuncGraph(func_graph);
bool changes = false;
static const auto traverse_mode =

143
tests/perf_test/mind_expression_perf/generate_report.py
View File

@ -1,143 +0,0 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import sys
import re
import json
import os
import time
import openpyxl as opx
def parse_arguments():
print(sys.argv)
me_report_path = sys.argv[1]
log_path = sys.argv[2]
n_iter = sys.argv[3]
out = sys.argv[4]
assert n_iter.isdigit()
return me_report_path, log_path, int(n_iter), out
def extract_by_keyword(doc, keyword, pattern):
rst = []
for i, s in enumerate(doc):
if keyword in s:
p = re.findall(pattern, s)
print("L%d: extracted %s from '%s'" % (i, p, s.strip()))
rst.extend(p)
return rst
def process_log(fname, log_path, n_iter, keyword, pattern):
rnt = {}
for i in range(1, 1+n_iter):
fname_path = os.path.join(log_path, fname % i)
with open(fname_path) as f:
print("\nLoading %s" % fname_path)
rst = extract_by_keyword(f, keyword, pattern)
rnt[fname % i] = rst
return rnt
def summarize(func):
def wrapper(*args, **kwargs):
log = func(*args, **kwargs)
times = list(log.items())
times.sort(key=lambda x: x[1])
min_file, min_time = times[0]
avg = sum(map(lambda x: x[1], times)) / len(times)
log["min_time"] = min_time
log["min_file"] = min_file
log["avg_time"] = avg
return log
return wrapper
@summarize
def process_bert_log(log_path, n_iter):
fname = "bert%d.log"
total = process_log(fname, log_path, n_iter, "TotalTime", r"\d+.\d+")
task = process_log(fname, log_path, n_iter, "task_emit", r"\d+.\d+")
log = {}
for fname in total:
log[fname] = float(total[fname][0]) - float(task[fname][0])
return log
@summarize
def process_resnet_log(log_path, n_iter):
fname = "resnet%d.log"
total = process_log(fname, log_path, n_iter, "TotalTime", r"\d+.\d+")
task = process_log(fname, log_path, n_iter, "task_emit", r"\d+.\d+")
log = {}
for fname in total:
log[fname] = float(total[fname][0]) - float(task[fname][0])
return log
@summarize
def process_gpt_log(log_path, n_iter):
fname = "gpt%d.log"
total = process_log(fname, log_path, n_iter, "TotalTime", r"\d+.\d+")
task = process_log(fname, log_path, n_iter, "task_emit", r"\d+.\d+")
log = {}
for fname in total:
log[fname] = float(total[fname][0]) - float(task[fname][0])
return log
@summarize
def process_reid_log(log_path, n_iter):
log = {}
for i in range(8):
fname = "reid_%d_"+str(i)+".log"
total = process_log(fname, log_path, n_iter, "TotalTime", r"\d+.\d+")
task = process_log(fname, log_path, n_iter, "task_emit", r"\d+.\d+")
for fname in total:
log[fname] = float(total[fname][0]) - float(task[fname][0])
return log
def write_to_me_report(log, me_report_path):
wb = opx.load_workbook(me_report_path)
sheet = wb["Sheet"]
idx = sheet.max_row + 1
date = time.strftime('%m%d', time.localtime())
sheet['A%d' % idx] = date
sheet['B%d' % idx] = round(log["reid"]["min_time"], 2)
sheet['C%d' % idx] = round(log["bert"]["min_time"], 2)
sheet['D%d' % idx] = round(log['resnet']["min_time"], 2)
sheet['E%d' % idx] = round(log['gpt']["min_time"], 2)
wb.save(me_report_path)
def generate_report():
me_report_path, log_path, n_iter, out = parse_arguments()
log_data = {}
bert_log = process_bert_log(log_path, n_iter)
resnet_log = process_resnet_log(log_path, n_iter)
gpt_log = process_gpt_log(log_path, n_iter)
reid_log = process_reid_log(log_path, n_iter)
log_data["bert"] = bert_log
log_data["resnet"] = resnet_log
log_data["gpt"] = gpt_log
log_data["reid"] = reid_log
with open(out, "w") as f:
json.dump(log_data, f, indent=2)
write_to_me_report(log_data, me_report_path)
if __name__ == "__main__":
generate_report()

104
tests/perf_test/mind_expression_perf/process_data.py
View File

@ -1,104 +0,0 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import os
import sys
import json
import matplotlib.ticker as ticker
import matplotlib.pyplot as plt
import openpyxl as opx
def parse_arguments():
log_path = sys.argv[1]
log_data = sys.argv[2]
me_report = sys.argv[3]
n_days = sys.argv[4]
assert n_days.isdigit()
return log_path, log_data, me_report, int(n_days)
def read_data(log_data, me_report_path, n_days):
with open(log_data) as f:
log = json.load(f)
wb = opx.load_workbook(me_report_path)
sheet = wb["Sheet"]
n_row = sheet.max_row
date = [cell[0].value for cell in sheet["A2":"A%d" % n_row]]
reid_data = [float(cell[0].value) for cell in sheet["B2":"B%d" % n_row]]
bert_data = [float(cell[0].value) for cell in sheet["C2":"C%d" % n_row]]
resnet_data = [float(cell[0].value) for cell in sheet["D2":"D%d" % n_row]]
gpt_data = [float(cell[0].value) for cell in sheet["E43":"E%d" % n_row]]
if n_days > 0:
date = date[-n_days:]
reid_data = reid_data[-n_days:]
bert_data = bert_data[-n_days:]
resnet_data = resnet_data[-n_days:]
gpt_data = gpt_data[-n_days:]
return log, date, reid_data, bert_data, resnet_data, gpt_data
def draw_figure(x_data, y_data, labels, title, out, height=24, width=8, tick_space=2):
print("Generating figure to: %s" % out)
plt.figure(figsize=(height, width))
for y, label in zip(y_data, labels):
x = x_data[-len(y):]
n_data = len(x)
assert len(x) == len(
y), "assume len(x) == len(y), while %d != %d" % (len(x), len(y))
plt.plot(x, y, linewidth=2, marker='o', markersize=5, label=label)
ax = plt.gca()
ax.xaxis.set_major_locator(ticker.MultipleLocator(tick_space))
for i in range(n_data):
if i % 2 == 0:
plt.text(x[i], y[i], y[i], ha='center',
va='bottom', fontsize=8)
plt.title(title)
plt.xlabel("Date")
plt.ylabel("Time(s)")
plt.grid()
plt.legend()
plt.savefig(out)
def generate_report(log, labels, log_path):
for label in labels:
fname = log[label]["min_file"]
fname_path = os.path.join(log_path, fname)
out_path = os.path.join(log_path, "reports", label+"_me.log")
print("Generating report to: %s" % out_path)
os.system("grep -A 230 'TotalTime = ' %s > %s" %
(fname_path, out_path))
def process_data():
log_path, log_data, me_report, n_days = parse_arguments()
log, date, reid_data, bert_data, resnet_data, gpt_data = read_data(
log_data, me_report, n_days)
draw_figure(date,
[reid_data, bert_data, gpt_data],
["ReID", "BERT", "GPT"],
"ReID&BERT&GPT",
os.path.join(log_path, "reports", "reid_bert_gpt.png")
)
draw_figure(date, [resnet_data], ["ResNet"], "ResNet",
os.path.join(log_path, "reports", "resnet.png"))
generate_report(log, list(log.keys()), log_path)
if __name__ == "__main__":
process_data()

146
tests/perf_test/mind_expression_perf/run.sh
View File

@ -1,146 +0,0 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
stage=0
days=7
iter=5
device_id=0
n_worker=128
work_dir="/opt/npu/me_monitor"
me_report_path=$work_dir/logs/ME_report_daily.xlsx
log_dir=logs_$(date "+%m%d-%H%M")
log_path=$work_dir/logs/$log_dir
ms_master="https://gitee.com/mindspore/mindspore.git"
log_data="data.json"
ci_mode=true
set -e
set -o pipefail
# parse arguments from command line
while getopts "s:d:i:l:" args
do
case $args in
s)
stage=$OPTARG
;;
d)
days=$OPTARG
;;
i)
iter=$OPTARG
;;
l)
log_dir=$OPTARG
log_path=$work_dir/logs/$log_dir
;;
?)
echo "unknown argument"
exit 1
;;
esac
done
source env.sh
export DEVICE_ID=$device_id
echo "Args: days=$days, iter=$iter, log_path=$log_path"
cd $work_dir
echo $WORKSPACE
WORKSPACE=/home/jenkins-slave/workspace/MindSpore_Network_reid_compile_performance
echo $WORKSPACE
if [ $stage -le 1 ]; then
echo ""
echo "===========Stage 1: Fetching latest mindspore from master==========="
if [ -d mindspore ]; then
rm -rf mindspore
fi
git clone $ms_master
fi
if [ $stage -le 2 ]; then
echo ""
echo "===========Stage 2: Building mindspore==========="
cd $work_dir/mindspore
bash build.sh -e ascend -j $n_worker -p on
fi
if [ $stage -le 3 ]; then
echo ""
echo "===========Stage 3: Compiling networks==========="
cd $work_dir
mkdir -p $log_path
# Compiling ReID-8
# split resource-consuming task from others
for count in $(seq 1 $iter); do
echo "[INFO] Compiling ReID-8p, iteration $count"
if [ -d reid$count ]; then
rm -rf reid$count
fi
mkdir reid$count
cd reid$count
bash $work_dir/faceReidToMe/dist_env/env_26/dist_env_26.sh
for num in {0..7}; do
cp device_$num/test_reid_stage123_1024node_graphdata_dynamiclossscale_log$num.log $log_path/reid_${count}_${num}.log
done
cd $work_dir
mv reid$count $log_path
done
# Compiling BERT
cd $work_dir
for count in $(seq 1 $iter); do
echo "[INFO] Compiling BERT, iteration $count"
pytest -s mindspore/tests/perf_test/bert/test_bert_train.py::test_bert_train | tee $log_path/bert$count.log
done
# Compiling ResNet50
for count in $(seq 1 $iter); do
echo "[INFO] Compiling ResNet50, iteration $count"
pytest -s mindspore/tests/perf_test/test_resnet_train.py::test_train_step | tee $log_path/resnet$count.log
done
# Compiling GPT
for count in $(seq 1 $iter); do
echo "[INFO] Compiling GPT, iteration $count"
cd gpt
bash scripts/run_standalone_train.sh 0 1 $work_dir/gpt_data | tee $log_path/gpt$count.log
done
fi
if [ $stage -le 4 ]; then
echo ""
echo "===========Stage 4: Processing log files==========="
cd $work_dir
python process_data.py $me_report_path $log_path $iter $log_path/$log_data
fi
if [ $stage -le 5 ]; then
echo ""
echo "===========Stage 5: Generating reports==========="
if [ ! -d $log_path/reports ]; then
mkdir $log_path/reports
fi
python generate_report.py $log_path $log_path/$log_data $me_report_path $days
if [ $ci_mode ]; then
echo "copying file to artifacts"
mkdir -p ${WORKSPACE}/archive
cp $log_path/reports/* ${WORKSPACE}/archive
fi
fi