Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!34789 Add auto dynamic shape for PyNative 

Merge pull request !34789 from zjun/auto_dynamic
This commit is contained in:
i-robot 2022-06-01 08:47:28 +00:00 committed by Gitee
parent a95fd90b07 dcf58cd750
commit 4ec1537e7f
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
10 changed files with 753 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

299
mindspore/ccsrc/pipeline/pynative/pynative_execute.cc
View File

@ -70,7 +70,6 @@ GradExecutorPtr PynativeExecutor::grad_executor_ = nullptr;
std::mutex PynativeExecutor::instance_lock_;
namespace {
const size_t PTR_LEN = 15;
const size_t ARG_SIZE = 2;
const size_t MAX_TOP_CELL_COUNTS = 20;
const size_t kThreshold = 3;
@ -156,6 +155,8 @@ std::string GetId(const py::handle &obj) {
return param_info->name();
}
return tensor_ptr->id();
} else if (py::isinstance<Cell>(obj)) {
return obj.cast<CellPtr>()->id();
} else if (py::isinstance<mindspore::Type>(obj)) {
auto type_ptr = py::cast<mindspore::TypePtr>(obj);
return "type" + type_ptr->ToString();
@ -180,7 +181,7 @@ std::string GetId(const py::handle &obj) {
return prefix;
}
if (py::isinstance<Cell>(obj) || py::isinstance<py::function>(obj)) {
if (py::isinstance<py::function>(obj)) {
auto it = g_pyobj_id_cache.find(obj);
if (it == g_pyobj_id_cache.end()) {
auto id = GetPyObjId(obj);
@ -492,6 +493,12 @@ bool RunOpConvertConstInputToAttr(const OpExecInfoPtr &op_run_info, size_t input
}
const auto &value = PyObjToValue(input_object);
auto input_name = input_names_vec[input_index];
if (value->isa<tensor::Tensor>()) {
auto tensor = value->cast<tensor::TensorPtr>();
if (tensor->data().const_data() == nullptr) {
return false;
}
}
op_prim->AddAttr(input_name, value);
op_run_info->index_with_value.emplace_back(std::make_pair(input_index, value));
return true;
@ -1077,6 +1084,23 @@ void SaveIdWithDynamicAbstract(const py::object &obj, const AbstractBasePtr &abs
}
}
ShapeVector GetTensorShape(const py::object &obj) {
if (py::isinstance<tensor::Tensor>(obj)) {
return obj.cast<tensor::TensorPtr>()->shape();
}
return {};
}
TypePtr GetTypeFromAbstract(const abstract::AbstractBasePtr &abs) {
MS_EXCEPTION_IF_NULL(abs);
if (abs->isa<abstract::AbstractSequence>()) {
MS_LOG(EXCEPTION) << "Get tuple or list abs";
}
const auto &type = abs->BuildType();
MS_EXCEPTION_IF_NULL(type);
return type;
}
abstract::ShapePtr GetShapeFromAbstract(const abstract::AbstractBasePtr &abs) {
MS_EXCEPTION_IF_NULL(abs);
if (abs->isa<abstract::AbstractSequence>()) {
@ -1103,8 +1127,8 @@ void SaveIdWithDynamicShape(const OpExecInfoPtr &op_exec_info, const std::string
}
} else {
const auto &dynamic_shape_vec = GetShapeFromAbstract(dynamic_abs);
MS_LOG(DEBUG) << "Save tensor " << id << ", real shape " << PyObjToValue(real_obj)->ToAbstract()
<< ", dynamic shape " << dynamic_shape_vec->ToString();
MS_LOG(DEBUG) << "Save tensor " << id << ", real shape " << GetTensorShape(real_obj) << ", dynamic shape "
<< dynamic_shape_vec->ToString();
op_exec_info->id_with_dynamic_shape.emplace(std::make_pair(id, dynamic_shape_vec));
}
}
@ -1123,8 +1147,8 @@ void UpdateInputTensorToDynamicShape(const OpExecInfoPtr &op_exec_info, std::vec
}
}
void UpdateOutputTensorToDynamicShape(
const ValuePtr &value, const OrderedMap<std::string, abstract::AbstractBasePtr> &obj_id_with_dynamic_abs) {
void UpdateValueToDynamicShape(const ValuePtr &value,
const OrderedMap<std::string, abstract::AbstractBasePtr> &obj_id_with_dynamic_abs) {
MS_EXCEPTION_IF_NULL(value);
if (value->isa<mindspore::tensor::Tensor>()) {
auto tensor_value = value->cast<tensor::TensorPtr>();
@ -1135,10 +1159,10 @@ void UpdateOutputTensorToDynamicShape(
} else if (value->isa<ValueTuple>()) {
auto value_tuple = value->cast<ValueTuplePtr>();
for (const auto &v : value_tuple->value()) {
UpdateOutputTensorToDynamicShape(v, obj_id_with_dynamic_abs);
UpdateValueToDynamicShape(v, obj_id_with_dynamic_abs);
}
} else {
MS_LOG(EXCEPTION) << "Out put is not a tensor";
MS_LOG(DEBUG) << "Out put is not a tensor";
}
}
@ -1174,6 +1198,45 @@ ValuePtr SetSensValue(const ValuePtr &value, TensorIdWithTensorObject *tensor_id
return value;
}
}
void FindMatchTopCell(const TopCellInfoPtr &top_cell, const py::args &args, std::vector<ShapeVector> *new_args_shape) {
MS_EXCEPTION_IF_NULL(top_cell);
MS_EXCEPTION_IF_NULL(new_args_shape);
for (size_t i = 0; i < args.size(); ++i) {
const auto &cur_value_abs = PyObjToValue(args[i])->ToAbstract();
MS_EXCEPTION_IF_NULL(cur_value_abs);
const auto &cur_type = GetTypeFromAbstract(cur_value_abs);
const auto &elem_type = top_cell->cell_self_info()->args_type[i];
// Type is not the same
if (cur_type->hash() != elem_type->hash()) {
MS_LOG(DEBUG) << "The " << i << "th args type is not the same, cur is " << cur_type->ToString()
<< " and the elem is " << elem_type->ToString();
return;
}
// Check shape
const auto &cur_shape = GetShapeFromAbstract(cur_value_abs)->shape();
auto elem_shape = top_cell->cell_self_info()->args_shape[i]->shape();
if (cur_shape.size() != elem_shape.size()) {
MS_LOG(DEBUG) << "The " << i << "th args shape size is not the same, cur is " << cur_shape.size()
<< " and the elem is " << elem_shape.size();
return;
}
ShapeVector new_shape;
for (size_t j = 0; j < cur_shape.size(); ++j) {
if (cur_shape[j] == elem_shape[j]) {
new_shape.emplace_back(cur_shape[j]);
} else {
new_shape.emplace_back(-1);
}
}
// All shape can not be -1
if (std::any_of(new_shape.begin(), new_shape.end(), [](int64_t s) { return s != -1; })) {
new_args_shape->emplace_back(new_shape);
} else {
MS_LOG(DEBUG) << "Cur shape " << cur_shape << ", elem shape " << elem_shape << ", and new shape is " << new_shape;
}
}
}
} // namespace
py::object RealRunOp(const py::args &args) {
@ -1201,6 +1264,25 @@ GradExecutorPtr ForwardExecutor::grad() const {
return grad_executor;
}
void TopCellInfo::SetCellSelfInfoForTopCell(const py::object &cell, const py::args &args) {
std::vector<std::string> args_id;
std::vector<abstract::ShapePtr> args_shape;
std::vector<TypePtr> args_type;
for (size_t i = 0; i < args.size(); ++i) {
auto value = PyObjToValue(args[i]);
MS_EXCEPTION_IF_NULL(value);
auto abs = value->ToAbstract();
auto shape_ptr = abs->BuildShape()->cast<abstract::ShapePtr>();
if (shape_ptr == nullptr) {
return;
}
args_id.emplace_back(GetId(args[i]));
args_shape.emplace_back(shape_ptr);
args_type.emplace_back(abs->BuildType());
}
set_cell_self_info(std::make_shared<CellSelfInfo>(GetId(cell), args_id, args_shape, args_type));
}
bool TopCellInfo::IsSubCell(const std::string &cell_id) const {
if (sub_cell_list_.empty()) {
MS_LOG(DEBUG) << "The sub cell list is empty, there is no sub cell";
@ -1394,7 +1476,7 @@ AbstractBasePtr ForwardExecutor::GetInputObjAbstract(const OpExecInfoPtr &op_exe
if (it != node_abs_map_.end()) {
abs = it->second;
}
MS_LOG(DEBUG) << "Abstract cache hit " << (abs == nullptr);
MS_LOG(DEBUG) << "Abstract cache hit " << (abs != nullptr);
bool is_const_prim_or_input = IsConstPrimOrConstInput(op_exec_info, i);
if (abs == nullptr || is_const_prim_or_input) {
abs = PyObjToValue(obj)->ToAbstract();
@ -2226,7 +2308,7 @@ void GradExecutor::DoOpGrad(const OpExecInfoPtr &op_exec_info, const CNodePtr &c
}
}
if (op_exec_info->has_dynamic_output) {
UpdateOutputTensorToDynamicShape(op_out, forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs);
UpdateValueToDynamicShape(op_out, forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs);
}
if (!ad::GradPynativeOp(top_cell()->k_pynative_cell_ptr(), cnode, input_args, op_out)) {
@ -2234,18 +2316,30 @@ void GradExecutor::DoOpGrad(const OpExecInfoPtr &op_exec_info, const CNodePtr &c
}
}
void GradExecutor::SaveDynShapeAbsForMsFunction(const py::object &forward_out, const FuncGraphPtr &ms_func_graph) {
void GradExecutor::SaveDynShapeAbsForMsFunction(const py::args &args, const py::object &out,
const FuncGraphPtr &ms_func_graph) {
MS_EXCEPTION_IF_NULL(ms_func_graph);
auto output_node = ms_func_graph->output();
MS_EXCEPTION_IF_NULL(output_node);
if (ms_func_graph->modify_output()) {
auto make_tuple = output_node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(make_tuple);
output_node = make_tuple->input(1);
MS_EXCEPTION_IF_NULL(output_node);
// Update input to dynamic
for (size_t i = 0; i < args.size(); ++i) {
if (py::isinstance<tensor::Tensor>(args[i])) {
const auto &input_i_tensor = args[i].cast<tensor::TensorPtr>();
UpdateValueToDynamicShape(input_i_tensor, forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs);
}
}
SaveIdWithDynamicAbstract(forward_out, output_node->abstract(),
// Update output to dynamic
SaveIdWithDynamicAbstract(out, output_node->abstract(),
&(forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs));
const auto &output_value = PyObjToValue(out);
UpdateValueToDynamicShape(output_value, forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs);
// Save output by one id for abs get performance
if (output_node->abstract()->BuildShape()->IsDynamic()) {
forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs[GetId(out)] = output_node->abstract();
}
}
void GradExecutor::UpdateMsFunctionForwardTensors(const OpExecInfoPtr &op_exec_info,
@ -2334,9 +2428,9 @@ void GradExecutor::MakeCNodeForMsFunction(const FuncGraphPtr &ms_func_graph, con
}
// Make adjoint for ms_function fprop graph and connect it with previous op
void GradExecutor::MakeAdjointForMsFunction(const FuncGraphPtr &ms_func_graph, const FuncGraphPtr &grad_graph,
const py::object &actual_out, const py::args &args,
const ValuePtr &actual_out_v) {
CNodePtr GradExecutor::MakeAdjointForMsFunction(const FuncGraphPtr &ms_func_graph, const FuncGraphPtr &grad_graph,
const py::object &actual_out, const py::args &args,
const ValuePtr &actual_out_v) {
ValuePtrList input_values;
CNodePtr ms_function_cnode = nullptr;
MakeCNodeForMsFunction(ms_func_graph, args, &input_values, &ms_function_cnode);
@ -2353,6 +2447,7 @@ void GradExecutor::MakeAdjointForMsFunction(const FuncGraphPtr &ms_func_graph, c
<< ms_function_cnode->DebugString();
}
top_cell()->set_ms_function_flag(true);
return ms_function_cnode;
}
void GradExecutor::UpdateForwardTensorInfoInBpropGraph(const string &op_info, const ValuePtr &op_out) {
@ -2426,11 +2521,7 @@ void GradExecutor::SaveForwardTensorInfoInBpropGraph(const pipeline::ResourcePtr
}
// Check exception case.
auto &tensor_id_with_tensor_object = top_cell()->tensor_id_with_tensor_object();
if (tensor_id_with_tensor_object.size() > 1) {
MS_LOG(EXCEPTION) << "When compile a top graph, the tensor_id_with_tensor_object map should be empty or only have "
"sens value. Top cell: "
<< top_cell()->cell_id();
}
MS_LOG(DEBUG) << "Current tensor_id_with_tensor_object size " << tensor_id_with_tensor_object.size();
// Save tensor in value node of bprop graph
for (const auto &tensor : tensors_in_bprop_graph) {
MS_EXCEPTION_IF_NULL(tensor);
@ -2555,7 +2646,6 @@ void ForwardExecutor::CheckIfNeedSyncForHeterogeneous(const std::string &cur_tar
}
void ForwardExecutor::SetDynamicInput(const py::object &cell, const py::args &args) {
MS_LOG(DEBUG) << "Set dynamic input for feed mode";
auto &dynamic_index = dynamic_shape_info_ptr()->feed_dynamic_input[GetId(cell)];
dynamic_index.resize(args.size());
for (size_t i = 0; i < args.size(); i++) {
@ -2575,8 +2665,8 @@ void ForwardExecutor::SetFeedDynamicInputAbs(const py::object &cell, const py::a
auto it = feed_dynamic_input.find(cell_id);
if (it != feed_dynamic_input.end()) {
if (it->second.size() != args.size()) {
MS_LOG(EXCEPTION) << "Dynamic input size " << it->second.size() << " is not equal to real input size "
<< args.size();
MS_LOG(DEBUG) << "Dynamic input size " << it->second.size() << " is not equal to real input size " << args.size();
return;
}
for (size_t i = 0; i < args.size(); i++) {
auto abs = it->second.at(i);
@ -2584,6 +2674,9 @@ void ForwardExecutor::SetFeedDynamicInputAbs(const py::object &cell, const py::a
auto shape = abs->BuildShape();
MS_EXCEPTION_IF_NULL(shape);
if (shape->IsDynamic()) {
MS_LOG(DEBUG) << "Set arg " << i << ", id " << GetId(args[i])
<< ", to be dynamic shape; Arg self abs: " << PyObjToValue(args[i])->ToAbstract()->ToString()
<< ", dynamic abs: " << abs->ToString();
dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs[GetId(args[i])] = abs;
}
}
@ -2826,7 +2919,9 @@ inline bool GradExecutor::IsNestedGrad() const {
bool GradExecutor::IsCellObjIdEq(const std::string &l_cell_id, const std::string &r_cell_id) const {
// just compare obj_id, ignore args id
return l_cell_id.compare(0, PTR_LEN, r_cell_id, 0, PTR_LEN) == 0;
auto l_index = l_cell_id.find('_');
auto r_index = r_cell_id.find('_');
return l_cell_id.substr(0, l_index) == r_cell_id.substr(0, r_index);
}
bool GradExecutor::IsBpropGraph(const std::string &cell_id) {
@ -3000,6 +3095,109 @@ void GradExecutor::NewGraphInner(py::object *ret, const py::object &cell, const
}
}
TopCellInfoPtr GradExecutor::ChangeTopCellToDynamicShapeByAuto(const TopCellInfoPtr &top_cell,
const std::vector<ShapeVector> &new_args_shape,
const py::object &cell, const py::args &args) {
MS_EXCEPTION_IF_NULL(top_cell);
// Change args shape
for (size_t i = 0; i < args.size(); ++i) {
top_cell->cell_self_info()->args_shape[i] = std::make_shared<abstract::Shape>(new_args_shape[i]);
if (py::isinstance<tensor::Tensor>(args[i])) {
auto tensor = args[i].cast<tensor::TensorPtr>();
tensor->set_base_shape(top_cell->cell_self_info()->args_shape[i]);
}
forward()->node_abs_map().erase(GetId(args[i]));
}
// Set to feed dynamic map, later shapes can match it
MS_LOG(DEBUG) << "Set dynamic input for auto dynamic shape";
forward()->SetDynamicInput(cell, args);
forward()->SetFeedDynamicInputAbs(cell, args);
// Change cell id
std::string new_cell_id = top_cell->cell_self_info()->cell_self_id;
for (size_t i = 0; i < new_args_shape.size(); ++i) {
new_cell_id += "_" + top_cell->cell_self_info()->args_shape[i]->ToString();
new_cell_id += top_cell->cell_self_info()->args_type[i]->ToString();
}
MS_LOG(DEBUG) << "Change top cell " << top_cell->cell_id() << " to be dynamic " << new_cell_id;
top_cell->set_cell_id(new_cell_id);
top_cell->set_already_run_cell_id(GetAlreadyRunCellId(new_cell_id));
return top_cell;
}
TopCellInfoPtr GradExecutor::ChangeTopCellToDynamicShapeBySetInputs(const TopCellInfoPtr &top_cell,
const std::vector<ShapeVector> &new_args_shape,
const py::object &cell) {
MS_EXCEPTION_IF_NULL(top_cell);
// Change args shape
for (size_t i = 0; i < new_args_shape.size(); ++i) {
top_cell->cell_self_info()->args_shape[i] = std::make_shared<abstract::Shape>(new_args_shape[i]);
}
const auto &feed_dynamic_input = forward()->dynamic_shape_info_ptr()->feed_dynamic_input;
auto it = feed_dynamic_input.find(GetId(cell));
if (it != feed_dynamic_input.end()) {
for (size_t i = 0; i < new_args_shape.size(); i++) {
auto abs = it->second.at(i);
MS_EXCEPTION_IF_NULL(abs);
auto shape = abs->BuildShape();
MS_EXCEPTION_IF_NULL(shape);
if (shape->IsDynamic()) {
const auto &arg_id = top_cell->cell_self_info()->args_id[i];
MS_LOG(DEBUG) << "Set arg " << i << ", id " << arg_id << ", dynamic abs: " << abs->ToString();
forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs[arg_id] = abs;
forward()->node_abs_map().clear();
}
}
}
// Change cell id
std::string new_cell_id = top_cell->cell_self_info()->cell_self_id;
for (size_t i = 0; i < new_args_shape.size(); ++i) {
new_cell_id += "_" + top_cell->cell_self_info()->args_shape[i]->ToString();
new_cell_id += top_cell->cell_self_info()->args_type[i]->ToString();
}
MS_LOG(DEBUG) << "Change top cell " << top_cell->cell_id() << " to be dynamic " << new_cell_id;
top_cell->set_cell_id(new_cell_id);
top_cell->set_already_run_cell_id(GetAlreadyRunCellId(new_cell_id));
return top_cell;
}
TopCellInfoPtr GradExecutor::GetTopCellWithDynamicShape(const py::object &cell, const py::args &args, bool is_auto) {
// Current return nullptr for disable auto dynamic shape feature; Later after a complete test will enable this
if (is_auto && py::isinstance<py::none>(cell)) {
return nullptr;
}
const auto &cell_self_id = GetId(cell);
auto it = std::find_if(top_cell_list_.begin(), top_cell_list_.end(), [&cell_self_id](const TopCellInfoPtr &elem) {
return elem->cell_self_info()->cell_self_id == cell_self_id;
});
if (it != top_cell_list_.end()) {
const auto &elem = *it;
if (elem->dynamic_shape()) {
MS_LOG(DEBUG) << "Elem have is already dynamic shape";
return nullptr;
}
std::vector<ShapeVector> new_args_shape;
FindMatchTopCell(elem, args, &new_args_shape);
// Change top cell to be dynamic
if (new_args_shape.size() == args.size()) {
if (is_auto) {
return ChangeTopCellToDynamicShapeByAuto(elem, new_args_shape, cell, args);
} else {
return ChangeTopCellToDynamicShapeBySetInputs(elem, new_args_shape, cell);
}
}
}
return nullptr;
}
void GradExecutor::CheckPreviousTopCellCanBeDynamicShape(const py::object &cell, const py::args &args) {
if (!grad_flag()) {
return;
}
// In ms_function, new graph run before construct, so top cell create first; After that, set_dynamic_input call
// in construct, here change top cell to dynamic.
GetTopCellWithDynamicShape(cell, args, false);
}
void GradExecutor::MakeNewTopGraph(const string &cell_id, const py::object &cell, const py::args &args,
bool is_topest) {
pipeline::CheckArgsValid(cell, args);
@ -3031,6 +3229,16 @@ void GradExecutor::MakeNewTopGraph(const string &cell_id, const py::object &cell
std::make_shared<TopCellInfo>(is_topest, grad_order_, resource, fg, df_builder, cell_id, already_run_cell_id);
top_cell->set_forward_already_run(true);
top_cell->set_input_args_id(input_args_id);
TopCellInfoPtr top_cell_with_dynamic_shape = GetTopCellWithDynamicShape(cell, args, true);
if (top_cell_with_dynamic_shape != nullptr) {
top_cell->set_cell_id(top_cell_with_dynamic_shape->cell_id());
top_cell->set_already_run_cell_id(top_cell_with_dynamic_shape->already_run_cell_id());
top_cell->set_cell_self_info(top_cell_with_dynamic_shape->cell_self_info());
EraseTopCellFromTopCellList(top_cell_with_dynamic_shape);
MS_LOG(DEBUG) << "Pre top cell and current top cell merged to one top cell with dynamic shape";
} else {
top_cell->SetCellSelfInfoForTopCell(cell, args);
}
top_cell_list_.emplace_back(top_cell);
PushHighOrderGraphStack(top_cell);
set_top_cell(top_cell);
@ -3184,7 +3392,14 @@ void GradExecutor::EndGraphInner(py::object *ret, const py::object &cell, const
MS_LOG(DEBUG) << "Cur top last cell " << cell_id;
PopHighOrderGraphStack();
auto output_node = GetObjNode(out, GetId(out));
auto last_node_abs = output_node->abstract();
MS_EXCEPTION_IF_NULL(output_node);
abstract::AbstractBasePtr last_node_abs = nullptr;
if (output_node->abstract() == nullptr) {
last_node_abs = PyObjToValue(out)->ToAbstract()->Broaden();
} else {
last_node_abs = output_node->abstract();
}
MS_EXCEPTION_IF_NULL(last_node_abs);
// Set last output abstract and will be used for sens
top_cell()->set_last_output_abs(last_node_abs);
auto sens_value = GetSensValueForDynamicShapeOutput(out, output_node);
@ -3559,8 +3774,9 @@ FuncGraphPtr GradExecutor::GetBpropGraph(const prim::GradOperationPtr &grad, con
ss << "grad{" << arg_size << "}";
bprop_graph->set_flag(FUNC_GRAPH_FLAG_CORE, true);
bprop_graph->debug_info()->set_name(ss.str());
// Get the parameters items and add the value to args_abs
if (grad->sens_param() && top_cell()->last_output_abs() != nullptr) {
// Get the parameters items and add the value to args_spec
if (grad->sens_param()) {
MS_EXCEPTION_IF_NULL(top_cell()->last_output_abs());
auto shape = top_cell()->last_output_abs()->BuildShape();
MS_EXCEPTION_IF_NULL(shape);
if (shape->IsDynamic()) {
@ -3916,7 +4132,12 @@ void GradExecutor::GradMsFunctionInner(const std::string &phase, const py::objec
// Identity op info for current running ms_func graph.
OpExecInfoPtr op_exec_info = std::make_shared<OpExecInfo>();
op_exec_info->op_name = phase;
op_exec_info->abstract = actual_out_v->ToAbstract();
auto it = forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs.find(GetId(actual_out));
if (it != forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs.end()) {
op_exec_info->abstract = it->second;
} else {
op_exec_info->abstract = actual_out_v->ToAbstract();
}
RecordGradOpInfo(op_exec_info);
MS_LOG(DEBUG) << "ms_function cnode op info: " << op_exec_info->op_info;
@ -3943,7 +4164,9 @@ void GradExecutor::GradMsFunctionInner(const std::string &phase, const py::objec
new_ms_func_graph->set_output(new_make_tuple->input(1));
// Make Adjoint for grad graph
MakeAdjointForMsFunction(new_ms_func_graph, new_grad_graph, actual_out, args, actual_out_v);
const auto &ms_function_cnode =
MakeAdjointForMsFunction(new_ms_func_graph, new_grad_graph, actual_out, args, actual_out_v);
ms_function_cnode->set_abstract(new_ms_func_graph->output()->abstract()->Broaden());
}
py::object GradExecutor::GradMsFunction(const py::object &out, const py::args &args) {
@ -3964,7 +4187,7 @@ py::object GradExecutor::GradMsFunction(const py::object &out, const py::args &a
ret = tuple_out[0];
}
// Save dynamic shape info if output tensors of forward graph have dynamic shapes
SaveDynShapeAbsForMsFunction(ret, ms_func_graph);
SaveDynShapeAbsForMsFunction(args, out, ms_func_graph);
// Make Adjoint for grad graph of ms_function.
if (!grad_flag_) {
MS_LOG(DEBUG) << "Only run forward infer computation, no need to construct grad graph.";
@ -4046,7 +4269,10 @@ void PynativeExecutor::set_graph_phase(const std::string &graph_phase) {
}
void PynativeExecutor::SetDynamicInput(const py::object &cell, const py::args &args) {
MS_LOG(DEBUG) << "Set dynamic input for feed mode from cell";
forward_executor()->SetDynamicInput(cell, args);
// After set input, check previous top cell can be make to dynamic shape
grad_executor()->CheckPreviousTopCellCanBeDynamicShape(cell, args);
}
py::object PynativeExecutor::GetDynamicInput(const py::object &actual_input) const {
@ -4090,7 +4316,8 @@ py::object PynativeExecutor::Run(const py::object &cell, const py::object &sens_
return ret;
}
void PynativeExecutor::ClearCell(const std::string &cell_id) {
void PynativeExecutor::ClearCell(const py::object &cell) {
const auto &cell_id = GetId(cell);
MS_LOG(DEBUG) << "Clear cell res, cell id " << cell_id;
grad_executor()->ClearCellRes(cell_id);
}

39
mindspore/ccsrc/pipeline/pynative/pynative_execute.h
View File

@ -67,6 +67,23 @@ struct DynamicShapeInfo {
};
using DynamicShapeInfoPtr = std::shared_ptr<DynamicShapeInfo>;
struct CellSelfInfo {
CellSelfInfo() = default;
~CellSelfInfo() = default;
CellSelfInfo(std::string cell_self_id, std::vector<std::string> args_id, std::vector<abstract::ShapePtr> args_shape,
std::vector<TypePtr> args_type)
: cell_self_id(std::move(cell_self_id)),
args_id(std::move(args_id)),
args_shape(std::move(args_shape)),
args_type(std::move(args_type)) {}
std::string cell_self_id;
std::vector<std::string> args_id;
std::vector<abstract::ShapePtr> args_shape;
std::vector<TypePtr> args_type;
};
using CellSelfInfoPtr = std::shared_ptr<CellSelfInfo>;
class TopCellInfo {
public:
TopCellInfo() = default;
@ -113,7 +130,9 @@ class TopCellInfo {
size_t op_num() const { return op_num_; }
void set_op_num(size_t op_num) { op_num_ = op_num; }
const std::string &cell_id() const { return cell_id_; }
void set_cell_id(const std::string &cell_id) { cell_id_ = cell_id; }
const std::string &already_run_cell_id() const { return already_run_cell_id_; }
void set_already_run_cell_id(const std::string &already_run_cell_id) { already_run_cell_id_ = already_run_cell_id; }
const std::string &input_args_id() const { return input_args_id_; }
void set_input_args_id(const std::string &input_args_id) { input_args_id_ = input_args_id; }
std::string &all_op_info() { return all_op_info_; }
@ -121,6 +140,9 @@ class TopCellInfo {
void set_grad_operation(const std::string &grad_operation) { grad_operation_ = grad_operation; }
const abstract::AbstractBasePtr &last_output_abs() const { return last_output_abs_; }
void set_last_output_abs(const abstract::AbstractBasePtr &last_output_abs) { last_output_abs_ = last_output_abs; }
CellSelfInfoPtr cell_self_info() const { return cell_self_info_; }
void set_cell_self_info(const CellSelfInfoPtr &cell_self_info) { cell_self_info_ = cell_self_info; }
void SetCellSelfInfoForTopCell(const py::object &cell, const py::args &args);
mindspore::HashSet<std::string> &sub_cell_list() { return sub_cell_list_; }
std::set<std::string> &forward_op_output_id() { return forward_op_output_id_; }
bool IsSubCell(const std::string &cell_id) const;
@ -165,6 +187,7 @@ class TopCellInfo {
std::string all_op_info_;
std::string grad_operation_;
abstract::AbstractBasePtr last_output_abs_;
CellSelfInfoPtr cell_self_info_;
OrderedMap<FuncGraphPtr, GraphInfoPtr> graph_info_map_;
mindspore::HashSet<std::string> sub_cell_list_;
// Record `register hook` or `remove hook` function has been called by sub cell
@ -246,10 +269,10 @@ class GradExecutor {
py::object GradMsFunction(const py::object &out, const py::args &args);
void GradMsFunctionInner(const std::string &phase, const py::object &out, const py::args &args,
const FuncGraphPtr &ms_func_graph, const FuncGraphPtr &grad_graph);
void SaveDynShapeAbsForMsFunction(const py::object &forward_out, const FuncGraphPtr &ms_func_graph);
void SaveDynShapeAbsForMsFunction(const py::args &args, const py::object &out, const FuncGraphPtr &ms_func_graph);
void UpdateMsFunctionForwardTensors(const OpExecInfoPtr &op_exec_info, const ValuePtr &new_forward_value);
void MakeAdjointForMsFunction(const FuncGraphPtr &ms_func_graph, const FuncGraphPtr &grad_graph,
const py::object &actual_out, const py::args &args, const ValuePtr &actual_out_v);
CNodePtr MakeAdjointForMsFunction(const FuncGraphPtr &ms_func_graph, const FuncGraphPtr &grad_graph,
const py::object &actual_out, const py::args &args, const ValuePtr &actual_out_v);
void MakeCNodeForMsFunction(const FuncGraphPtr &ms_func_graph, const py::args &args, ValuePtrList *input_values,
CNodePtr *ms_function_cnode);
void SaveOutputNodeMap(const std::string &obj_id, const py::object &out_real, const CNodePtr &cnode);
@ -258,6 +281,14 @@ class GradExecutor {
void UpdateForwardTensorInfoInBpropGraph(const string &op_info, const ValuePtr &op_out);
void SaveForwardTensorInfoInBpropGraph(const pipeline::ResourcePtr &resource) const;
py::object CheckGraph(const py::object &cell, const py::args &args);
TopCellInfoPtr ChangeTopCellToDynamicShapeByAuto(const TopCellInfoPtr &top_cell,
const std::vector<ShapeVector> &new_args_shape,
const py::object &cell, const py::args &args);
TopCellInfoPtr ChangeTopCellToDynamicShapeBySetInputs(const TopCellInfoPtr &top_cell,
const std::vector<ShapeVector> &new_args_shape,
const py::object &cell);
TopCellInfoPtr GetTopCellWithDynamicShape(const py::object &cell, const py::args &args, bool is_auto);
void CheckPreviousTopCellCanBeDynamicShape(const py::object &cell, const py::args &args);
void RunGradGraph(py::object *ret, const py::object &cell, const py::object &sens_param, const py::tuple &args);
py::object CheckAlreadyRun(const prim::GradOperationPtr &grad, const py::object &cell, const py::args &args);
void EraseTopCellFromTopCellList(const TopCellInfoPtr &top_cell);
@ -472,7 +503,7 @@ class PynativeExecutor : public std::enable_shared_from_this<PynativeExecutor> {
// Used by graph clean
// Cell destruct will call
void ClearCell(const std::string &cell_id);
void ClearCell(const py::object &cell);
void ClearGrad(const py::object &cell, const py::args &args);
// Abnormal existed
void ClearRes();

1
mindspore/ccsrc/plugin/device/cpu/kernel/masked_select_cpu_kernel.cc
View File

@ -27,6 +27,7 @@ constexpr size_t kMaskedSelectOutputsNum = 1;
void MaskedSelectCpuKernelMod::InitKernel(const CNodePtr &kernel_node) {
MS_EXCEPTION_IF_NULL(kernel_node);
tensor_size_ = 1;
kernel_name_ = common::AnfAlgo::GetCNodeName(kernel_node);
input_shape_a_ = AnfAlgo::GetInputDeviceShape(kernel_node, 0);
input_shape_b_ = AnfAlgo::GetInputDeviceShape(kernel_node, 1);

1
mindspore/ccsrc/plugin/device/cpu/kernel/masked_select_grad_cpu_kernel.cc
View File

@ -30,6 +30,7 @@ constexpr size_t kIndexGrad = 2;
void MaskedSelectGradCpuKernelMod::InitKernel(const CNodePtr &kernel_node) {
MS_EXCEPTION_IF_NULL(kernel_node);
tensor_size_ = 1;
kernel_name_ = common::AnfAlgo::GetCNodeName(kernel_node);
input_shape_a_ = AnfAlgo::GetInputDeviceShape(kernel_node, kIndexInput);
input_shape_b_ = AnfAlgo::GetInputDeviceShape(kernel_node, kIndexMask);

8
mindspore/core/ir/cell.cc
View File

@ -20,10 +20,18 @@
#include "utils/ms_utils.h"
namespace mindspore {
static std::string MakeId() {
// Use atomic to make id generator thread safe.
static std::atomic<uint64_t> last_id{1};
return "C" + std::to_string(last_id.fetch_add(1, std::memory_order_relaxed));
}
using mindspore::abstract::AbstractFunction;
abstract::AbstractBasePtr Cell::ToAbstract() { return nullptr; }
Cell::Cell(const std::string &name) : Named(name), id_(MakeId()) {}
bool Cell::operator==(const Value &other) const {
if (other.isa<Cell>()) {
auto other_prim = static_cast<const Cell &>(other);

8
mindspore/core/ir/cell.h
View File

@ -37,13 +37,18 @@ class MS_CORE_API Cell final : public Named {
/// \brief Constructor.
///
/// \param[in] name The name of Cell.
explicit Cell(const std::string &name) : Named(name) {}
explicit Cell(const std::string &name);
MS_DECLARE_PARENT(Cell, Named);
abstract::AbstractBasePtr ToAbstract() override;
std::string ToString() const override;
/// \brief Get the id of this Cell.
///
/// \return The id of this Cell.
string id() const { return id_; }
/// \brief Get information about all attributes.
///
/// \return Details of all attributes.
@ -110,6 +115,7 @@ class MS_CORE_API Cell final : public Named {
~Cell() override = default;
private:
string id_;
mindspore::HashMap<std::string, ValuePtr> attrs_;
enum MixedPrecisionType mixed_type_ { kNotSet };
};

7
mindspore/python/mindspore/common/api.py
View File

@ -403,6 +403,7 @@ class _MindsporeFunctionExecutor:
self.input_signature.append(args_list[-1])
Validator.check_dynamic_shape(self.input_signature, args_list)
compile_args = tuple(self.input_signature)
_pynative_executor.set_dynamic_input(self.obj, *compile_args)
return compile_args
@ -767,17 +768,17 @@ class _PynativeExecutor:
"""
self._executor.grad_net(grad, obj, weights, grad_position, *args, *(kwargs.values()))
def del_cell(self, cell_id=""):
def del_cell(self, obj):
"""
Clean resource for cell.
Args:
cell_id (str): The ID of cell object.
obj (Function/Cell): The function or cell instance.
Return:
None.
"""
self._executor.clear_cell(cell_id)
self._executor.clear_cell(obj)
def clear_res(self):
"""

7
mindspore/python/mindspore/nn/cell.py
View File

@ -325,7 +325,7 @@ class Cell(Cell_):
def __del__(self):
if context.get_context is not None and context._get_mode() == context.PYNATIVE_MODE:
_pynative_executor.del_cell(str(id(self)))
_pynative_executor.del_cell(self)
# while deepcopy a cell instance, the copied cell instance can't be added to cells_compile_cache
# here using pop(id(self), None) to avoid KeyError exception
@ -588,10 +588,8 @@ class Cell(Cell_):
if self.requires_grad:
_pynative_executor.set_grad_flag(True)
# PyNative feed dynamic shape inputs
if self._dynamic_shape_inputs is not None:
self._check_compile_dynamic_shape(*args)
_pynative_executor.set_dynamic_input(self, *self._dynamic_shape_inputs)
try:
_pynative_executor.new_graph(self, *args, **kwargs)
@ -900,6 +898,9 @@ class Cell(Cell_):
for ele in self._dynamic_shape_inputs:
if isinstance(ele, (str, int, dict)):
raise TypeError(f"For element in 'set_inputs', the type must be Tensor, but got {type(ele)}.")
if context._get_mode() == context.PYNATIVE_MODE:
_pynative_executor.set_dynamic_input(self, *self._dynamic_shape_inputs)
def get_inputs(self):
"""

228
tests/st/pynative/dynamic_shape/test_pynative_auto_dynamic_shape.py Normal file
View File

@ -0,0 +1,228 @@
# Copyright 2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import platform
import numpy as np
import pytest
import mindspore as ms
from mindspore import nn
from mindspore import ops
from mindspore import context, Tensor
context.set_context(mode=context.PYNATIVE_MODE)
class NetInner(nn.Cell):
def __init__(self):
super(NetInner, self).__init__()
self.log = ops.Log()
self.exp = ops.Exp()
self.addn = ops.AddN()
self.relu = nn.ReLU()
def construct(self, x, y):
x = self.addn((x, y))
x = self.log(x)
x = self.exp(x)
x = self.relu(x)
x = self.addn((x, y))
return x
class Net(nn.Cell):
def __init__(self):
super(Net, self).__init__()
self.log = ops.Log()
self.exp = ops.Exp()
self.addn = ops.AddN()
self.relu = nn.ReLU()
self.inner = NetInner()
def construct(self, x, y):
x = self.addn((x, y))
x = self.inner(x, y)
x = self.log(x)
x = self.exp(x)
x = self.relu(x)
return x
class CmpNetInner(nn.Cell):
def __init__(self):
super(CmpNetInner, self).__init__()
self.log = ops.Log()
self.exp = ops.Exp()
self.addn = ops.AddN()
self.relu = nn.ReLU()
def construct(self, x, y):
x = self.addn((x, y))
x = self.log(x)
x = self.exp(x)
x = self.relu(x)
x = self.addn((x, y))
return x
class CmpNet(nn.Cell):
def __init__(self):
super(CmpNet, self).__init__()
self.log = ops.Log()
self.exp = ops.Exp()
self.addn = ops.AddN()
self.relu = nn.ReLU()
self.inner = CmpNetInner()
def construct(self, x, y):
x = self.addn((x, y))
x = self.inner(x, y)
x = self.log(x)
x = self.exp(x)
x = self.relu(x)
return x
@pytest.mark.level0
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.env_onecard
def test_pynative_auto_dynamic_shape_with_three_static_shape():
"""
Feature: PyNative auto dynamic shape.
Description: The static shape is automatically converted to a dynamic shape.
Expectation: The calculation result is correct.
"""
if platform.system() == 'Windows':
return
net = Net()
grad_op = ops.GradOperation(get_all=True, get_by_list=False, sens_param=True)
# run first shape
input_x = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 2)
input_y = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 5)
out = net(input_x, input_y)
_ = grad_op(net)(input_x, input_y, out)
# run second shape
input_x2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 2)
input_y2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 5)
out = net(input_x2, input_y2)
_ = grad_op(net)(input_x2, input_y2, out)
# run third shape
input_x3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 2)
input_y3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 5)
out = net(input_x3, input_y3)
grad = grad_op(net)(input_x3, input_y3, out)
cmp_net = CmpNet()
cmp_out = cmp_net(input_x3, input_y3)
cmp_grad = grad_op(cmp_net)(input_x3, input_y3, cmp_out)
assert np.allclose(grad[0].asnumpy(), cmp_grad[0].asnumpy(), 0.00001, 0.00001)
assert np.allclose(grad[1].asnumpy(), cmp_grad[1].asnumpy(), 0.00001, 0.00001)
@pytest.mark.level0
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.env_onecard
def test_pynative_auto_dynamic_shape_mixing_static_shape_and_dynamic_shape_1():
"""
Feature: PyNative auto dynamic shape.
Description: Mixing static shape and dynamic shape.
Expectation: The calculation result is correct.
"""
if platform.system() == 'Windows':
return
net = Net()
grad_op = ops.GradOperation(get_all=True, get_by_list=False, sens_param=True)
# run first shape
input_x = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 2)
input_y = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 5)
out = net(input_x, input_y)
_ = grad_op(net)(input_x, input_y, out)
# run second shape
input_x2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 2)
input_y2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 5)
net.set_inputs(Tensor(shape=[2, 3, 6, None], dtype=ms.float32),
Tensor(shape=[2, 3, None, None], dtype=ms.float32))
out = net(input_x2, input_y2)
_ = grad_op(net)(input_x2, input_y2, out)
# run third shape
input_x3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 2)
input_y3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 5)
out = net(input_x3, input_y3)
grad = grad_op(net)(input_x3, input_y3, out)
cmp_net = CmpNet()
cmp_out = cmp_net(input_x3, input_y3)
cmp_grad = grad_op(cmp_net)(input_x3, input_y3, cmp_out)
assert np.allclose(grad[0].asnumpy(), cmp_grad[0].asnumpy(), 0.00001, 0.00001)
assert np.allclose(grad[1].asnumpy(), cmp_grad[1].asnumpy(), 0.00001, 0.00001)
@pytest.mark.level0
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.env_onecard
def test_pynative_auto_dynamic_shape_mixing_static_shape_and_dynamic_shape_2():
"""
Feature: PyNative auto dynamic shape.
Description: Mixing static shape and dynamic shape.
Expectation: The calculation result is correct.
"""
if platform.system() == 'Windows':
return
net = Net()
grad_op = ops.GradOperation(get_all=True, get_by_list=False, sens_param=True)
# run first shape
input_x = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 2)
input_y = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 5)
net.set_inputs(Tensor(shape=[2, 3, 6, None], dtype=ms.float32),
Tensor(shape=[2, 3, None, None], dtype=ms.float32))
out = net(input_x, input_y)
_ = grad_op(net)(input_x, input_y, out)
# run second shape
input_x2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 2)
input_y2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 5)
out = net(input_x2, input_y2)
_ = grad_op(net)(input_x2, input_y2, out)
# run third shape
input_x3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 2)
input_y3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 5)
out = net(input_x3, input_y3)
grad = grad_op(net)(input_x3, input_y3, out)
cmp_net = CmpNet()
cmp_out = cmp_net(input_x3, input_y3)
cmp_grad = grad_op(cmp_net)(input_x3, input_y3, cmp_out)
assert np.allclose(grad[0].asnumpy(), cmp_grad[0].asnumpy(), 0.00001, 0.00001)
assert np.allclose(grad[1].asnumpy(), cmp_grad[1].asnumpy(), 0.00001, 0.00001)

202
tests/st/pynative/ms_function/test_auto_dynamic_shape_with_ms_function.py Normal file
View File

@ -0,0 +1,202 @@
# Copyright 2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import platform
import numpy as np
import pytest
from mindspore import nn
from mindspore import ops
from mindspore import context, Tensor
from mindspore import ms_function
context.set_context(mode=context.PYNATIVE_MODE)
class NetInner(nn.Cell):
def __init__(self):
super(NetInner, self).__init__()
self.log = ops.Log()
self.exp = ops.Exp()
self.addn = ops.AddN()
self.relu = nn.ReLU()
@ms_function
def construct(self, x, y):
x = self.addn((x, y))
x = self.log(x)
x = self.exp(x)
x = self.relu(x)
x = self.addn((x, y))
return x
class Net(nn.Cell):
def __init__(self):
super(Net, self).__init__()
self.log = ops.Log()
self.exp = ops.Exp()
self.addn = ops.AddN()
self.relu = nn.ReLU()
self.inner = NetInner()
def construct(self, x, y):
x = self.addn((x, y))
x = self.inner(x, y)
x = self.log(x)
x = self.exp(x)
x = self.relu(x)
return x
class NetOuter(nn.Cell):
def __init__(self):
super(NetOuter, self).__init__()
self.log = ops.Log()
self.exp = ops.Exp()
self.addn = ops.AddN()
self.relu = nn.ReLU()
self.inner = NetInner()
@ms_function
def construct(self, x, y):
x = self.addn((x, y))
x = self.inner(x, y)
x = self.log(x)
x = self.exp(x)
x = self.relu(x)
return x
class CmpNetInner(nn.Cell):
def __init__(self):
super(CmpNetInner, self).__init__()
self.log = ops.Log()
self.exp = ops.Exp()
self.addn = ops.AddN()
self.relu = nn.ReLU()
@ms_function
def construct(self, x, y):
x = self.addn((x, y))
x = self.log(x)
x = self.exp(x)
x = self.relu(x)
x = self.addn((x, y))
return x
class CmpNet(nn.Cell):
def __init__(self):
super(CmpNet, self).__init__()
self.log = ops.Log()
self.exp = ops.Exp()
self.addn = ops.AddN()
self.relu = nn.ReLU()
self.inner = CmpNetInner()
def construct(self, x, y):
x = self.addn((x, y))
x = self.inner(x, y)
x = self.log(x)
x = self.exp(x)
x = self.relu(x)
return x
@pytest.mark.level0
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.env_onecard
def test_pynative_auto_dynamic_shape_with_inner_ms_function():
"""
Feature: PyNative auto dynamic shape.
Description: The static shape is automatically converted to a dynamic shape.
Expectation: The calculation result is correct.
"""
if platform.system() == 'Windows':
return
net = Net()
grad_op = ops.GradOperation(get_all=True, get_by_list=False, sens_param=True)
# run first shape
input_x = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 2)
input_y = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 5)
out = net(input_x, input_y)
_ = grad_op(net)(input_x, input_y, out)
# run second shape
input_x2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 2)
input_y2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 5)
out = net(input_x2, input_y2)
_ = grad_op(net)(input_x2, input_y2, out)
# run third shape
input_x3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 2)
input_y3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 5)
out = net(input_x3, input_y3)
grad = grad_op(net)(input_x3, input_y3, out)
cmp_net = CmpNet()
cmp_out = cmp_net(input_x3, input_y3)
cmp_grad = grad_op(cmp_net)(input_x3, input_y3, cmp_out)
assert np.allclose(grad[0].asnumpy(), cmp_grad[0].asnumpy(), 0.00001, 0.00001)
assert np.allclose(grad[1].asnumpy(), cmp_grad[1].asnumpy(), 0.00001, 0.00001)
@pytest.mark.level0
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.env_onecard
def test_pynative_auto_dynamic_shape_with_outer_ms_function():
"""
Feature: PyNative auto dynamic shape.
Description: The static shape is automatically converted to a dynamic shape.
Expectation: The calculation result is correct.
"""
if platform.system() == 'Windows':
return
net = NetOuter()
grad_op = ops.GradOperation(get_all=True, get_by_list=False, sens_param=True)
# run first shape
input_x = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 2)
input_y = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 5)
out = net(input_x, input_y)
_ = grad_op(net)(input_x, input_y, out)
# run second shape
input_x2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 2)
input_y2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 5)
out = net(input_x2, input_y2)
_ = grad_op(net)(input_x2, input_y2, out)
# run third shape
input_x3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 2)
input_y3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 5)
out = net(input_x3, input_y3)
grad = grad_op(net)(input_x3, input_y3, out)
cmp_net = CmpNet()
cmp_out = cmp_net(input_x3, input_y3)
cmp_grad = grad_op(cmp_net)(input_x3, input_y3, cmp_out)
assert np.allclose(grad[0].asnumpy(), cmp_grad[0].asnumpy(), 0.00001, 0.00001)
assert np.allclose(grad[1].asnumpy(), cmp_grad[1].asnumpy(), 0.00001, 0.00001)