From 5d2d3f5c131b20f4f51fa97dd2a65351bf3654e2 Mon Sep 17 00:00:00 2001 From: zjun Date: Thu, 8 Sep 2022 10:22:57 +0800 Subject: [PATCH] Open auto dynamic shape Signed-off-by: zjun opitmize logic Signed-off-by: zjun fix Signed-off-by: zjun add dynamic rank Signed-off-by: zjun --- .../ccsrc/frontend/optimizer/ad/kpynative.cc | 2 +- .../ccsrc/pipeline/pynative/dynamic_shape.cc | 106 +++++++----------- .../ccsrc/pipeline/pynative/dynamic_shape.h | 2 +- .../pipeline/pynative/forward/forward.cc | 2 +- .../ccsrc/pipeline/pynative/grad/grad.cc | 67 ++++++----- mindspore/ccsrc/pipeline/pynative/grad/grad.h | 2 +- .../ccsrc/pipeline/pynative/grad/top_cell.cc | 44 ++++---- .../ccsrc/pipeline/pynative/grad/top_cell.h | 34 +++--- .../mindspore/_extends/builtin_operations.py | 24 ++++ .../ops/dynamic_shape/test_dynamic_getitem.py | 6 +- .../ops/dynamic_shape/test_dynamic_setitem.py | 8 +- .../test_pynative_auto_dynamic_shape.py | 8 +- 12 files changed, 157 insertions(+), 148 deletions(-) diff --git a/mindspore/ccsrc/frontend/optimizer/ad/kpynative.cc b/mindspore/ccsrc/frontend/optimizer/ad/kpynative.cc index ddd252275c8..579683ac2c5 100644 --- a/mindspore/ccsrc/frontend/optimizer/ad/kpynative.cc +++ b/mindspore/ccsrc/frontend/optimizer/ad/kpynative.cc @@ -1096,7 +1096,7 @@ void KPynativeCellImpl::SetSensAndWeights(const AnfNodePtrList &weights, bool ha if (has_sens_arg) { auto sens_param = tape_->add_parameter(); sens_param->debug_info()->set_name("sens"); - sens_param->set_abstract(last_node_adjoint_iter->second->out()->ToAbstract()->Broaden()); + sens_param->set_abstract(last_node_->abstract()->Broaden()); // Set dout of last node to sens; last_node_adjoint_iter->second->AccumulateDout(sens_param); } else { diff --git a/mindspore/ccsrc/pipeline/pynative/dynamic_shape.cc b/mindspore/ccsrc/pipeline/pynative/dynamic_shape.cc index d8544cccb30..ae8126ef2eb 100644 --- a/mindspore/ccsrc/pipeline/pynative/dynamic_shape.cc +++ b/mindspore/ccsrc/pipeline/pynative/dynamic_shape.cc @@ -22,6 +22,8 @@ namespace mindspore { namespace pynative { const char kSensInfo[] = "SensInfo"; +static const ShapeValueDType UNKNOWN_DIM = -1; +static const ShapeValueDType UNKNOWN_RANK = -2; ShapeVector DynamicShape::GetTensorShape(const ValuePtr &v) const { MS_EXCEPTION_IF_NULL(v); @@ -134,6 +136,10 @@ void DynamicShape::SaveDynShapeAbsForMsFunction(const py::args &args, const py:: void DynamicShape::SaveOutputDynamicShape(const FrontendOpRunInfoPtr &op_run_info, const ValuePtr &v) { MS_EXCEPTION_IF_NULL(op_run_info); MS_EXCEPTION_IF_NULL(v); + // Do not use GetNext out abstract, top cell will change to dynamic shape by different tensor shape automaticity + if (op_run_info->base_op_run_info.op_name == kGetNextOpName) { + return; + } // Save dynamic abs if (op_run_info->base_op_run_info.has_dynamic_output) { SaveIdWithDynamicAbstract(v, op_run_info->base_op_run_info.abstract); @@ -151,7 +157,7 @@ void DynamicShape::SetDynamicInput(const py::object &cell, const py::args &args) } } -void DynamicShape::SetFeedDynamicInputAbs(const py::object &cell, const py::args &args, bool is_auto) { +void DynamicShape::SetFeedDynamicInputAbs(const py::object &cell, const py::args &args) { if (!HasFeedDynamicInput()) { return; } @@ -164,7 +170,6 @@ void DynamicShape::SetFeedDynamicInputAbs(const py::object &cell, const py::args MS_LOG(DEBUG) << "Dynamic input size " << it->second.size() << " is not equal to real input size " << args.size(); return; } - bool id_changed = false; for (size_t i = 0; i < args.size(); i++) { auto abs = it->second.at(i); MS_EXCEPTION_IF_NULL(abs); @@ -172,17 +177,13 @@ void DynamicShape::SetFeedDynamicInputAbs(const py::object &cell, const py::args MS_EXCEPTION_IF_NULL(shape); if (shape->IsDynamic()) { const auto &arg_id = PyNativeAlgo::PyParser::GetIdByPyObj(args[i]); - MS_LOG(DEBUG) << "Set arg " << i << ", id " << arg_id << " to be dynamic shape; Arg self abs: " + MS_LOG(DEBUG) << "Set cur arg " << i << ", id " << arg_id << " to be dynamic shape; Arg self abs: " << PyNativeAlgo::DataConvert::PyObjToValue(args[i])->ToAbstract()->Broaden()->ToString() << ", dynamic abs: " << abs->ToString(); id_with_dynamic_abs_[arg_id] = abs; PyNativeAlgo::Common::GetPyNativeExecutor()->forward_executor()->EraseFromNodeAbsMap(arg_id); - id_changed = true; } } - if (id_changed && !is_auto) { - CheckPreviousTopCellCanBeDynamicShape(cell, args); - } } py::object DynamicShape::GetDynamicInput(const py::object &actual_input) const { @@ -311,23 +312,18 @@ TopCellInfoPtr DynamicShape::ChangeTopCellToDynamicShapeBySetInputs(const TopCel const py::object &cell) { MS_EXCEPTION_IF_NULL(top_cell); // Change args shape + const auto it = feed_dynamic_input_.find(PyNativeAlgo::PyParser::GetIdByPyObj(cell)); + if (it == feed_dynamic_input_.end()) { + return nullptr; + } for (size_t i = 0; i < new_args_shape.size(); ++i) { top_cell->cell_self_info()->args_shape[i] = std::make_shared(new_args_shape[i]); - } - auto it = feed_dynamic_input_.find(PyNativeAlgo::PyParser::GetIdByPyObj(cell)); - if (it != feed_dynamic_input_.end()) { - for (size_t i = 0; i < new_args_shape.size(); i++) { - auto abs = it->second.at(i); - MS_EXCEPTION_IF_NULL(abs); - auto shape = abs->BuildShape(); - MS_EXCEPTION_IF_NULL(shape); - if (shape->IsDynamic()) { - const auto &arg_id = top_cell->cell_self_info()->args_id[i]; - MS_LOG(DEBUG) << "Set arg " << i << ", id " << arg_id << ", dynamic abs: " << abs->ToString(); - id_with_dynamic_abs_[arg_id] = abs; - PyNativeAlgo::Common::GetPyNativeExecutor()->forward_executor()->EraseFromNodeAbsMap(arg_id); - } - } + const auto &arg_id = top_cell->cell_self_info()->args_id[i]; + it->second.at(i)->set_shape(top_cell->cell_self_info()->args_shape[i]); + MS_LOG(DEBUG) << "Change cur top cell arg " << i << ", id " << arg_id + << ", dynamic abs: " << it->second.at(i)->ToString(); + id_with_dynamic_abs_[arg_id] = it->second.at(i); + PyNativeAlgo::Common::GetPyNativeExecutor()->forward_executor()->EraseFromNodeAbsMap(arg_id); } top_cell->ChangeTopCellInfo(new_args_shape.size()); return top_cell; @@ -350,7 +346,6 @@ TopCellInfoPtr DynamicShape::ChangeTopCellToDynamicShapeByAuto(const TopCellInfo // Set to feed dynamic map, later shapes can match it MS_LOG(DEBUG) << "Set dynamic input for auto dynamic shape"; SetDynamicInput(cell, args); - SetFeedDynamicInputAbs(cell, args, true); top_cell->ChangeTopCellInfo(new_args_shape.size()); return top_cell; } @@ -395,30 +390,23 @@ void DynamicShape::UpdateTopCellId(const py::args &args) const { } TopCellInfoPtr DynamicShape::GetTopCellWithDynamicShape(const py::object &cell, const py::args &args, bool is_auto) { - // Current return nullptr for disable auto dynamic shape feature; Later after a complete test will enable this - if (is_auto && !py::isinstance(cell)) { - return nullptr; - } const auto &cell_self_id = PyNativeAlgo::PyParser::GetIdByPyObj(cell); - const auto &top_cell_list = PyNativeAlgo::Common::GetPyNativeExecutor()->grad_executor()->top_cell_list(); - const auto it = std::find_if(top_cell_list.begin(), top_cell_list.end(), [&cell_self_id](const TopCellInfoPtr &elem) { - return elem->cell_self_info() != nullptr && elem->cell_self_info()->cell_self_id == cell_self_id; - }); - if (it != top_cell_list.end()) { - const auto &elem = *it; - if (elem->dynamic_shape()) { - MS_LOG(DEBUG) << "Elem has already dynamic shape"; - return nullptr; - } + const auto grad_executor = PyNativeAlgo::Common::GetPyNativeExecutor()->grad_executor(); + size_t grad_order = grad_executor->grad_order(); + std::vector match_top_cell_list; + std::copy_if(grad_executor->top_cell_list().begin(), grad_executor->top_cell_list().end(), + std::back_inserter(match_top_cell_list), [&cell_self_id, grad_order](const TopCellInfoPtr &elem) { + return elem->cell_self_info() != nullptr && elem->cell_self_info()->cell_self_id == cell_self_id && + grad_order == elem->grad_order(); + }); + for (const auto &it : match_top_cell_list) { std::vector new_args_shape; - FindMatchTopCell(elem, args, &new_args_shape); + FindMatchTopCell(it, args, &new_args_shape); // Change top cell to be dynamic - if (new_args_shape.size() == args.size()) { - if (is_auto) { - return ChangeTopCellToDynamicShapeByAuto(elem, new_args_shape, cell, args); - } else { - return ChangeTopCellToDynamicShapeBySetInputs(elem, new_args_shape, cell); - } + if (is_auto) { + return ChangeTopCellToDynamicShapeByAuto(it, new_args_shape, cell, args); + } else { + return ChangeTopCellToDynamicShapeBySetInputs(it, new_args_shape, cell); } } UpdateTopCellId(args); @@ -432,7 +420,7 @@ void DynamicShape::CheckPreviousTopCellCanBeDynamicShape(const py::object &cell, // In ms_function, new graph run before construct, so top cell create first; After that, set_dynamic_input call // in construct, here change top cell to dynamic. if (GetTopCellWithDynamicShape(cell, args, false) != nullptr) { - MS_LOG(DEBUG) << "Convert ms_function top cell to dynamic shape."; + MS_LOG(DEBUG) << "Convert cur top cell to dynamic shape."; } } @@ -476,29 +464,21 @@ void DynamicShape::FindMatchTopCell(const TopCellInfoPtr &top_cell, const py::ar } // Check shape const auto &cur_shape = GetShapeFromAbstract(cur_value_abs)->shape(); - auto elem_shape = top_cell->cell_self_info()->args_shape[i]->shape(); + const auto elem_shape = top_cell->cell_self_info()->args_shape[i]->shape(); + ShapeVector new_shape; + // Rank dynamic if (cur_shape.size() != elem_shape.size()) { MS_LOG(DEBUG) << "The " << i << "th args shape size is not the same, cur is " << cur_shape.size() - << " and the elem is " << elem_shape.size(); - return; + << " and the elem is " << elem_shape.size() << ", change shape to dynamic rank"; + new_shape.emplace_back(UNKNOWN_RANK); + continue; } - ShapeVector new_shape; + // Shape dynamic for (size_t j = 0; j < cur_shape.size(); ++j) { - if (cur_shape[j] == elem_shape[j]) { - (void)new_shape.emplace_back(cur_shape[j]); - } else { - (void)new_shape.emplace_back(-1); - } - } - // All shape can not be -1, and all shape can not be actual. - bool is_any_unknown = std::any_of(new_shape.begin(), new_shape.end(), [](int64_t s) { return s == -1; }); - bool is_any_actual = std::any_of(new_shape.begin(), new_shape.end(), [](int64_t s) { return s != -1; }); - if (is_any_unknown && is_any_actual) { - (void)new_args_shape->emplace_back(new_shape); - } else { - MS_LOG(DEBUG) << "Not support all shape unknown or actual.Cur shape " << cur_shape << ", elem shape " - << elem_shape << ", and new shape is " << new_shape; + (void)new_shape.emplace_back(UNKNOWN_DIM); } + (void)new_args_shape->emplace_back(new_shape); + MS_LOG(DEBUG) << "Cur shape " << cur_shape << ", elem shape " << elem_shape << ", new shape " << new_shape; } } } // namespace pynative diff --git a/mindspore/ccsrc/pipeline/pynative/dynamic_shape.h b/mindspore/ccsrc/pipeline/pynative/dynamic_shape.h index 362be01a18b..9121db5964f 100644 --- a/mindspore/ccsrc/pipeline/pynative/dynamic_shape.h +++ b/mindspore/ccsrc/pipeline/pynative/dynamic_shape.h @@ -37,7 +37,7 @@ class DynamicShape { DynamicShape() = default; ~DynamicShape() = default; void SetDynamicInput(const py::object &cell, const py::args &args); - void SetFeedDynamicInputAbs(const py::object &cell, const py::args &args, bool is_auto); + void SetFeedDynamicInputAbs(const py::object &cell, const py::args &args); py::object GetDynamicInput(const py::object &actual_input) const; ValuePtr GetSensValueForDynamicShapeOutput(const TopCellInfoPtr &top_cell, const ValuePtr &v, const AnfNodePtr &node) const; diff --git a/mindspore/ccsrc/pipeline/pynative/forward/forward.cc b/mindspore/ccsrc/pipeline/pynative/forward/forward.cc index e2ad7d0c3c6..076498a5fce 100644 --- a/mindspore/ccsrc/pipeline/pynative/forward/forward.cc +++ b/mindspore/ccsrc/pipeline/pynative/forward/forward.cc @@ -369,7 +369,7 @@ void ForwardExecutor::ProcessBeforeNewGraph(const py::object &cell, const py::ar if (py::isinstance(cell)) { PushForwardCell(cell); } - dynamic_shape()->SetFeedDynamicInputAbs(cell, args, false); + dynamic_shape()->SetFeedDynamicInputAbs(cell, args); } void ForwardExecutor::ProcessBeforeEndGraph(const py::object &cell, const py::args &args) { diff --git a/mindspore/ccsrc/pipeline/pynative/grad/grad.cc b/mindspore/ccsrc/pipeline/pynative/grad/grad.cc index 84ee13a19ed..00eb3582845 100644 --- a/mindspore/ccsrc/pipeline/pynative/grad/grad.cc +++ b/mindspore/ccsrc/pipeline/pynative/grad/grad.cc @@ -180,7 +180,7 @@ void GradExecutor::ClearCellRes(const std::string &cell_id) { for (auto it = top_cell_list_.begin(); it != top_cell_list_.end();) { MS_EXCEPTION_IF_NULL(*it); const auto &top_cell_id = (*it)->cell_id(); - const auto &already_run_cell_id = (*it)->already_run_cell_id(); + auto already_run_cell_id = (*it)->already_run_cell_id(); if (IsCellObjIdEq(cell_id, top_cell_id)) { MS_LOG(DEBUG) << "Clear top cell resource. Top cell id " << top_cell_id; (*it)->Clear(); @@ -285,7 +285,7 @@ void GradExecutor::NewGraphInner(const py::object *ret, const py::object &cell, auto top_it = already_run_top_cell_.find(already_run_cell_id); if (top_it != already_run_top_cell_.end()) { // Top cell forward run. - const auto &pre_top_cell = top_it->second; + auto pre_top_cell = top_it->second; MS_EXCEPTION_IF_NULL(pre_top_cell); MS_LOG(DEBUG) << "Pre top cell, hook_changed " << pre_top_cell->hook_changed() << ", is_dynamic_structure " << pre_top_cell->is_dynamic_structure(); @@ -348,26 +348,28 @@ void GradExecutor::MakeNewTopGraph(const string &cell_id, const py::object &cell MS_LOG(WARNING) << "Too many top cell has been built, please check if the cell " << cell.cast()->ToString() << " is repeatedly defined in each step/epoch, or the net input shape changes frequently."; } - // Create top cell + // Find matched dynamic shape top cell auto fg = std::make_shared(); auto df_builder = std::make_shared(); auto resource = std::make_shared(); - const auto &already_run_cell_id = GetAlreadyRunCellId(cell_id); - auto top_cell = - std::make_shared(is_topest, grad_order_, resource, fg, df_builder, cell_id, already_run_cell_id); + auto top_cell = dynamic_shape()->GetTopCellWithDynamicShape(cell, args, true); + if (top_cell == nullptr) { + const auto &already_run_cell_id = GetAlreadyRunCellId(cell_id); + top_cell = + std::make_shared(is_topest, grad_order_, cell_id, already_run_cell_id, resource, fg, df_builder); + top_cell->SetCellSelfInfoForTopCell(cell, args); + (void)top_cell_list_.emplace_back(top_cell); + } else { + auto new_top_cell = std::make_shared(*top_cell, resource, fg, df_builder); + top_cell->Clear(); + EraseTopCellFromTopCellList(top_cell); + top_cell = new_top_cell; + MS_LOG(INFO) << "The shape change of the network input tensor is detected, " + "and the dynamic shape process is triggered. The bprop graph needs to be recompiled, " + "which may take some time"; + } top_cell->set_forward_already_run(true); top_cell->set_input_args_id(input_args_id); - TopCellInfoPtr top_cell_with_dynamic_shape = dynamic_shape()->GetTopCellWithDynamicShape(cell, args, true); - if (top_cell_with_dynamic_shape != nullptr) { - top_cell->set_cell_id(top_cell_with_dynamic_shape->cell_id()); - top_cell->set_already_run_cell_id(top_cell_with_dynamic_shape->already_run_cell_id()); - top_cell->set_cell_self_info(top_cell_with_dynamic_shape->cell_self_info()); - EraseTopCellFromTopCellList(top_cell_with_dynamic_shape); - MS_LOG(DEBUG) << "Pre top cell and current top cell merged to one top cell with dynamic shape"; - } else { - top_cell->SetCellSelfInfoForTopCell(cell, args); - } - (void)top_cell_list_.emplace_back(top_cell); PushHighOrderGraphStack(top_cell); set_top_cell(top_cell); MS_LOG(DEBUG) << "New top graph, fg ptr " << fg.get() << " resource ptr " << resource.get(); @@ -377,17 +379,16 @@ void GradExecutor::SetForwardLastNodeInfo(const ValuePtr &v, const std::string & MS_EXCEPTION_IF_NULL(v); auto output_node = GetObjNode(v, obj_id); MS_EXCEPTION_IF_NULL(output_node); - if (top_cell()->dynamic_shape()) { - abstract::AbstractBasePtr last_node_abs = nullptr; - if (output_node->abstract() == nullptr) { - last_node_abs = v->ToAbstract()->Broaden(); - } else { - last_node_abs = output_node->abstract(); - } - MS_EXCEPTION_IF_NULL(last_node_abs); - // Set last output abstract and will be used for sens - top_cell()->set_last_output_abs(last_node_abs); + abstract::AbstractBasePtr last_node_abs = nullptr; + if (output_node->abstract() == nullptr) { + last_node_abs = v->ToAbstract()->Broaden(); + output_node->set_abstract(last_node_abs); + } else { + last_node_abs = output_node->abstract(); } + MS_EXCEPTION_IF_NULL(last_node_abs); + // Set last output abstract and will be used for sens + top_cell()->set_last_output_abs(last_node_abs); // Set last node and sens for build adjoint const auto &sens_value = dynamic_shape()->GetSensValueForDynamicShapeOutput(top_cell(), v, output_node); auto k_pynative_cell_ptr = top_cell()->k_pynative_cell_ptr(); @@ -805,15 +806,11 @@ FuncGraphPtr GradExecutor::GetBpropGraph(const prim::GradOperationPtr &grad, con ss << "grad{" << arg_size << "}"; bprop_graph->set_flag(FUNC_GRAPH_FLAG_CORE, true); bprop_graph->debug_info()->set_name(ss.str()); - // Get the parameters items and add the value to args_spec - if (top_cell()->dynamic_shape() && grad->sens_param()) { + // Set sens abstract + if (grad->sens_param()) { MS_EXCEPTION_IF_NULL(top_cell()->last_output_abs()); - auto shape = top_cell()->last_output_abs()->BuildShape(); - MS_EXCEPTION_IF_NULL(shape); - if (shape->IsDynamic()) { - const auto &sens_id = PyNativeAlgo::PyParser::GetIdByPyObj(args[arg_size - 1]); - dynamic_shape()->SetIdWithDynamicAbs(sens_id, top_cell()->last_output_abs()); - } + const auto &sens_id = PyNativeAlgo::PyParser::GetIdByPyObj(args[arg_size - 1]); + dynamic_shape()->SetIdWithDynamicAbs(sens_id, top_cell()->last_output_abs()); } UpdateParamAbsByArgs(PyNativeAlgo::PyParser::FilterTensorArgs(args, grad->sens_param_), bprop_graph); // Dynamic shape graph need add some other pass diff --git a/mindspore/ccsrc/pipeline/pynative/grad/grad.h b/mindspore/ccsrc/pipeline/pynative/grad/grad.h index c3e49754253..58d1ce5ea58 100644 --- a/mindspore/ccsrc/pipeline/pynative/grad/grad.h +++ b/mindspore/ccsrc/pipeline/pynative/grad/grad.h @@ -109,6 +109,7 @@ class GradExecutor { CNodePtr ConstructForwardGraph(const FrontendOpRunInfoPtr &op_run_info) const; py::object CheckAlreadyRun(const prim::GradOperationPtr &grad, const py::object &cell, const py::object &grad_position, const py::args &args); + std::string GetAlreadyRunCellId(const std::string &cell_id) const; void ProcessOpGradInfo(const FrontendOpRunInfoPtr &op_run_info, const ValuePtr &v) const; AnfNodePtr GetInput(const ValuePtr &v) const; void ClearGrad(const py::object &cell, const py::args &args); @@ -159,7 +160,6 @@ class GradExecutor { void SetForwardLastNodeInfo(const ValuePtr &v, const std::string &obj_id) const; void DoGradForCustomBprop(const py::object &cell, const py::args &args, const ValuePtr &out, const std::string &out_id); - std::string GetAlreadyRunCellId(const std::string &cell_id) const; std::string GetGradCellId(bool has_sens, const py::object &cell, const py::args &args) const; void GradNetInner(const py::object *ret, const prim::GradOperationPtr &grad, const py::object &cell, const py::object &weights, const py::object &grad_position, const py::args &args); diff --git a/mindspore/ccsrc/pipeline/pynative/grad/top_cell.cc b/mindspore/ccsrc/pipeline/pynative/grad/top_cell.cc index 8ef563854c6..00d372c85cc 100644 --- a/mindspore/ccsrc/pipeline/pynative/grad/top_cell.cc +++ b/mindspore/ccsrc/pipeline/pynative/grad/top_cell.cc @@ -36,8 +36,8 @@ void TopCellInfo::SetCellSelfInfoForTopCell(const py::object &cell, const py::ar (void)args_shape.emplace_back(shape_ptr); (void)args_type.emplace_back(abs->BuildType()); } - set_cell_self_info( - std::make_shared(PyNativeAlgo::PyParser::GetIdByPyObj(cell), args_id, args_shape, args_type)); + cell_self_info_ = + std::make_shared(PyNativeAlgo::PyParser::GetIdByPyObj(cell), args_id, args_shape, args_type); } bool TopCellInfo::IsSubCell(const std::string &cell_id) const { @@ -108,21 +108,31 @@ void TopCellInfo::ClearDeviceMemory() const { void TopCellInfo::Clear() { MS_LOG(DEBUG) << "Clear top cell info. Cell id " << cell_id_; - op_num_ = 0; is_dynamic_structure_ = false; + is_real_dynamic_structure_ = false; vm_compiled_ = false; + hook_changed_ = false; ms_function_flag_ = false; is_init_kpynative_ = false; - need_compile_graph_ = false; forward_already_run_ = false; + need_compile_graph_ = false; + op_num_ = 0; + grad_order_ = 0; + fg_ = nullptr; + df_builder_ = nullptr; + k_pynative_cell_ptr_ = nullptr; + k_pynative_cell_ptr_ = nullptr; + last_output_abs_ = nullptr; + cell_self_info_ = nullptr; + cell_id_.clear(); + already_run_cell_id_.clear(); input_args_id_.clear(); all_op_info_.clear(); - resource_ = nullptr; - df_builder_ = nullptr; - fg_ = nullptr; - k_pynative_cell_ptr_ = nullptr; + grad_operation_.clear(); graph_info_map_.clear(); sub_cell_list_.clear(); + sub_cell_hook_changed_.clear(); + cell_backward_hook_op_.clear(); op_info_with_tensor_id_.clear(); tensor_id_with_tensor_object_.clear(); op_info_with_ms_func_forward_tensors_.clear(); @@ -257,24 +267,16 @@ void TopCellInfo::SetTupleItemArgsToGraphInfoMap(const FuncGraphPtr &g, const Va } } -std::string TopCellInfo::GetAlreadyRunCellId(const std::string &cell_id) const { - std::string already_run_cell_id(cell_id); - size_t grad_order = PyNativeAlgo::Common::GetPyNativeExecutor()->grad_executor()->grad_order(); - already_run_cell_id += std::to_string(grad_order == 0 ? 1 : grad_order); - already_run_cell_id += "_" + grad_operation_; - MS_LOG(DEBUG) << "Get already run top cell id " << already_run_cell_id; - return already_run_cell_id; -} - void TopCellInfo::ChangeTopCellInfo(size_t args_size) { + dynamic_shape_ = true; std::string new_cell_id = this->cell_self_info()->cell_self_id; for (size_t i = 0; i < args_size; ++i) { new_cell_id += "_" + this->cell_self_info()->args_shape[i]->ToString(); - new_cell_id += this->cell_self_info()->args_type[i]->ToString(); + new_cell_id += cell_self_info_->args_type[i]->ToString(); } - MS_LOG(DEBUG) << "Change top cell " << this->cell_id() << " to be dynamic " << new_cell_id; - set_cell_id(new_cell_id); - set_already_run_cell_id(GetAlreadyRunCellId(new_cell_id)); + MS_LOG(DEBUG) << "Change pre top cell " << cell_id_ << " to be dynamic " << new_cell_id; + cell_id_ = new_cell_id; + already_run_cell_id_ = PyNativeAlgo::Common::GetPyNativeExecutor()->grad_executor()->GetAlreadyRunCellId(new_cell_id); } } // namespace pynative } // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/pynative/grad/top_cell.h b/mindspore/ccsrc/pipeline/pynative/grad/top_cell.h index 36a589aebf6..5cc9a99faec 100644 --- a/mindspore/ccsrc/pipeline/pynative/grad/top_cell.h +++ b/mindspore/ccsrc/pipeline/pynative/grad/top_cell.h @@ -78,17 +78,26 @@ using CellSelfInfoPtr = std::shared_ptr; class TopCellInfo { public: - TopCellInfo() = default; ~TopCellInfo() = default; - TopCellInfo(bool topest, size_t grad_order, pipeline::ResourcePtr r, FuncGraphPtr fg, FuncGraphPtr df, - std::string cellid, std::string already_run_cell_id) + TopCellInfo(bool topest, size_t grad_order, std::string cellid, std::string already_run_cell_id, + pipeline::ResourcePtr r, FuncGraphPtr fg, FuncGraphPtr df) : is_topest_(topest), grad_order_(grad_order), + cell_id_(std::move(cellid)), + already_run_cell_id_(std::move(already_run_cell_id)), resource_(std::move(r)), fg_(std::move(fg)), - df_builder_(std::move(df)), - cell_id_(std::move(cellid)), - already_run_cell_id_(std::move(already_run_cell_id)) {} + df_builder_(std::move(df)) {} + + TopCellInfo(const TopCellInfo &top_cell, pipeline::ResourcePtr r, FuncGraphPtr fg, FuncGraphPtr df) + : is_topest_(top_cell.is_topest_), + grad_order_(top_cell.grad_order_), + cell_id_(top_cell.cell_id_), + already_run_cell_id_(top_cell.already_run_cell_id_), + cell_self_info_(top_cell.cell_self_info_), + resource_(std::move(r)), + fg_(std::move(fg)), + df_builder_(std::move(df)) {} bool is_init_kpynative() const { return is_init_kpynative_; } void set_init_kpynative(bool init) { is_init_kpynative_ = init; } @@ -126,14 +135,12 @@ class TopCellInfo { const std::string &cell_id() const { return cell_id_; } void set_cell_id(const std::string &cell_id) { cell_id_ = cell_id; } const std::string &already_run_cell_id() const { return already_run_cell_id_; } - void set_already_run_cell_id(const std::string &already_run_cell_id) { already_run_cell_id_ = already_run_cell_id; } void set_input_args_id(const std::string &input_args_id) { input_args_id_ = input_args_id; } const std::string &grad_operation() const { return grad_operation_; } void set_grad_operation(const std::string &grad_operation) { grad_operation_ = grad_operation; } const abstract::AbstractBasePtr &last_output_abs() const { return last_output_abs_; } void set_last_output_abs(const abstract::AbstractBasePtr &last_output_abs) { last_output_abs_ = last_output_abs; } CellSelfInfoPtr cell_self_info() const { return cell_self_info_; } - void set_cell_self_info(const CellSelfInfoPtr &cell_self_info) { cell_self_info_ = cell_self_info; } void SetCellSelfInfoForTopCell(const py::object &cell, const py::args &args); void EraseFromSubCellList(const std::string &cell_id) { (void)sub_cell_list_.erase(cell_id); } void SetSubCellList(const std::string &cell_id) { (void)sub_cell_list_.emplace(cell_id); } @@ -171,7 +178,6 @@ class TopCellInfo { const std::vector &index); void SetTupleArgsToGraphInfoMap(const FuncGraphPtr &g, const ValuePtr &v, const AnfNodePtr &node, bool is_param = false); - std::string GetAlreadyRunCellId(const std::string &cell_id) const; void ChangeTopCellInfo(size_t args_size); void ClearDeviceMemory() const; void Clear(); @@ -199,17 +205,17 @@ class TopCellInfo { bool need_compile_graph_{false}; size_t op_num_{0}; size_t grad_order_{0}; - pipeline::ResourcePtr resource_{nullptr}; - FuncGraphPtr fg_{nullptr}; - FuncGraphPtr df_builder_{nullptr}; - ad::KPynativeCellPtr k_pynative_cell_ptr_{nullptr}; std::string cell_id_; std::string already_run_cell_id_; std::string input_args_id_; std::string all_op_info_; std::string grad_operation_; - abstract::AbstractBasePtr last_output_abs_; CellSelfInfoPtr cell_self_info_{nullptr}; + pipeline::ResourcePtr resource_{nullptr}; + FuncGraphPtr fg_{nullptr}; + FuncGraphPtr df_builder_{nullptr}; + ad::KPynativeCellPtr k_pynative_cell_ptr_{nullptr}; + abstract::AbstractBasePtr last_output_abs_{nullptr}; OrderedMap graph_info_map_; mindspore::HashSet sub_cell_list_; // Record `register hook` or `remove hook` function has been called by sub cell diff --git a/mindspore/python/mindspore/_extends/builtin_operations.py b/mindspore/python/mindspore/_extends/builtin_operations.py index f0c8fe4daaa..c1468e13b1e 100644 --- a/mindspore/python/mindspore/_extends/builtin_operations.py +++ b/mindspore/python/mindspore/_extends/builtin_operations.py @@ -181,6 +181,30 @@ def stop_gradient(x): return x +def TensorShape(x): + """Implement `TensorShape`.""" + return Tensor(F.shape(x)) + + +def DynamicBroadcastGradientArgs(x, y): + """Implement `DynamicBroadcastGradientArgs`.""" + return -1, -1 + + +def StridedSlice(x, begin, end, stride): + """Implement `StridedSlice`.""" + if isinstance(x, Tensor): + x = x.asnumpy() + ret = x + for i in range(len(x)): + ret[i:] = x[begin[i], end[i], stride[i]] + return Tensor(ret) + ret = x + for i in range(len(x)): + ret[i:] = x[begin[i], end[i], stride[i]] + return ret + + hyper_map = C.HyperMap() diff --git a/tests/st/ops/dynamic_shape/test_dynamic_getitem.py b/tests/st/ops/dynamic_shape/test_dynamic_getitem.py index feab7aeae19..43e0f2b0397 100644 --- a/tests/st/ops/dynamic_shape/test_dynamic_getitem.py +++ b/tests/st/ops/dynamic_shape/test_dynamic_getitem.py @@ -64,18 +64,18 @@ class CommonFunc(): self.np_net = np_net self.input_np = input_np - self.input_np_bp = input_np + self.input_np_t = Tensor(input_np) self.out_np = np.array(1).astype(input_np.dtype) def forward_cmp(self): - out_ms = self.ms_net(Tensor(self.input_np)) + out_ms = self.ms_net(self.input_np_t) self.out_np = self.np_net(self.input_np) assert np.all(out_ms.asnumpy() == self.out_np) def grad_impl(self): grad_net = GradOfFirstInput(self.ms_net) grad_net.set_train() - grad_net(Tensor(self.input_np_bp), Tensor(self.out_np)) + grad_net(self.input_np_t, Tensor(self.out_np)) @pytest.mark.level0 diff --git a/tests/st/ops/dynamic_shape/test_dynamic_setitem.py b/tests/st/ops/dynamic_shape/test_dynamic_setitem.py index e8aa0aaadd9..e92901e69ef 100644 --- a/tests/st/ops/dynamic_shape/test_dynamic_setitem.py +++ b/tests/st/ops/dynamic_shape/test_dynamic_setitem.py @@ -70,14 +70,14 @@ class CommonFunc(): 8 * 16 * 3).reshape(8, 16, 3).astype(np.float32) self.input_np1 = np.arange( 16 * 32 * 3).reshape(16, 32, 3).astype(np.float32) - self.input_np0_bp = self.input_np0.copy() - self.input_np1_bp = self.input_np1.copy() + self.input_np0_t = Tensor(self.input_np0) + self.input_np1_t = Tensor(self.input_np1) self.out_np0 = np.array(1).astype(self.input_np0.dtype) self.out_np1 = np.array(1).astype(self.input_np1.dtype) def forward_cmp(self): out_ms0, out_ms1 = self.ms_net( - Tensor(self.input_np0), Tensor(self.input_np1)) + self.input_np0_t, self.input_np1_t) self.out_np0, self.out_np1 = self. np_net( self.input_np0, self.input_np1) assert np.all(out_ms0.asnumpy() == self.out_np0) @@ -86,7 +86,7 @@ class CommonFunc(): def grad_impl(self): grad_net = GradOfAllInputs(self.ms_net) grad_net.set_train() - grad_net(Tensor(self.input_np0_bp), Tensor(self.input_np1_bp), + grad_net(self.input_np0_t, self.input_np1_t, (Tensor(self.out_np0), Tensor(self.out_np1))) diff --git a/tests/st/pynative/dynamic_shape/test_pynative_auto_dynamic_shape.py b/tests/st/pynative/dynamic_shape/test_pynative_auto_dynamic_shape.py index e4aa6c77485..3647cb15113 100644 --- a/tests/st/pynative/dynamic_shape/test_pynative_auto_dynamic_shape.py +++ b/tests/st/pynative/dynamic_shape/test_pynative_auto_dynamic_shape.py @@ -165,8 +165,8 @@ def test_pynative_auto_dynamic_shape_mixing_static_shape_and_dynamic_shape_1(): # run second shape input_x2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 2) input_y2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 5) - net.set_inputs(Tensor(shape=[2, 3, 6, None], dtype=ms.float32), - Tensor(shape=[2, 3, None, None], dtype=ms.float32)) + net.set_inputs(Tensor(shape=[None, None, None, None], dtype=ms.float32), + Tensor(shape=[None, None, None, None], dtype=ms.float32)) out = net(input_x2, input_y2) _ = grad_op(net)(input_x2, input_y2, out) @@ -204,8 +204,8 @@ def test_pynative_auto_dynamic_shape_mixing_static_shape_and_dynamic_shape_2(): # run first shape input_x = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 2) input_y = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 5) - net.set_inputs(Tensor(shape=[2, 3, 6, None], dtype=ms.float32), - Tensor(shape=[2, 3, None, None], dtype=ms.float32)) + net.set_inputs(Tensor(shape=[None, None, None, None], dtype=ms.float32), + Tensor(shape=[None, None, None, None], dtype=ms.float32)) out = net(input_x, input_y) _ = grad_op(net)(input_x, input_y, out)