forked from mindspore-Ecosystem/mindspore
!34789 Add auto dynamic shape for PyNative
Merge pull request !34789 from zjun/auto_dynamic
This commit is contained in:
commit
4ec1537e7f
|
@ -70,7 +70,6 @@ GradExecutorPtr PynativeExecutor::grad_executor_ = nullptr;
|
|||
std::mutex PynativeExecutor::instance_lock_;
|
||||
|
||||
namespace {
|
||||
const size_t PTR_LEN = 15;
|
||||
const size_t ARG_SIZE = 2;
|
||||
const size_t MAX_TOP_CELL_COUNTS = 20;
|
||||
const size_t kThreshold = 3;
|
||||
|
@ -156,6 +155,8 @@ std::string GetId(const py::handle &obj) {
|
|||
return param_info->name();
|
||||
}
|
||||
return tensor_ptr->id();
|
||||
} else if (py::isinstance<Cell>(obj)) {
|
||||
return obj.cast<CellPtr>()->id();
|
||||
} else if (py::isinstance<mindspore::Type>(obj)) {
|
||||
auto type_ptr = py::cast<mindspore::TypePtr>(obj);
|
||||
return "type" + type_ptr->ToString();
|
||||
|
@ -180,7 +181,7 @@ std::string GetId(const py::handle &obj) {
|
|||
return prefix;
|
||||
}
|
||||
|
||||
if (py::isinstance<Cell>(obj) || py::isinstance<py::function>(obj)) {
|
||||
if (py::isinstance<py::function>(obj)) {
|
||||
auto it = g_pyobj_id_cache.find(obj);
|
||||
if (it == g_pyobj_id_cache.end()) {
|
||||
auto id = GetPyObjId(obj);
|
||||
|
@ -492,6 +493,12 @@ bool RunOpConvertConstInputToAttr(const OpExecInfoPtr &op_run_info, size_t input
|
|||
}
|
||||
const auto &value = PyObjToValue(input_object);
|
||||
auto input_name = input_names_vec[input_index];
|
||||
if (value->isa<tensor::Tensor>()) {
|
||||
auto tensor = value->cast<tensor::TensorPtr>();
|
||||
if (tensor->data().const_data() == nullptr) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
op_prim->AddAttr(input_name, value);
|
||||
op_run_info->index_with_value.emplace_back(std::make_pair(input_index, value));
|
||||
return true;
|
||||
|
@ -1077,6 +1084,23 @@ void SaveIdWithDynamicAbstract(const py::object &obj, const AbstractBasePtr &abs
|
|||
}
|
||||
}
|
||||
|
||||
ShapeVector GetTensorShape(const py::object &obj) {
|
||||
if (py::isinstance<tensor::Tensor>(obj)) {
|
||||
return obj.cast<tensor::TensorPtr>()->shape();
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
TypePtr GetTypeFromAbstract(const abstract::AbstractBasePtr &abs) {
|
||||
MS_EXCEPTION_IF_NULL(abs);
|
||||
if (abs->isa<abstract::AbstractSequence>()) {
|
||||
MS_LOG(EXCEPTION) << "Get tuple or list abs";
|
||||
}
|
||||
const auto &type = abs->BuildType();
|
||||
MS_EXCEPTION_IF_NULL(type);
|
||||
return type;
|
||||
}
|
||||
|
||||
abstract::ShapePtr GetShapeFromAbstract(const abstract::AbstractBasePtr &abs) {
|
||||
MS_EXCEPTION_IF_NULL(abs);
|
||||
if (abs->isa<abstract::AbstractSequence>()) {
|
||||
|
@ -1103,8 +1127,8 @@ void SaveIdWithDynamicShape(const OpExecInfoPtr &op_exec_info, const std::string
|
|||
}
|
||||
} else {
|
||||
const auto &dynamic_shape_vec = GetShapeFromAbstract(dynamic_abs);
|
||||
MS_LOG(DEBUG) << "Save tensor " << id << ", real shape " << PyObjToValue(real_obj)->ToAbstract()
|
||||
<< ", dynamic shape " << dynamic_shape_vec->ToString();
|
||||
MS_LOG(DEBUG) << "Save tensor " << id << ", real shape " << GetTensorShape(real_obj) << ", dynamic shape "
|
||||
<< dynamic_shape_vec->ToString();
|
||||
op_exec_info->id_with_dynamic_shape.emplace(std::make_pair(id, dynamic_shape_vec));
|
||||
}
|
||||
}
|
||||
|
@ -1123,8 +1147,8 @@ void UpdateInputTensorToDynamicShape(const OpExecInfoPtr &op_exec_info, std::vec
|
|||
}
|
||||
}
|
||||
|
||||
void UpdateOutputTensorToDynamicShape(
|
||||
const ValuePtr &value, const OrderedMap<std::string, abstract::AbstractBasePtr> &obj_id_with_dynamic_abs) {
|
||||
void UpdateValueToDynamicShape(const ValuePtr &value,
|
||||
const OrderedMap<std::string, abstract::AbstractBasePtr> &obj_id_with_dynamic_abs) {
|
||||
MS_EXCEPTION_IF_NULL(value);
|
||||
if (value->isa<mindspore::tensor::Tensor>()) {
|
||||
auto tensor_value = value->cast<tensor::TensorPtr>();
|
||||
|
@ -1135,10 +1159,10 @@ void UpdateOutputTensorToDynamicShape(
|
|||
} else if (value->isa<ValueTuple>()) {
|
||||
auto value_tuple = value->cast<ValueTuplePtr>();
|
||||
for (const auto &v : value_tuple->value()) {
|
||||
UpdateOutputTensorToDynamicShape(v, obj_id_with_dynamic_abs);
|
||||
UpdateValueToDynamicShape(v, obj_id_with_dynamic_abs);
|
||||
}
|
||||
} else {
|
||||
MS_LOG(EXCEPTION) << "Out put is not a tensor";
|
||||
MS_LOG(DEBUG) << "Out put is not a tensor";
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1174,6 +1198,45 @@ ValuePtr SetSensValue(const ValuePtr &value, TensorIdWithTensorObject *tensor_id
|
|||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
void FindMatchTopCell(const TopCellInfoPtr &top_cell, const py::args &args, std::vector<ShapeVector> *new_args_shape) {
|
||||
MS_EXCEPTION_IF_NULL(top_cell);
|
||||
MS_EXCEPTION_IF_NULL(new_args_shape);
|
||||
for (size_t i = 0; i < args.size(); ++i) {
|
||||
const auto &cur_value_abs = PyObjToValue(args[i])->ToAbstract();
|
||||
MS_EXCEPTION_IF_NULL(cur_value_abs);
|
||||
const auto &cur_type = GetTypeFromAbstract(cur_value_abs);
|
||||
const auto &elem_type = top_cell->cell_self_info()->args_type[i];
|
||||
// Type is not the same
|
||||
if (cur_type->hash() != elem_type->hash()) {
|
||||
MS_LOG(DEBUG) << "The " << i << "th args type is not the same, cur is " << cur_type->ToString()
|
||||
<< " and the elem is " << elem_type->ToString();
|
||||
return;
|
||||
}
|
||||
// Check shape
|
||||
const auto &cur_shape = GetShapeFromAbstract(cur_value_abs)->shape();
|
||||
auto elem_shape = top_cell->cell_self_info()->args_shape[i]->shape();
|
||||
if (cur_shape.size() != elem_shape.size()) {
|
||||
MS_LOG(DEBUG) << "The " << i << "th args shape size is not the same, cur is " << cur_shape.size()
|
||||
<< " and the elem is " << elem_shape.size();
|
||||
return;
|
||||
}
|
||||
ShapeVector new_shape;
|
||||
for (size_t j = 0; j < cur_shape.size(); ++j) {
|
||||
if (cur_shape[j] == elem_shape[j]) {
|
||||
new_shape.emplace_back(cur_shape[j]);
|
||||
} else {
|
||||
new_shape.emplace_back(-1);
|
||||
}
|
||||
}
|
||||
// All shape can not be -1
|
||||
if (std::any_of(new_shape.begin(), new_shape.end(), [](int64_t s) { return s != -1; })) {
|
||||
new_args_shape->emplace_back(new_shape);
|
||||
} else {
|
||||
MS_LOG(DEBUG) << "Cur shape " << cur_shape << ", elem shape " << elem_shape << ", and new shape is " << new_shape;
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
py::object RealRunOp(const py::args &args) {
|
||||
|
@ -1201,6 +1264,25 @@ GradExecutorPtr ForwardExecutor::grad() const {
|
|||
return grad_executor;
|
||||
}
|
||||
|
||||
void TopCellInfo::SetCellSelfInfoForTopCell(const py::object &cell, const py::args &args) {
|
||||
std::vector<std::string> args_id;
|
||||
std::vector<abstract::ShapePtr> args_shape;
|
||||
std::vector<TypePtr> args_type;
|
||||
for (size_t i = 0; i < args.size(); ++i) {
|
||||
auto value = PyObjToValue(args[i]);
|
||||
MS_EXCEPTION_IF_NULL(value);
|
||||
auto abs = value->ToAbstract();
|
||||
auto shape_ptr = abs->BuildShape()->cast<abstract::ShapePtr>();
|
||||
if (shape_ptr == nullptr) {
|
||||
return;
|
||||
}
|
||||
args_id.emplace_back(GetId(args[i]));
|
||||
args_shape.emplace_back(shape_ptr);
|
||||
args_type.emplace_back(abs->BuildType());
|
||||
}
|
||||
set_cell_self_info(std::make_shared<CellSelfInfo>(GetId(cell), args_id, args_shape, args_type));
|
||||
}
|
||||
|
||||
bool TopCellInfo::IsSubCell(const std::string &cell_id) const {
|
||||
if (sub_cell_list_.empty()) {
|
||||
MS_LOG(DEBUG) << "The sub cell list is empty, there is no sub cell";
|
||||
|
@ -1394,7 +1476,7 @@ AbstractBasePtr ForwardExecutor::GetInputObjAbstract(const OpExecInfoPtr &op_exe
|
|||
if (it != node_abs_map_.end()) {
|
||||
abs = it->second;
|
||||
}
|
||||
MS_LOG(DEBUG) << "Abstract cache hit " << (abs == nullptr);
|
||||
MS_LOG(DEBUG) << "Abstract cache hit " << (abs != nullptr);
|
||||
bool is_const_prim_or_input = IsConstPrimOrConstInput(op_exec_info, i);
|
||||
if (abs == nullptr || is_const_prim_or_input) {
|
||||
abs = PyObjToValue(obj)->ToAbstract();
|
||||
|
@ -2226,7 +2308,7 @@ void GradExecutor::DoOpGrad(const OpExecInfoPtr &op_exec_info, const CNodePtr &c
|
|||
}
|
||||
}
|
||||
if (op_exec_info->has_dynamic_output) {
|
||||
UpdateOutputTensorToDynamicShape(op_out, forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs);
|
||||
UpdateValueToDynamicShape(op_out, forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs);
|
||||
}
|
||||
|
||||
if (!ad::GradPynativeOp(top_cell()->k_pynative_cell_ptr(), cnode, input_args, op_out)) {
|
||||
|
@ -2234,18 +2316,30 @@ void GradExecutor::DoOpGrad(const OpExecInfoPtr &op_exec_info, const CNodePtr &c
|
|||
}
|
||||
}
|
||||
|
||||
void GradExecutor::SaveDynShapeAbsForMsFunction(const py::object &forward_out, const FuncGraphPtr &ms_func_graph) {
|
||||
void GradExecutor::SaveDynShapeAbsForMsFunction(const py::args &args, const py::object &out,
|
||||
const FuncGraphPtr &ms_func_graph) {
|
||||
MS_EXCEPTION_IF_NULL(ms_func_graph);
|
||||
auto output_node = ms_func_graph->output();
|
||||
MS_EXCEPTION_IF_NULL(output_node);
|
||||
if (ms_func_graph->modify_output()) {
|
||||
auto make_tuple = output_node->cast<CNodePtr>();
|
||||
MS_EXCEPTION_IF_NULL(make_tuple);
|
||||
output_node = make_tuple->input(1);
|
||||
MS_EXCEPTION_IF_NULL(output_node);
|
||||
|
||||
// Update input to dynamic
|
||||
for (size_t i = 0; i < args.size(); ++i) {
|
||||
if (py::isinstance<tensor::Tensor>(args[i])) {
|
||||
const auto &input_i_tensor = args[i].cast<tensor::TensorPtr>();
|
||||
UpdateValueToDynamicShape(input_i_tensor, forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs);
|
||||
}
|
||||
SaveIdWithDynamicAbstract(forward_out, output_node->abstract(),
|
||||
}
|
||||
|
||||
// Update output to dynamic
|
||||
SaveIdWithDynamicAbstract(out, output_node->abstract(),
|
||||
&(forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs));
|
||||
const auto &output_value = PyObjToValue(out);
|
||||
UpdateValueToDynamicShape(output_value, forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs);
|
||||
|
||||
// Save output by one id for abs get performance
|
||||
if (output_node->abstract()->BuildShape()->IsDynamic()) {
|
||||
forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs[GetId(out)] = output_node->abstract();
|
||||
}
|
||||
}
|
||||
|
||||
void GradExecutor::UpdateMsFunctionForwardTensors(const OpExecInfoPtr &op_exec_info,
|
||||
|
@ -2334,7 +2428,7 @@ void GradExecutor::MakeCNodeForMsFunction(const FuncGraphPtr &ms_func_graph, con
|
|||
}
|
||||
|
||||
// Make adjoint for ms_function fprop graph and connect it with previous op
|
||||
void GradExecutor::MakeAdjointForMsFunction(const FuncGraphPtr &ms_func_graph, const FuncGraphPtr &grad_graph,
|
||||
CNodePtr GradExecutor::MakeAdjointForMsFunction(const FuncGraphPtr &ms_func_graph, const FuncGraphPtr &grad_graph,
|
||||
const py::object &actual_out, const py::args &args,
|
||||
const ValuePtr &actual_out_v) {
|
||||
ValuePtrList input_values;
|
||||
|
@ -2353,6 +2447,7 @@ void GradExecutor::MakeAdjointForMsFunction(const FuncGraphPtr &ms_func_graph, c
|
|||
<< ms_function_cnode->DebugString();
|
||||
}
|
||||
top_cell()->set_ms_function_flag(true);
|
||||
return ms_function_cnode;
|
||||
}
|
||||
|
||||
void GradExecutor::UpdateForwardTensorInfoInBpropGraph(const string &op_info, const ValuePtr &op_out) {
|
||||
|
@ -2426,11 +2521,7 @@ void GradExecutor::SaveForwardTensorInfoInBpropGraph(const pipeline::ResourcePtr
|
|||
}
|
||||
// Check exception case.
|
||||
auto &tensor_id_with_tensor_object = top_cell()->tensor_id_with_tensor_object();
|
||||
if (tensor_id_with_tensor_object.size() > 1) {
|
||||
MS_LOG(EXCEPTION) << "When compile a top graph, the tensor_id_with_tensor_object map should be empty or only have "
|
||||
"sens value. Top cell: "
|
||||
<< top_cell()->cell_id();
|
||||
}
|
||||
MS_LOG(DEBUG) << "Current tensor_id_with_tensor_object size " << tensor_id_with_tensor_object.size();
|
||||
// Save tensor in value node of bprop graph
|
||||
for (const auto &tensor : tensors_in_bprop_graph) {
|
||||
MS_EXCEPTION_IF_NULL(tensor);
|
||||
|
@ -2555,7 +2646,6 @@ void ForwardExecutor::CheckIfNeedSyncForHeterogeneous(const std::string &cur_tar
|
|||
}
|
||||
|
||||
void ForwardExecutor::SetDynamicInput(const py::object &cell, const py::args &args) {
|
||||
MS_LOG(DEBUG) << "Set dynamic input for feed mode";
|
||||
auto &dynamic_index = dynamic_shape_info_ptr()->feed_dynamic_input[GetId(cell)];
|
||||
dynamic_index.resize(args.size());
|
||||
for (size_t i = 0; i < args.size(); i++) {
|
||||
|
@ -2575,8 +2665,8 @@ void ForwardExecutor::SetFeedDynamicInputAbs(const py::object &cell, const py::a
|
|||
auto it = feed_dynamic_input.find(cell_id);
|
||||
if (it != feed_dynamic_input.end()) {
|
||||
if (it->second.size() != args.size()) {
|
||||
MS_LOG(EXCEPTION) << "Dynamic input size " << it->second.size() << " is not equal to real input size "
|
||||
<< args.size();
|
||||
MS_LOG(DEBUG) << "Dynamic input size " << it->second.size() << " is not equal to real input size " << args.size();
|
||||
return;
|
||||
}
|
||||
for (size_t i = 0; i < args.size(); i++) {
|
||||
auto abs = it->second.at(i);
|
||||
|
@ -2584,6 +2674,9 @@ void ForwardExecutor::SetFeedDynamicInputAbs(const py::object &cell, const py::a
|
|||
auto shape = abs->BuildShape();
|
||||
MS_EXCEPTION_IF_NULL(shape);
|
||||
if (shape->IsDynamic()) {
|
||||
MS_LOG(DEBUG) << "Set arg " << i << ", id " << GetId(args[i])
|
||||
<< ", to be dynamic shape; Arg self abs: " << PyObjToValue(args[i])->ToAbstract()->ToString()
|
||||
<< ", dynamic abs: " << abs->ToString();
|
||||
dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs[GetId(args[i])] = abs;
|
||||
}
|
||||
}
|
||||
|
@ -2826,7 +2919,9 @@ inline bool GradExecutor::IsNestedGrad() const {
|
|||
|
||||
bool GradExecutor::IsCellObjIdEq(const std::string &l_cell_id, const std::string &r_cell_id) const {
|
||||
// just compare obj_id, ignore args id
|
||||
return l_cell_id.compare(0, PTR_LEN, r_cell_id, 0, PTR_LEN) == 0;
|
||||
auto l_index = l_cell_id.find('_');
|
||||
auto r_index = r_cell_id.find('_');
|
||||
return l_cell_id.substr(0, l_index) == r_cell_id.substr(0, r_index);
|
||||
}
|
||||
|
||||
bool GradExecutor::IsBpropGraph(const std::string &cell_id) {
|
||||
|
@ -3000,6 +3095,109 @@ void GradExecutor::NewGraphInner(py::object *ret, const py::object &cell, const
|
|||
}
|
||||
}
|
||||
|
||||
TopCellInfoPtr GradExecutor::ChangeTopCellToDynamicShapeByAuto(const TopCellInfoPtr &top_cell,
|
||||
const std::vector<ShapeVector> &new_args_shape,
|
||||
const py::object &cell, const py::args &args) {
|
||||
MS_EXCEPTION_IF_NULL(top_cell);
|
||||
// Change args shape
|
||||
for (size_t i = 0; i < args.size(); ++i) {
|
||||
top_cell->cell_self_info()->args_shape[i] = std::make_shared<abstract::Shape>(new_args_shape[i]);
|
||||
if (py::isinstance<tensor::Tensor>(args[i])) {
|
||||
auto tensor = args[i].cast<tensor::TensorPtr>();
|
||||
tensor->set_base_shape(top_cell->cell_self_info()->args_shape[i]);
|
||||
}
|
||||
forward()->node_abs_map().erase(GetId(args[i]));
|
||||
}
|
||||
// Set to feed dynamic map, later shapes can match it
|
||||
MS_LOG(DEBUG) << "Set dynamic input for auto dynamic shape";
|
||||
forward()->SetDynamicInput(cell, args);
|
||||
forward()->SetFeedDynamicInputAbs(cell, args);
|
||||
// Change cell id
|
||||
std::string new_cell_id = top_cell->cell_self_info()->cell_self_id;
|
||||
for (size_t i = 0; i < new_args_shape.size(); ++i) {
|
||||
new_cell_id += "_" + top_cell->cell_self_info()->args_shape[i]->ToString();
|
||||
new_cell_id += top_cell->cell_self_info()->args_type[i]->ToString();
|
||||
}
|
||||
MS_LOG(DEBUG) << "Change top cell " << top_cell->cell_id() << " to be dynamic " << new_cell_id;
|
||||
top_cell->set_cell_id(new_cell_id);
|
||||
top_cell->set_already_run_cell_id(GetAlreadyRunCellId(new_cell_id));
|
||||
return top_cell;
|
||||
}
|
||||
|
||||
TopCellInfoPtr GradExecutor::ChangeTopCellToDynamicShapeBySetInputs(const TopCellInfoPtr &top_cell,
|
||||
const std::vector<ShapeVector> &new_args_shape,
|
||||
const py::object &cell) {
|
||||
MS_EXCEPTION_IF_NULL(top_cell);
|
||||
// Change args shape
|
||||
for (size_t i = 0; i < new_args_shape.size(); ++i) {
|
||||
top_cell->cell_self_info()->args_shape[i] = std::make_shared<abstract::Shape>(new_args_shape[i]);
|
||||
}
|
||||
const auto &feed_dynamic_input = forward()->dynamic_shape_info_ptr()->feed_dynamic_input;
|
||||
auto it = feed_dynamic_input.find(GetId(cell));
|
||||
if (it != feed_dynamic_input.end()) {
|
||||
for (size_t i = 0; i < new_args_shape.size(); i++) {
|
||||
auto abs = it->second.at(i);
|
||||
MS_EXCEPTION_IF_NULL(abs);
|
||||
auto shape = abs->BuildShape();
|
||||
MS_EXCEPTION_IF_NULL(shape);
|
||||
if (shape->IsDynamic()) {
|
||||
const auto &arg_id = top_cell->cell_self_info()->args_id[i];
|
||||
MS_LOG(DEBUG) << "Set arg " << i << ", id " << arg_id << ", dynamic abs: " << abs->ToString();
|
||||
forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs[arg_id] = abs;
|
||||
forward()->node_abs_map().clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
// Change cell id
|
||||
std::string new_cell_id = top_cell->cell_self_info()->cell_self_id;
|
||||
for (size_t i = 0; i < new_args_shape.size(); ++i) {
|
||||
new_cell_id += "_" + top_cell->cell_self_info()->args_shape[i]->ToString();
|
||||
new_cell_id += top_cell->cell_self_info()->args_type[i]->ToString();
|
||||
}
|
||||
MS_LOG(DEBUG) << "Change top cell " << top_cell->cell_id() << " to be dynamic " << new_cell_id;
|
||||
top_cell->set_cell_id(new_cell_id);
|
||||
top_cell->set_already_run_cell_id(GetAlreadyRunCellId(new_cell_id));
|
||||
return top_cell;
|
||||
}
|
||||
|
||||
TopCellInfoPtr GradExecutor::GetTopCellWithDynamicShape(const py::object &cell, const py::args &args, bool is_auto) {
|
||||
// Current return nullptr for disable auto dynamic shape feature; Later after a complete test will enable this
|
||||
if (is_auto && py::isinstance<py::none>(cell)) {
|
||||
return nullptr;
|
||||
}
|
||||
const auto &cell_self_id = GetId(cell);
|
||||
auto it = std::find_if(top_cell_list_.begin(), top_cell_list_.end(), [&cell_self_id](const TopCellInfoPtr &elem) {
|
||||
return elem->cell_self_info()->cell_self_id == cell_self_id;
|
||||
});
|
||||
if (it != top_cell_list_.end()) {
|
||||
const auto &elem = *it;
|
||||
if (elem->dynamic_shape()) {
|
||||
MS_LOG(DEBUG) << "Elem have is already dynamic shape";
|
||||
return nullptr;
|
||||
}
|
||||
std::vector<ShapeVector> new_args_shape;
|
||||
FindMatchTopCell(elem, args, &new_args_shape);
|
||||
// Change top cell to be dynamic
|
||||
if (new_args_shape.size() == args.size()) {
|
||||
if (is_auto) {
|
||||
return ChangeTopCellToDynamicShapeByAuto(elem, new_args_shape, cell, args);
|
||||
} else {
|
||||
return ChangeTopCellToDynamicShapeBySetInputs(elem, new_args_shape, cell);
|
||||
}
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void GradExecutor::CheckPreviousTopCellCanBeDynamicShape(const py::object &cell, const py::args &args) {
|
||||
if (!grad_flag()) {
|
||||
return;
|
||||
}
|
||||
// In ms_function, new graph run before construct, so top cell create first; After that, set_dynamic_input call
|
||||
// in construct, here change top cell to dynamic.
|
||||
GetTopCellWithDynamicShape(cell, args, false);
|
||||
}
|
||||
|
||||
void GradExecutor::MakeNewTopGraph(const string &cell_id, const py::object &cell, const py::args &args,
|
||||
bool is_topest) {
|
||||
pipeline::CheckArgsValid(cell, args);
|
||||
|
@ -3031,6 +3229,16 @@ void GradExecutor::MakeNewTopGraph(const string &cell_id, const py::object &cell
|
|||
std::make_shared<TopCellInfo>(is_topest, grad_order_, resource, fg, df_builder, cell_id, already_run_cell_id);
|
||||
top_cell->set_forward_already_run(true);
|
||||
top_cell->set_input_args_id(input_args_id);
|
||||
TopCellInfoPtr top_cell_with_dynamic_shape = GetTopCellWithDynamicShape(cell, args, true);
|
||||
if (top_cell_with_dynamic_shape != nullptr) {
|
||||
top_cell->set_cell_id(top_cell_with_dynamic_shape->cell_id());
|
||||
top_cell->set_already_run_cell_id(top_cell_with_dynamic_shape->already_run_cell_id());
|
||||
top_cell->set_cell_self_info(top_cell_with_dynamic_shape->cell_self_info());
|
||||
EraseTopCellFromTopCellList(top_cell_with_dynamic_shape);
|
||||
MS_LOG(DEBUG) << "Pre top cell and current top cell merged to one top cell with dynamic shape";
|
||||
} else {
|
||||
top_cell->SetCellSelfInfoForTopCell(cell, args);
|
||||
}
|
||||
top_cell_list_.emplace_back(top_cell);
|
||||
PushHighOrderGraphStack(top_cell);
|
||||
set_top_cell(top_cell);
|
||||
|
@ -3184,7 +3392,14 @@ void GradExecutor::EndGraphInner(py::object *ret, const py::object &cell, const
|
|||
MS_LOG(DEBUG) << "Cur top last cell " << cell_id;
|
||||
PopHighOrderGraphStack();
|
||||
auto output_node = GetObjNode(out, GetId(out));
|
||||
auto last_node_abs = output_node->abstract();
|
||||
MS_EXCEPTION_IF_NULL(output_node);
|
||||
abstract::AbstractBasePtr last_node_abs = nullptr;
|
||||
if (output_node->abstract() == nullptr) {
|
||||
last_node_abs = PyObjToValue(out)->ToAbstract()->Broaden();
|
||||
} else {
|
||||
last_node_abs = output_node->abstract();
|
||||
}
|
||||
MS_EXCEPTION_IF_NULL(last_node_abs);
|
||||
// Set last output abstract and will be used for sens
|
||||
top_cell()->set_last_output_abs(last_node_abs);
|
||||
auto sens_value = GetSensValueForDynamicShapeOutput(out, output_node);
|
||||
|
@ -3559,8 +3774,9 @@ FuncGraphPtr GradExecutor::GetBpropGraph(const prim::GradOperationPtr &grad, con
|
|||
ss << "grad{" << arg_size << "}";
|
||||
bprop_graph->set_flag(FUNC_GRAPH_FLAG_CORE, true);
|
||||
bprop_graph->debug_info()->set_name(ss.str());
|
||||
// Get the parameters items and add the value to args_abs
|
||||
if (grad->sens_param() && top_cell()->last_output_abs() != nullptr) {
|
||||
// Get the parameters items and add the value to args_spec
|
||||
if (grad->sens_param()) {
|
||||
MS_EXCEPTION_IF_NULL(top_cell()->last_output_abs());
|
||||
auto shape = top_cell()->last_output_abs()->BuildShape();
|
||||
MS_EXCEPTION_IF_NULL(shape);
|
||||
if (shape->IsDynamic()) {
|
||||
|
@ -3916,7 +4132,12 @@ void GradExecutor::GradMsFunctionInner(const std::string &phase, const py::objec
|
|||
// Identity op info for current running ms_func graph.
|
||||
OpExecInfoPtr op_exec_info = std::make_shared<OpExecInfo>();
|
||||
op_exec_info->op_name = phase;
|
||||
auto it = forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs.find(GetId(actual_out));
|
||||
if (it != forward()->dynamic_shape_info_ptr()->obj_id_with_dynamic_output_abs.end()) {
|
||||
op_exec_info->abstract = it->second;
|
||||
} else {
|
||||
op_exec_info->abstract = actual_out_v->ToAbstract();
|
||||
}
|
||||
RecordGradOpInfo(op_exec_info);
|
||||
MS_LOG(DEBUG) << "ms_function cnode op info: " << op_exec_info->op_info;
|
||||
|
||||
|
@ -3943,7 +4164,9 @@ void GradExecutor::GradMsFunctionInner(const std::string &phase, const py::objec
|
|||
new_ms_func_graph->set_output(new_make_tuple->input(1));
|
||||
|
||||
// Make Adjoint for grad graph
|
||||
const auto &ms_function_cnode =
|
||||
MakeAdjointForMsFunction(new_ms_func_graph, new_grad_graph, actual_out, args, actual_out_v);
|
||||
ms_function_cnode->set_abstract(new_ms_func_graph->output()->abstract()->Broaden());
|
||||
}
|
||||
|
||||
py::object GradExecutor::GradMsFunction(const py::object &out, const py::args &args) {
|
||||
|
@ -3964,7 +4187,7 @@ py::object GradExecutor::GradMsFunction(const py::object &out, const py::args &a
|
|||
ret = tuple_out[0];
|
||||
}
|
||||
// Save dynamic shape info if output tensors of forward graph have dynamic shapes
|
||||
SaveDynShapeAbsForMsFunction(ret, ms_func_graph);
|
||||
SaveDynShapeAbsForMsFunction(args, out, ms_func_graph);
|
||||
// Make Adjoint for grad graph of ms_function.
|
||||
if (!grad_flag_) {
|
||||
MS_LOG(DEBUG) << "Only run forward infer computation, no need to construct grad graph.";
|
||||
|
@ -4046,7 +4269,10 @@ void PynativeExecutor::set_graph_phase(const std::string &graph_phase) {
|
|||
}
|
||||
|
||||
void PynativeExecutor::SetDynamicInput(const py::object &cell, const py::args &args) {
|
||||
MS_LOG(DEBUG) << "Set dynamic input for feed mode from cell";
|
||||
forward_executor()->SetDynamicInput(cell, args);
|
||||
// After set input, check previous top cell can be make to dynamic shape
|
||||
grad_executor()->CheckPreviousTopCellCanBeDynamicShape(cell, args);
|
||||
}
|
||||
|
||||
py::object PynativeExecutor::GetDynamicInput(const py::object &actual_input) const {
|
||||
|
@ -4090,7 +4316,8 @@ py::object PynativeExecutor::Run(const py::object &cell, const py::object &sens_
|
|||
return ret;
|
||||
}
|
||||
|
||||
void PynativeExecutor::ClearCell(const std::string &cell_id) {
|
||||
void PynativeExecutor::ClearCell(const py::object &cell) {
|
||||
const auto &cell_id = GetId(cell);
|
||||
MS_LOG(DEBUG) << "Clear cell res, cell id " << cell_id;
|
||||
grad_executor()->ClearCellRes(cell_id);
|
||||
}
|
||||
|
|
|
@ -67,6 +67,23 @@ struct DynamicShapeInfo {
|
|||
};
|
||||
using DynamicShapeInfoPtr = std::shared_ptr<DynamicShapeInfo>;
|
||||
|
||||
struct CellSelfInfo {
|
||||
CellSelfInfo() = default;
|
||||
~CellSelfInfo() = default;
|
||||
CellSelfInfo(std::string cell_self_id, std::vector<std::string> args_id, std::vector<abstract::ShapePtr> args_shape,
|
||||
std::vector<TypePtr> args_type)
|
||||
: cell_self_id(std::move(cell_self_id)),
|
||||
args_id(std::move(args_id)),
|
||||
args_shape(std::move(args_shape)),
|
||||
args_type(std::move(args_type)) {}
|
||||
|
||||
std::string cell_self_id;
|
||||
std::vector<std::string> args_id;
|
||||
std::vector<abstract::ShapePtr> args_shape;
|
||||
std::vector<TypePtr> args_type;
|
||||
};
|
||||
using CellSelfInfoPtr = std::shared_ptr<CellSelfInfo>;
|
||||
|
||||
class TopCellInfo {
|
||||
public:
|
||||
TopCellInfo() = default;
|
||||
|
@ -113,7 +130,9 @@ class TopCellInfo {
|
|||
size_t op_num() const { return op_num_; }
|
||||
void set_op_num(size_t op_num) { op_num_ = op_num; }
|
||||
const std::string &cell_id() const { return cell_id_; }
|
||||
void set_cell_id(const std::string &cell_id) { cell_id_ = cell_id; }
|
||||
const std::string &already_run_cell_id() const { return already_run_cell_id_; }
|
||||
void set_already_run_cell_id(const std::string &already_run_cell_id) { already_run_cell_id_ = already_run_cell_id; }
|
||||
const std::string &input_args_id() const { return input_args_id_; }
|
||||
void set_input_args_id(const std::string &input_args_id) { input_args_id_ = input_args_id; }
|
||||
std::string &all_op_info() { return all_op_info_; }
|
||||
|
@ -121,6 +140,9 @@ class TopCellInfo {
|
|||
void set_grad_operation(const std::string &grad_operation) { grad_operation_ = grad_operation; }
|
||||
const abstract::AbstractBasePtr &last_output_abs() const { return last_output_abs_; }
|
||||
void set_last_output_abs(const abstract::AbstractBasePtr &last_output_abs) { last_output_abs_ = last_output_abs; }
|
||||
CellSelfInfoPtr cell_self_info() const { return cell_self_info_; }
|
||||
void set_cell_self_info(const CellSelfInfoPtr &cell_self_info) { cell_self_info_ = cell_self_info; }
|
||||
void SetCellSelfInfoForTopCell(const py::object &cell, const py::args &args);
|
||||
mindspore::HashSet<std::string> &sub_cell_list() { return sub_cell_list_; }
|
||||
std::set<std::string> &forward_op_output_id() { return forward_op_output_id_; }
|
||||
bool IsSubCell(const std::string &cell_id) const;
|
||||
|
@ -165,6 +187,7 @@ class TopCellInfo {
|
|||
std::string all_op_info_;
|
||||
std::string grad_operation_;
|
||||
abstract::AbstractBasePtr last_output_abs_;
|
||||
CellSelfInfoPtr cell_self_info_;
|
||||
OrderedMap<FuncGraphPtr, GraphInfoPtr> graph_info_map_;
|
||||
mindspore::HashSet<std::string> sub_cell_list_;
|
||||
// Record `register hook` or `remove hook` function has been called by sub cell
|
||||
|
@ -246,9 +269,9 @@ class GradExecutor {
|
|||
py::object GradMsFunction(const py::object &out, const py::args &args);
|
||||
void GradMsFunctionInner(const std::string &phase, const py::object &out, const py::args &args,
|
||||
const FuncGraphPtr &ms_func_graph, const FuncGraphPtr &grad_graph);
|
||||
void SaveDynShapeAbsForMsFunction(const py::object &forward_out, const FuncGraphPtr &ms_func_graph);
|
||||
void SaveDynShapeAbsForMsFunction(const py::args &args, const py::object &out, const FuncGraphPtr &ms_func_graph);
|
||||
void UpdateMsFunctionForwardTensors(const OpExecInfoPtr &op_exec_info, const ValuePtr &new_forward_value);
|
||||
void MakeAdjointForMsFunction(const FuncGraphPtr &ms_func_graph, const FuncGraphPtr &grad_graph,
|
||||
CNodePtr MakeAdjointForMsFunction(const FuncGraphPtr &ms_func_graph, const FuncGraphPtr &grad_graph,
|
||||
const py::object &actual_out, const py::args &args, const ValuePtr &actual_out_v);
|
||||
void MakeCNodeForMsFunction(const FuncGraphPtr &ms_func_graph, const py::args &args, ValuePtrList *input_values,
|
||||
CNodePtr *ms_function_cnode);
|
||||
|
@ -258,6 +281,14 @@ class GradExecutor {
|
|||
void UpdateForwardTensorInfoInBpropGraph(const string &op_info, const ValuePtr &op_out);
|
||||
void SaveForwardTensorInfoInBpropGraph(const pipeline::ResourcePtr &resource) const;
|
||||
py::object CheckGraph(const py::object &cell, const py::args &args);
|
||||
TopCellInfoPtr ChangeTopCellToDynamicShapeByAuto(const TopCellInfoPtr &top_cell,
|
||||
const std::vector<ShapeVector> &new_args_shape,
|
||||
const py::object &cell, const py::args &args);
|
||||
TopCellInfoPtr ChangeTopCellToDynamicShapeBySetInputs(const TopCellInfoPtr &top_cell,
|
||||
const std::vector<ShapeVector> &new_args_shape,
|
||||
const py::object &cell);
|
||||
TopCellInfoPtr GetTopCellWithDynamicShape(const py::object &cell, const py::args &args, bool is_auto);
|
||||
void CheckPreviousTopCellCanBeDynamicShape(const py::object &cell, const py::args &args);
|
||||
void RunGradGraph(py::object *ret, const py::object &cell, const py::object &sens_param, const py::tuple &args);
|
||||
py::object CheckAlreadyRun(const prim::GradOperationPtr &grad, const py::object &cell, const py::args &args);
|
||||
void EraseTopCellFromTopCellList(const TopCellInfoPtr &top_cell);
|
||||
|
@ -472,7 +503,7 @@ class PynativeExecutor : public std::enable_shared_from_this<PynativeExecutor> {
|
|||
|
||||
// Used by graph clean
|
||||
// Cell destruct will call
|
||||
void ClearCell(const std::string &cell_id);
|
||||
void ClearCell(const py::object &cell);
|
||||
void ClearGrad(const py::object &cell, const py::args &args);
|
||||
// Abnormal existed
|
||||
void ClearRes();
|
||||
|
|
|
@ -27,6 +27,7 @@ constexpr size_t kMaskedSelectOutputsNum = 1;
|
|||
|
||||
void MaskedSelectCpuKernelMod::InitKernel(const CNodePtr &kernel_node) {
|
||||
MS_EXCEPTION_IF_NULL(kernel_node);
|
||||
tensor_size_ = 1;
|
||||
kernel_name_ = common::AnfAlgo::GetCNodeName(kernel_node);
|
||||
input_shape_a_ = AnfAlgo::GetInputDeviceShape(kernel_node, 0);
|
||||
input_shape_b_ = AnfAlgo::GetInputDeviceShape(kernel_node, 1);
|
||||
|
|
|
@ -30,6 +30,7 @@ constexpr size_t kIndexGrad = 2;
|
|||
|
||||
void MaskedSelectGradCpuKernelMod::InitKernel(const CNodePtr &kernel_node) {
|
||||
MS_EXCEPTION_IF_NULL(kernel_node);
|
||||
tensor_size_ = 1;
|
||||
kernel_name_ = common::AnfAlgo::GetCNodeName(kernel_node);
|
||||
input_shape_a_ = AnfAlgo::GetInputDeviceShape(kernel_node, kIndexInput);
|
||||
input_shape_b_ = AnfAlgo::GetInputDeviceShape(kernel_node, kIndexMask);
|
||||
|
|
|
@ -20,10 +20,18 @@
|
|||
#include "utils/ms_utils.h"
|
||||
|
||||
namespace mindspore {
|
||||
static std::string MakeId() {
|
||||
// Use atomic to make id generator thread safe.
|
||||
static std::atomic<uint64_t> last_id{1};
|
||||
return "C" + std::to_string(last_id.fetch_add(1, std::memory_order_relaxed));
|
||||
}
|
||||
|
||||
using mindspore::abstract::AbstractFunction;
|
||||
|
||||
abstract::AbstractBasePtr Cell::ToAbstract() { return nullptr; }
|
||||
|
||||
Cell::Cell(const std::string &name) : Named(name), id_(MakeId()) {}
|
||||
|
||||
bool Cell::operator==(const Value &other) const {
|
||||
if (other.isa<Cell>()) {
|
||||
auto other_prim = static_cast<const Cell &>(other);
|
||||
|
|
|
@ -37,13 +37,18 @@ class MS_CORE_API Cell final : public Named {
|
|||
/// \brief Constructor.
|
||||
///
|
||||
/// \param[in] name The name of Cell.
|
||||
explicit Cell(const std::string &name) : Named(name) {}
|
||||
explicit Cell(const std::string &name);
|
||||
MS_DECLARE_PARENT(Cell, Named);
|
||||
|
||||
abstract::AbstractBasePtr ToAbstract() override;
|
||||
|
||||
std::string ToString() const override;
|
||||
|
||||
/// \brief Get the id of this Cell.
|
||||
///
|
||||
/// \return The id of this Cell.
|
||||
string id() const { return id_; }
|
||||
|
||||
/// \brief Get information about all attributes.
|
||||
///
|
||||
/// \return Details of all attributes.
|
||||
|
@ -110,6 +115,7 @@ class MS_CORE_API Cell final : public Named {
|
|||
~Cell() override = default;
|
||||
|
||||
private:
|
||||
string id_;
|
||||
mindspore::HashMap<std::string, ValuePtr> attrs_;
|
||||
enum MixedPrecisionType mixed_type_ { kNotSet };
|
||||
};
|
||||
|
|
|
@ -403,6 +403,7 @@ class _MindsporeFunctionExecutor:
|
|||
self.input_signature.append(args_list[-1])
|
||||
Validator.check_dynamic_shape(self.input_signature, args_list)
|
||||
compile_args = tuple(self.input_signature)
|
||||
_pynative_executor.set_dynamic_input(self.obj, *compile_args)
|
||||
return compile_args
|
||||
|
||||
|
||||
|
@ -767,17 +768,17 @@ class _PynativeExecutor:
|
|||
"""
|
||||
self._executor.grad_net(grad, obj, weights, grad_position, *args, *(kwargs.values()))
|
||||
|
||||
def del_cell(self, cell_id=""):
|
||||
def del_cell(self, obj):
|
||||
"""
|
||||
Clean resource for cell.
|
||||
|
||||
Args:
|
||||
cell_id (str): The ID of cell object.
|
||||
obj (Function/Cell): The function or cell instance.
|
||||
|
||||
Return:
|
||||
None.
|
||||
"""
|
||||
self._executor.clear_cell(cell_id)
|
||||
self._executor.clear_cell(obj)
|
||||
|
||||
def clear_res(self):
|
||||
"""
|
||||
|
|
|
@ -325,7 +325,7 @@ class Cell(Cell_):
|
|||
|
||||
def __del__(self):
|
||||
if context.get_context is not None and context._get_mode() == context.PYNATIVE_MODE:
|
||||
_pynative_executor.del_cell(str(id(self)))
|
||||
_pynative_executor.del_cell(self)
|
||||
|
||||
# while deepcopy a cell instance, the copied cell instance can't be added to cells_compile_cache
|
||||
# here using pop(id(self), None) to avoid KeyError exception
|
||||
|
@ -588,10 +588,8 @@ class Cell(Cell_):
|
|||
if self.requires_grad:
|
||||
_pynative_executor.set_grad_flag(True)
|
||||
|
||||
# PyNative feed dynamic shape inputs
|
||||
if self._dynamic_shape_inputs is not None:
|
||||
self._check_compile_dynamic_shape(*args)
|
||||
_pynative_executor.set_dynamic_input(self, *self._dynamic_shape_inputs)
|
||||
|
||||
try:
|
||||
_pynative_executor.new_graph(self, *args, **kwargs)
|
||||
|
@ -900,6 +898,9 @@ class Cell(Cell_):
|
|||
for ele in self._dynamic_shape_inputs:
|
||||
if isinstance(ele, (str, int, dict)):
|
||||
raise TypeError(f"For element in 'set_inputs', the type must be Tensor, but got {type(ele)}.")
|
||||
if context._get_mode() == context.PYNATIVE_MODE:
|
||||
_pynative_executor.set_dynamic_input(self, *self._dynamic_shape_inputs)
|
||||
|
||||
|
||||
def get_inputs(self):
|
||||
"""
|
||||
|
|
|
@ -0,0 +1,228 @@
|
|||
# Copyright 2022 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
|
||||
import platform
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import mindspore as ms
|
||||
from mindspore import nn
|
||||
from mindspore import ops
|
||||
from mindspore import context, Tensor
|
||||
|
||||
context.set_context(mode=context.PYNATIVE_MODE)
|
||||
|
||||
|
||||
class NetInner(nn.Cell):
|
||||
def __init__(self):
|
||||
super(NetInner, self).__init__()
|
||||
self.log = ops.Log()
|
||||
self.exp = ops.Exp()
|
||||
self.addn = ops.AddN()
|
||||
self.relu = nn.ReLU()
|
||||
|
||||
def construct(self, x, y):
|
||||
x = self.addn((x, y))
|
||||
x = self.log(x)
|
||||
x = self.exp(x)
|
||||
x = self.relu(x)
|
||||
x = self.addn((x, y))
|
||||
return x
|
||||
|
||||
|
||||
class Net(nn.Cell):
|
||||
def __init__(self):
|
||||
super(Net, self).__init__()
|
||||
self.log = ops.Log()
|
||||
self.exp = ops.Exp()
|
||||
self.addn = ops.AddN()
|
||||
self.relu = nn.ReLU()
|
||||
self.inner = NetInner()
|
||||
|
||||
def construct(self, x, y):
|
||||
x = self.addn((x, y))
|
||||
x = self.inner(x, y)
|
||||
x = self.log(x)
|
||||
x = self.exp(x)
|
||||
x = self.relu(x)
|
||||
return x
|
||||
|
||||
|
||||
class CmpNetInner(nn.Cell):
|
||||
def __init__(self):
|
||||
super(CmpNetInner, self).__init__()
|
||||
self.log = ops.Log()
|
||||
self.exp = ops.Exp()
|
||||
self.addn = ops.AddN()
|
||||
self.relu = nn.ReLU()
|
||||
|
||||
def construct(self, x, y):
|
||||
x = self.addn((x, y))
|
||||
x = self.log(x)
|
||||
x = self.exp(x)
|
||||
x = self.relu(x)
|
||||
x = self.addn((x, y))
|
||||
return x
|
||||
|
||||
|
||||
class CmpNet(nn.Cell):
|
||||
def __init__(self):
|
||||
super(CmpNet, self).__init__()
|
||||
self.log = ops.Log()
|
||||
self.exp = ops.Exp()
|
||||
self.addn = ops.AddN()
|
||||
self.relu = nn.ReLU()
|
||||
self.inner = CmpNetInner()
|
||||
|
||||
def construct(self, x, y):
|
||||
x = self.addn((x, y))
|
||||
x = self.inner(x, y)
|
||||
x = self.log(x)
|
||||
x = self.exp(x)
|
||||
x = self.relu(x)
|
||||
return x
|
||||
|
||||
|
||||
@pytest.mark.level0
|
||||
@pytest.mark.platform_x86_cpu
|
||||
@pytest.mark.platform_arm_ascend_training
|
||||
@pytest.mark.platform_x86_ascend_training
|
||||
@pytest.mark.platform_x86_gpu_training
|
||||
@pytest.mark.env_onecard
|
||||
def test_pynative_auto_dynamic_shape_with_three_static_shape():
|
||||
"""
|
||||
Feature: PyNative auto dynamic shape.
|
||||
Description: The static shape is automatically converted to a dynamic shape.
|
||||
Expectation: The calculation result is correct.
|
||||
"""
|
||||
if platform.system() == 'Windows':
|
||||
return
|
||||
|
||||
net = Net()
|
||||
grad_op = ops.GradOperation(get_all=True, get_by_list=False, sens_param=True)
|
||||
|
||||
# run first shape
|
||||
input_x = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 2)
|
||||
input_y = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 5)
|
||||
out = net(input_x, input_y)
|
||||
_ = grad_op(net)(input_x, input_y, out)
|
||||
|
||||
# run second shape
|
||||
input_x2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 2)
|
||||
input_y2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 5)
|
||||
out = net(input_x2, input_y2)
|
||||
_ = grad_op(net)(input_x2, input_y2, out)
|
||||
|
||||
# run third shape
|
||||
input_x3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 2)
|
||||
input_y3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 5)
|
||||
out = net(input_x3, input_y3)
|
||||
grad = grad_op(net)(input_x3, input_y3, out)
|
||||
|
||||
cmp_net = CmpNet()
|
||||
cmp_out = cmp_net(input_x3, input_y3)
|
||||
cmp_grad = grad_op(cmp_net)(input_x3, input_y3, cmp_out)
|
||||
assert np.allclose(grad[0].asnumpy(), cmp_grad[0].asnumpy(), 0.00001, 0.00001)
|
||||
assert np.allclose(grad[1].asnumpy(), cmp_grad[1].asnumpy(), 0.00001, 0.00001)
|
||||
|
||||
|
||||
@pytest.mark.level0
|
||||
@pytest.mark.platform_x86_cpu
|
||||
@pytest.mark.platform_arm_ascend_training
|
||||
@pytest.mark.platform_x86_ascend_training
|
||||
@pytest.mark.platform_x86_gpu_training
|
||||
@pytest.mark.env_onecard
|
||||
def test_pynative_auto_dynamic_shape_mixing_static_shape_and_dynamic_shape_1():
|
||||
"""
|
||||
Feature: PyNative auto dynamic shape.
|
||||
Description: Mixing static shape and dynamic shape.
|
||||
Expectation: The calculation result is correct.
|
||||
"""
|
||||
if platform.system() == 'Windows':
|
||||
return
|
||||
|
||||
net = Net()
|
||||
grad_op = ops.GradOperation(get_all=True, get_by_list=False, sens_param=True)
|
||||
|
||||
# run first shape
|
||||
input_x = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 2)
|
||||
input_y = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 5)
|
||||
out = net(input_x, input_y)
|
||||
_ = grad_op(net)(input_x, input_y, out)
|
||||
|
||||
# run second shape
|
||||
input_x2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 2)
|
||||
input_y2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 5)
|
||||
net.set_inputs(Tensor(shape=[2, 3, 6, None], dtype=ms.float32),
|
||||
Tensor(shape=[2, 3, None, None], dtype=ms.float32))
|
||||
out = net(input_x2, input_y2)
|
||||
_ = grad_op(net)(input_x2, input_y2, out)
|
||||
|
||||
# run third shape
|
||||
input_x3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 2)
|
||||
input_y3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 5)
|
||||
out = net(input_x3, input_y3)
|
||||
grad = grad_op(net)(input_x3, input_y3, out)
|
||||
|
||||
cmp_net = CmpNet()
|
||||
cmp_out = cmp_net(input_x3, input_y3)
|
||||
cmp_grad = grad_op(cmp_net)(input_x3, input_y3, cmp_out)
|
||||
assert np.allclose(grad[0].asnumpy(), cmp_grad[0].asnumpy(), 0.00001, 0.00001)
|
||||
assert np.allclose(grad[1].asnumpy(), cmp_grad[1].asnumpy(), 0.00001, 0.00001)
|
||||
|
||||
|
||||
@pytest.mark.level0
|
||||
@pytest.mark.platform_x86_cpu
|
||||
@pytest.mark.platform_arm_ascend_training
|
||||
@pytest.mark.platform_x86_ascend_training
|
||||
@pytest.mark.platform_x86_gpu_training
|
||||
@pytest.mark.env_onecard
|
||||
def test_pynative_auto_dynamic_shape_mixing_static_shape_and_dynamic_shape_2():
|
||||
"""
|
||||
Feature: PyNative auto dynamic shape.
|
||||
Description: Mixing static shape and dynamic shape.
|
||||
Expectation: The calculation result is correct.
|
||||
"""
|
||||
if platform.system() == 'Windows':
|
||||
return
|
||||
|
||||
net = Net()
|
||||
grad_op = ops.GradOperation(get_all=True, get_by_list=False, sens_param=True)
|
||||
|
||||
# run first shape
|
||||
input_x = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 2)
|
||||
input_y = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 5)
|
||||
net.set_inputs(Tensor(shape=[2, 3, 6, None], dtype=ms.float32),
|
||||
Tensor(shape=[2, 3, None, None], dtype=ms.float32))
|
||||
out = net(input_x, input_y)
|
||||
_ = grad_op(net)(input_x, input_y, out)
|
||||
|
||||
# run second shape
|
||||
input_x2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 2)
|
||||
input_y2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 5)
|
||||
out = net(input_x2, input_y2)
|
||||
_ = grad_op(net)(input_x2, input_y2, out)
|
||||
|
||||
# run third shape
|
||||
input_x3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 2)
|
||||
input_y3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 5)
|
||||
out = net(input_x3, input_y3)
|
||||
grad = grad_op(net)(input_x3, input_y3, out)
|
||||
|
||||
cmp_net = CmpNet()
|
||||
cmp_out = cmp_net(input_x3, input_y3)
|
||||
cmp_grad = grad_op(cmp_net)(input_x3, input_y3, cmp_out)
|
||||
assert np.allclose(grad[0].asnumpy(), cmp_grad[0].asnumpy(), 0.00001, 0.00001)
|
||||
assert np.allclose(grad[1].asnumpy(), cmp_grad[1].asnumpy(), 0.00001, 0.00001)
|
|
@ -0,0 +1,202 @@
|
|||
# Copyright 2022 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
|
||||
import platform
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from mindspore import nn
|
||||
from mindspore import ops
|
||||
from mindspore import context, Tensor
|
||||
from mindspore import ms_function
|
||||
|
||||
context.set_context(mode=context.PYNATIVE_MODE)
|
||||
|
||||
|
||||
class NetInner(nn.Cell):
|
||||
def __init__(self):
|
||||
super(NetInner, self).__init__()
|
||||
self.log = ops.Log()
|
||||
self.exp = ops.Exp()
|
||||
self.addn = ops.AddN()
|
||||
self.relu = nn.ReLU()
|
||||
|
||||
@ms_function
|
||||
def construct(self, x, y):
|
||||
x = self.addn((x, y))
|
||||
x = self.log(x)
|
||||
x = self.exp(x)
|
||||
x = self.relu(x)
|
||||
x = self.addn((x, y))
|
||||
return x
|
||||
|
||||
|
||||
class Net(nn.Cell):
|
||||
def __init__(self):
|
||||
super(Net, self).__init__()
|
||||
self.log = ops.Log()
|
||||
self.exp = ops.Exp()
|
||||
self.addn = ops.AddN()
|
||||
self.relu = nn.ReLU()
|
||||
self.inner = NetInner()
|
||||
|
||||
def construct(self, x, y):
|
||||
x = self.addn((x, y))
|
||||
x = self.inner(x, y)
|
||||
x = self.log(x)
|
||||
x = self.exp(x)
|
||||
x = self.relu(x)
|
||||
return x
|
||||
|
||||
|
||||
class NetOuter(nn.Cell):
|
||||
def __init__(self):
|
||||
super(NetOuter, self).__init__()
|
||||
self.log = ops.Log()
|
||||
self.exp = ops.Exp()
|
||||
self.addn = ops.AddN()
|
||||
self.relu = nn.ReLU()
|
||||
self.inner = NetInner()
|
||||
|
||||
@ms_function
|
||||
def construct(self, x, y):
|
||||
x = self.addn((x, y))
|
||||
x = self.inner(x, y)
|
||||
x = self.log(x)
|
||||
x = self.exp(x)
|
||||
x = self.relu(x)
|
||||
return x
|
||||
|
||||
|
||||
class CmpNetInner(nn.Cell):
|
||||
def __init__(self):
|
||||
super(CmpNetInner, self).__init__()
|
||||
self.log = ops.Log()
|
||||
self.exp = ops.Exp()
|
||||
self.addn = ops.AddN()
|
||||
self.relu = nn.ReLU()
|
||||
|
||||
@ms_function
|
||||
def construct(self, x, y):
|
||||
x = self.addn((x, y))
|
||||
x = self.log(x)
|
||||
x = self.exp(x)
|
||||
x = self.relu(x)
|
||||
x = self.addn((x, y))
|
||||
return x
|
||||
|
||||
|
||||
class CmpNet(nn.Cell):
|
||||
def __init__(self):
|
||||
super(CmpNet, self).__init__()
|
||||
self.log = ops.Log()
|
||||
self.exp = ops.Exp()
|
||||
self.addn = ops.AddN()
|
||||
self.relu = nn.ReLU()
|
||||
self.inner = CmpNetInner()
|
||||
|
||||
def construct(self, x, y):
|
||||
x = self.addn((x, y))
|
||||
x = self.inner(x, y)
|
||||
x = self.log(x)
|
||||
x = self.exp(x)
|
||||
x = self.relu(x)
|
||||
return x
|
||||
|
||||
|
||||
@pytest.mark.level0
|
||||
@pytest.mark.platform_x86_cpu
|
||||
@pytest.mark.platform_arm_ascend_training
|
||||
@pytest.mark.platform_x86_ascend_training
|
||||
@pytest.mark.platform_x86_gpu_training
|
||||
@pytest.mark.env_onecard
|
||||
def test_pynative_auto_dynamic_shape_with_inner_ms_function():
|
||||
"""
|
||||
Feature: PyNative auto dynamic shape.
|
||||
Description: The static shape is automatically converted to a dynamic shape.
|
||||
Expectation: The calculation result is correct.
|
||||
"""
|
||||
if platform.system() == 'Windows':
|
||||
return
|
||||
|
||||
net = Net()
|
||||
grad_op = ops.GradOperation(get_all=True, get_by_list=False, sens_param=True)
|
||||
|
||||
# run first shape
|
||||
input_x = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 2)
|
||||
input_y = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 5)
|
||||
out = net(input_x, input_y)
|
||||
_ = grad_op(net)(input_x, input_y, out)
|
||||
|
||||
# run second shape
|
||||
input_x2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 2)
|
||||
input_y2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 5)
|
||||
out = net(input_x2, input_y2)
|
||||
_ = grad_op(net)(input_x2, input_y2, out)
|
||||
|
||||
# run third shape
|
||||
input_x3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 2)
|
||||
input_y3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 5)
|
||||
out = net(input_x3, input_y3)
|
||||
grad = grad_op(net)(input_x3, input_y3, out)
|
||||
|
||||
cmp_net = CmpNet()
|
||||
cmp_out = cmp_net(input_x3, input_y3)
|
||||
cmp_grad = grad_op(cmp_net)(input_x3, input_y3, cmp_out)
|
||||
assert np.allclose(grad[0].asnumpy(), cmp_grad[0].asnumpy(), 0.00001, 0.00001)
|
||||
assert np.allclose(grad[1].asnumpy(), cmp_grad[1].asnumpy(), 0.00001, 0.00001)
|
||||
|
||||
|
||||
@pytest.mark.level0
|
||||
@pytest.mark.platform_x86_cpu
|
||||
@pytest.mark.platform_arm_ascend_training
|
||||
@pytest.mark.platform_x86_ascend_training
|
||||
@pytest.mark.platform_x86_gpu_training
|
||||
@pytest.mark.env_onecard
|
||||
def test_pynative_auto_dynamic_shape_with_outer_ms_function():
|
||||
"""
|
||||
Feature: PyNative auto dynamic shape.
|
||||
Description: The static shape is automatically converted to a dynamic shape.
|
||||
Expectation: The calculation result is correct.
|
||||
"""
|
||||
if platform.system() == 'Windows':
|
||||
return
|
||||
|
||||
net = NetOuter()
|
||||
grad_op = ops.GradOperation(get_all=True, get_by_list=False, sens_param=True)
|
||||
|
||||
# run first shape
|
||||
input_x = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 2)
|
||||
input_y = Tensor(np.random.rand(2, 3, 6, 8).astype(np.float32) * 5)
|
||||
out = net(input_x, input_y)
|
||||
_ = grad_op(net)(input_x, input_y, out)
|
||||
|
||||
# run second shape
|
||||
input_x2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 2)
|
||||
input_y2 = Tensor(np.random.rand(2, 3, 6, 16).astype(np.float32) * 5)
|
||||
out = net(input_x2, input_y2)
|
||||
_ = grad_op(net)(input_x2, input_y2, out)
|
||||
|
||||
# run third shape
|
||||
input_x3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 2)
|
||||
input_y3 = Tensor(np.random.rand(2, 3, 6, 34).astype(np.float32) * 5)
|
||||
out = net(input_x3, input_y3)
|
||||
grad = grad_op(net)(input_x3, input_y3, out)
|
||||
|
||||
cmp_net = CmpNet()
|
||||
cmp_out = cmp_net(input_x3, input_y3)
|
||||
cmp_grad = grad_op(cmp_net)(input_x3, input_y3, cmp_out)
|
||||
assert np.allclose(grad[0].asnumpy(), cmp_grad[0].asnumpy(), 0.00001, 0.00001)
|
||||
assert np.allclose(grad[1].asnumpy(), cmp_grad[1].asnumpy(), 0.00001, 0.00001)
|
Loading…
Reference in New Issue