diff --git a/mindspore/lite/src/extendrt/CMakeLists.txt b/mindspore/lite/src/extendrt/CMakeLists.txt index e45075e2dbc..e4135359a32 100644 --- a/mindspore/lite/src/extendrt/CMakeLists.txt +++ b/mindspore/lite/src/extendrt/CMakeLists.txt @@ -60,6 +60,11 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE OR MSLITE_ENABLE_CLOUD_INFERENCE) ${CMAKE_CURRENT_SOURCE_DIR}/graph_executor/mindrt_graph_executor.cc ${CMAKE_CURRENT_SOURCE_DIR}/execution_flow.cc ${CMAKE_CURRENT_SOURCE_DIR}/execution_plan.cc + ${CMAKE_CURRENT_SOURCE_DIR}/graph_compiler/compile_result.cc + ${CMAKE_CURRENT_SOURCE_DIR}/graph_compiler/single_graph_compiler.cc + ${CMAKE_CURRENT_SOURCE_DIR}/graph_compiler/single_graph_scheduler.cc + ${CMAKE_CURRENT_SOURCE_DIR}/graph_compiler/compile_result_builder.cc + ${CMAKE_CURRENT_SOURCE_DIR}/graph_compiler/anfnode_tensor_adapter.cc ) if(MSLITE_ENABLE_BFC_MEMORY) set(MSLITE_EXTEND_RUNTIME_SRC ${MSLITE_EXTEND_RUNTIME_SRC} @@ -140,9 +145,10 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE OR MSLITE_ENABLE_CLOUD_INFERENCE) ${CCSRC_DIR}/utils/signal_util.cc ${CORE_DIR}/utils/status.cc ) + add_library(mindspore-kernel-graph OBJECT ${KERNEL_GRAPH_SRC}) - add_library(mindspore-extendrt SHARED ${MSLITE_EXTEND_RUNTIME_SRC}) + add_library(mindspore-extendrt SHARED ${MSLITE_EXTEND_RUNTIME_SRC} ${MSLITE_EXTEND_CPU_RUNTIME_SRC}) add_dependencies(mindspore-extendrt fbs_inner_src) add_dependencies(mindspore-extendrt mindspore-infer-anfalgo) add_dependencies(mindspore-extendrt mindspore-kernel-graph) @@ -260,6 +266,9 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE OR MSLITE_ENABLE_CLOUD_INFERENCE) add_dependencies(cloud_infer_test mindspore-extendrt) target_link_libraries(cloud_infer_test mindspore-extendrt) endif() + add_executable(unify-cpu-infer graph_compiler/main.cc) + add_dependencies(unify-cpu-infer mindspore-extendrt) + target_link_libraries(unify-cpu-infer mindspore-extendrt) else() set(MSLITE_EXTEND_RUNTIME_SRC ${MODEL_LOADER_FRAMEWORK_SRC}) add_library(mindspore-extendrt OBJECT ${MSLITE_EXTEND_RUNTIME_SRC}) @@ -269,4 +278,3 @@ else() endif() set_target_properties(mindspore-extendrt PROPERTIES OUTPUT_NAME "mindspore-lite") - diff --git a/mindspore/lite/src/extendrt/execution_flow.cc b/mindspore/lite/src/extendrt/execution_flow.cc index a8d9dad1984..b403856252b 100644 --- a/mindspore/lite/src/extendrt/execution_flow.cc +++ b/mindspore/lite/src/extendrt/execution_flow.cc @@ -16,28 +16,19 @@ #include -#include "extendrt/execution_flow.h" +#include "src/extendrt/execution_flow.h" -#include "litert/lite_kernel.h" -#include "litert/kernel_exec_util.h" -#include "litert/sub_graph_kernel.h" +#include "src/extendrt/tensor.h" +#include "src/litert/kernel_exec_util.h" +#include "src/litert/sub_graph_kernel.h" namespace mindspore::infer { ExecutionFlow::~ExecutionFlow() { - for (auto tensor : inputs_) { - if (tensor != nullptr) { - delete tensor; - } - } - for (auto tensor : outputs_) { - if (tensor != nullptr) { - delete tensor; - } + for (auto tensor : tensors_) { + delete tensor; } for (auto kernel : kernels_) { - if (kernel != nullptr) { - delete kernel; - } + delete kernel; } } diff --git a/mindspore/lite/src/extendrt/execution_flow.h b/mindspore/lite/src/extendrt/execution_flow.h index c662723c21b..f8668acb99b 100644 --- a/mindspore/lite/src/extendrt/execution_flow.h +++ b/mindspore/lite/src/extendrt/execution_flow.h @@ -26,11 +26,11 @@ namespace mindspore::infer { class ExecutionFlow : public abstract::ExecutionFlow { public: ExecutionFlow() = default; - virtual ~ExecutionFlow(); + ~ExecutionFlow() override; std::vector GetKernels() override { return kernels_; } - void SetKernels(std::vector kernels) { kernels_ = kernels; } + void SetKernels(const std::vector &kernels) override { kernels_ = kernels; } std::vector GetInputs() override { return inputs_; } @@ -58,10 +58,12 @@ class ExecutionFlow : public abstract::ExecutionFlow { std::vector kernels_; std::vector inputs_; std::vector outputs_; + std::vector tensors_; std::shared_ptr context_; abstract::KernelCallBack before_; abstract::KernelCallBack after_; }; +using ExecutionFlowPtr = std::shared_ptr; } // namespace mindspore::infer #endif // MINDSPORE_LITE_SRC_EXTENDRT_EXECUTION_FLOW_H_ diff --git a/mindspore/lite/src/extendrt/graph_compiler.cc b/mindspore/lite/src/extendrt/graph_compiler.cc index 57f9b5515c0..2b79631d1cb 100644 --- a/mindspore/lite/src/extendrt/graph_compiler.cc +++ b/mindspore/lite/src/extendrt/graph_compiler.cc @@ -17,28 +17,8 @@ namespace mindspore { namespace infer { -ExcutionPlan GraphCompiler::Compile(FuncGraphPtr func_graph) { - // Split graph to segments. - const auto &segments = graph_partition_->Partition(func_graph, &contain_multi_target); - MS_LOG(INFO) << "Compile graph: " << func_graph->ToString() << ", Split segments size:" << segments.size(); - - // Foreach the segments to compile graph. - for (const auto &segment : segments) { - CompileSegment(segment); - } - - CompileResult result = LinkSegments(); - return Schedule(result); -} -GraphId GraphCompiler::CompileSegment(const GraphSegmentPtr &segment) { - // Generate kernel graph. - auto nodes = segment->nodes_; - KernelGraphPtr graph = KernelGraphUtils::Instance()->ConstructKernelGraph(nodes, outputs, device_terget); - MS_EXCEPTION_IF_NULL(graph); - opt::EliminateIllegalDataTypePass(graph); - SetGraphDependency(graph, segment); - return graph->GraphId(); -} +ExcutionPlan GraphCompiler::Compile(FuncGraphPtr func_graph) { return {}; } +GraphId GraphCompiler::CompileSegment(const GraphSegmentPtr segment) { return -1; } CompileResult GraphCompiler::LinkSegment() { return CompileResult(); } ExcutionPlan GraphCompiler::Schedule(const CompileResult &compile_result) { return scheduler_.Schedule(compile_result); diff --git a/mindspore/lite/src/extendrt/graph_compiler.h b/mindspore/lite/src/extendrt/graph_compiler.h index 059e3839793..08c0449b571 100644 --- a/mindspore/lite/src/extendrt/graph_compiler.h +++ b/mindspore/lite/src/extendrt/graph_compiler.h @@ -23,45 +23,34 @@ #include "include/api/model.h" #include "include/api/graph.h" #include "include/api/status.h" +#include "include/api/kernel.h" #include "include/common/utils/utils.h" #include "ir/func_graph.h" +#include "src/extendrt/graph_scheduler.h" + namespace mindspore { namespace infer { +using GraphId = uint32_t; struct CompileOptions { int optimize_level_; }; -struct CompileResult { - GraphId root_; - std::vector control_nodes_; - std::vector graphs_; -}; -struct KernelInfo { - KernelPtr kernel_; - bool isSubgraphKernel; - std::vector inputs_; - std::vector outputs_; -}; -struct ExcutionPlan { - std::vector kernels_; - std::vector inputs_; - std::vector outputs_; -}; +struct CompileResult {}; + class GraphCompiler : public std::enable_shared_from_this { public: explicit GraphCompiler(const CompileOptions &opts); virtual ~GraphCompiler(); ExcutionPlan Compile(FuncGraphPtr graph); + ExcutionPlan Compile(GraphSegmentPtr segment); protected: ExcutionPlan Schedule(const CompileResult &); - GraphId CompileSegment(const GraphSegmentPtr &segment); + GraphId CompileSegment(const GraphSegmentPtr segment); CompileResult LinkSegment(); protected: - GraphPartition graph_partition_; GraphScheduler scheduler_; CompileOptions options_; - mindspore::HashMap> graphs_; }; } // namespace infer } // namespace mindspore diff --git a/mindspore/lite/src/extendrt/graph_compiler/anfnode_tensor_adapter.cc b/mindspore/lite/src/extendrt/graph_compiler/anfnode_tensor_adapter.cc new file mode 100644 index 00000000000..f218f164872 --- /dev/null +++ b/mindspore/lite/src/extendrt/graph_compiler/anfnode_tensor_adapter.cc @@ -0,0 +1,349 @@ +/** + * Copyright 2023 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/extendrt/graph_compiler/anfnode_tensor_adapter.h" +#include "src/extendrt/graph_compiler/compile_result_builder.h" +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "ir/primitive.h" +#include "utils/ms_utils_secure.h" + +using ShapePtr = mindspore::abstract::ShapePtr; +using AbstractBasePtr = mindspore::abstract::AbstractBasePtr; +using AbstractTensorPtr = mindspore::abstract::AbstractTensorPtr; + +namespace mindspore { +namespace infer { +Tensor *TensorAdapter::Convert2Tensor(const ParameterPtr ¶m_node, const std::string &tensor_name) { + auto adapter = TensorAdapter::Create(param_node); + if (adapter == nullptr) { + MS_LOG(ERROR) << "Create tensor-adapter from parameter failed, parameter : " << param_node; + return nullptr; + } + return adapter->ToTensor(tensor_name); +} + +Tensor *TensorAdapter::Convert2Tensor(const ValueNodePtr &value_node, const std::string &tensor_name) { + auto adapter = TensorAdapter::Create(value_node); + if (adapter == nullptr) { + MS_LOG(ERROR) << "Create tensor-adapter from value-node failed, value-node : " << value_node; + return nullptr; + } + return adapter->ToTensor(tensor_name); +} + +Tensor *TensorAdapter::Convert2Tensor(const AbstractTensorPtr &abstract, const std::string &tensor_name) { + auto adapter = TensorAdapter::Create(abstract); + if (adapter == nullptr) { + MS_LOG(ERROR) << "Create tensor-adapter from abstracttensor failed, abstract : " << abstract; + return nullptr; + } + return adapter->ToTensor(tensor_name); +} + +Tensor *TensorAdapter::Convert2Tensor(const AbstractBasePtr &abstract, const std::string &tensor_name) { + auto adapter = TensorAdapter::Create(abstract); + if (adapter == nullptr) { + MS_LOG(ERROR) << "Create tensor-adapter from abstractbase failed, abstract : " << abstract; + return nullptr; + } + return adapter->ToTensor(tensor_name); +} + +Tensor *TensorAdapter::ToTensor(const std::string &tensor_name) { + std::vector int32_shape(shape_.size()); + for (size_t i = 0; i < shape_.size(); i++) { + int32_shape[i] = static_cast(shape_[i]); + } + auto tensor = Tensor::CreateTensor(tensor_name, data_type_, int32_shape, data_, data_len_); + if (tensor == nullptr) { + return nullptr; + } + // move data to tensor + tensor->set_own_data(own_data_); + own_data_ = false; + return tensor; +} + +TensorAdapterPtr TensorAdapter::Create(const ParameterPtr ¶m_node) { + if (param_node == nullptr) { + MS_LOG(ERROR) << "Input parameter is nullptr."; + return nullptr; + } + ShapeVector shape_vector; + TypeId data_type = kTypeUnknown; + auto status = GetDTAndShapeFromParameter(param_node, &data_type, &shape_vector); + if (status != kSuccess) { + MS_LOG(ERROR) << "Get data type and shape from param node failed."; + return nullptr; + } + if (data_type == kObjectTypeString) { + MS_LOG(ERROR) << "Not support kObjectTypeString type DefaultParam."; + return nullptr; + } + auto adapter = std::make_shared(); + adapter->data_type_ = data_type; + adapter->shape_ = shape_vector; + adapter->format_ = NHWC; + adapter->is_const_ = param_node->has_default(); + if (!adapter->is_const_) { + return adapter; + } + auto tensor_info = std::dynamic_pointer_cast(param_node->default_param()); + if (tensor_info == nullptr) { + MS_LOG(ERROR) << "Cast default-param to tensor failed."; + return nullptr; + } + adapter->compress_type_ = tensor_info->compression_type(); + adapter->data_ = tensor_info->data_c(); + adapter->data_len_ = tensor_info->Size(); + adapter->own_data_ = false; + return adapter; +} + +TensorAdapterPtr TensorAdapter::CreateFromTensorValueNode(const ValueNodePtr &value_node) { + auto value_abstract = value_node->abstract(); + if (value_abstract == nullptr) { + MS_LOG(ERROR) << "Abstract of value is nullptr"; + return nullptr; + } + auto adapter = TensorAdapter::Create(value_abstract); + if (adapter == nullptr) { + MS_LOG(ERROR) << "Create tensor adapter from abstract of valuenode failed, valuenode: " + << value_node->fullname_with_scope(); + return nullptr; + } + adapter->is_const_ = true; + + auto value = value_node->value(); + if (value == nullptr) { + MS_LOG(ERROR) << "Value of value-node is nullptr, " << value_node->fullname_with_scope(); + return nullptr; + } + auto data = value->cast(); + if (data == nullptr) { + MS_LOG(ERROR) << "Value of tensor-type value-node is not a Tensor, " << value_node->fullname_with_scope(); + return nullptr; + } + adapter->data_ = data->data_c(); + adapter->data_len_ = data->Size(); + adapter->own_data_ = false; + return adapter; +} + +TensorAdapterPtr TensorAdapter::CreateFromInt32ImmValue(const ValueNodePtr &value_node) { + MS_ASSERT(value_node != nullptr); + auto adapter = std::make_shared(); + adapter->is_const_ = true; + adapter->data_type_ = kNumberTypeInt32; + adapter->shape_ = {1}; + auto value = value_node->value(); + if (value == nullptr) { + MS_LOG(ERROR) << "Value of value-node is nullptr, " << value_node->fullname_with_scope(); + return nullptr; + } + auto data = GetValue(value); + adapter->data_ = &data; + adapter->data_len_ = sizeof(int32_t); + adapter->own_data_ = false; + return adapter; +} + +TensorAdapterPtr TensorAdapter::CreateFromInt64ImmValue(const ValueNodePtr &value_node) { + MS_ASSERT(value_node != nullptr); + auto adapter = std::make_shared(); + adapter->is_const_ = true; + adapter->data_type_ = kNumberTypeInt64; + adapter->shape_ = {1}; + auto value = value_node->value(); + if (value == nullptr) { + MS_LOG(ERROR) << "Value of value-node is nullptr, " << value_node->fullname_with_scope(); + return nullptr; + } + auto data = GetValue(value); + adapter->data_ = &data; + adapter->data_len_ = sizeof(int64_t); + adapter->own_data_ = false; + return adapter; +} + +TensorAdapterPtr TensorAdapter::CreateFromBoolImmValue(const ValueNodePtr &value_node) { + MS_ASSERT(value_node != nullptr); + auto adapter = std::make_shared(); + adapter->is_const_ = true; + adapter->data_type_ = kNumberTypeBool; + adapter->shape_ = {1}; + auto value = value_node->value(); + if (value == nullptr) { + MS_LOG(ERROR) << "Value of value-node is nullptr, " << value_node->fullname_with_scope(); + return nullptr; + } + auto data = value->cast(); + if (data == nullptr) { + MS_LOG(ERROR) << "BoolImm Value of cast to BoolImmPtr failed, " << value_node->fullname_with_scope(); + return nullptr; + } + auto data_value = data->value(); + adapter->data_ = &data_value; + adapter->data_len_ = sizeof(bool); + adapter->own_data_ = false; + return adapter; +} + +TensorAdapterPtr TensorAdapter::CreateFromNumberTypeValue(const ValueNodePtr &value_node) { + MS_ASSERT(value_node != nullptr); + auto adapter = std::make_shared(); + adapter->is_const_ = true; + adapter->data_type_ = kNumberTypeInt32; + adapter->shape_ = {1}; + auto data = utils::cast(value_node->value()); + if (data == nullptr) { + MS_LOG(ERROR) << "Value of Number type value-node is not a NumberPtr, " << value_node->fullname_with_scope(); + return nullptr; + } + TypeId number_type = data->number_type(); + static const std::unordered_map TypeToTypeMap = { + {kNumberTypeInt, kNumberTypeInt32}, {kNumberTypeUInt, kNumberTypeUInt32}, {kNumberTypeFloat, kNumberTypeFloat32}}; + if (TypeToTypeMap.find(number_type) != TypeToTypeMap.end()) { + number_type = TypeToTypeMap.at(number_type); + } + auto number_data = static_cast(number_type); + adapter->data_ = &number_data; + adapter->data_len_ = sizeof(int32_t); + adapter->own_data_ = false; + return adapter; +} + +TensorAdapterPtr TensorAdapter::CreateFromIntSequenceValue(const ValueNodePtr &value_node) { + MS_ASSERT(value_node != nullptr); + auto value_seq = utils::cast(value_node->value()); + if (value_seq == nullptr) { + MS_LOG(ERROR) << "Value of Sequence type value-node is not a ValueSequencePtr, " + << value_node->fullname_with_scope(); + return nullptr; + } + auto adapter = std::make_shared(); + adapter->is_const_ = true; + if (!value_seq->value().empty()) { + if (value_seq->value().front()->type()->number_type() == kNumberTypeInt32 || + value_seq->value().front()->type()->number_type() == kNumberTypeInt) { + adapter->data_type_ = kNumberTypeInt32; + auto data = GetValue>(value_seq); + adapter->shape_ = {static_cast(data.size())}; + adapter->data_ = data.data(); + adapter->data_len_ = data.size() * sizeof(int32_t); + adapter->own_data_ = false; + } else if (value_seq->value().front()->type()->number_type() == kNumberTypeInt64) { + adapter->data_type_ = kNumberTypeInt64; + auto data = GetValue>(value_seq); + adapter->shape_ = {static_cast(data.size())}; + adapter->data_ = data.data(); + adapter->data_len_ = data.size() * sizeof(int64_t); + adapter->own_data_ = false; + } else { + MS_LOG(ERROR) << "only support integer value ValueSequence."; + return nullptr; + } + } + return adapter; +} + +TensorAdapterPtr TensorAdapter::Create(const ValueNodePtr &value_node) { + MS_ASSERT(value_node != nullptr); + auto value = value_node->value(); + if (value->isa()) { + return CreateFromTensorValueNode(value_node); + } else if (value->isa()) { + return CreateFromInt32ImmValue(value_node); + } else if (value->isa()) { + return CreateFromInt64ImmValue(value_node); + } else if (value->isa()) { + return CreateFromBoolImmValue(value_node); + } else if (value->isa()) { + return CreateFromIntSequenceValue(value_node); + } else if (value->isa()) { + return CreateFromNumberTypeValue(value_node); + } else { + MS_LOG(ERROR) << "Not support value type: " << value->type(); + return nullptr; + } +} + +TensorAdapterPtr TensorAdapter::Create(const AbstractBasePtr &abs) { + auto abs_tensor = utils::cast(abs); + if (abs_tensor == nullptr) { + MS_LOG(ERROR) << "Input abstract is not a AbstractTensor."; + return nullptr; + } + return TensorAdapter::Create(abs_tensor); +} + +TensorAdapterPtr TensorAdapter::Create(const AbstractTensorPtr &abs_tensor) { + if (abs_tensor == nullptr) { + MS_LOG(ERROR) << "Input abstract is not a AbstractTensor."; + return nullptr; + } + ShapeVector shape_vector; + TypeId data_type = kTypeUnknown; + auto ret = GetDTAndShapeFromAbTensor(abs_tensor, &data_type, &shape_vector); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Get data type and shape from value node failed."; + return nullptr; + } + auto adapter = std::make_shared(); + adapter->data_type_ = data_type; + adapter->shape_ = shape_vector; + return adapter; +} + +StatusCode TensorAdapter::GetDTAndShapeFromAbTensor(const AbstractTensorPtr &abstract, TypeId *data_type, + ShapeVector *shape_vector) { + MS_ASSERT(abstract != nullptr && data_type != nullptr && shape_vector != nullptr); + if (abstract->element() == nullptr) { + MS_LOG(ERROR) << "`element` of abstract is nullptr"; + return kLiteError; + } + auto type_ptr = abstract->element()->GetTypeTrack(); + if (type_ptr == nullptr) { + MS_LOG(ERROR) << "Type of abstract is nullptr"; + return kLiteError; + } + *data_type = type_ptr->type_id(); + if (!utils::isa(abstract->BuildShape())) { + MS_LOG(ERROR) << "Shape of Abstract of parameter should be ShapePtr"; + return kLiteError; + } + *shape_vector = utils::cast(abstract->BuildShape())->shape(); + return kSuccess; +} + +StatusCode TensorAdapter::GetDTAndShapeFromParameter(const ParameterPtr ¶m_node, TypeId *data_type, + ShapeVector *shape_vector) { + MS_ASSERT(param_node != nullptr && data_type != nullptr && shape_vector != nullptr); + auto abstract_base = param_node->abstract(); + if (abstract_base == nullptr) { + MS_LOG(ERROR) << "Abstract of parameter is nullptr, " << param_node->name(); + return kLiteError; + } + auto abstract_tensor = utils::cast(abstract_base); + if (abstract_tensor == nullptr) { + MS_LOG(ERROR) << "Abstract of parameter should be abstract tensor, " << param_node->name(); + return kLiteError; + } + return GetDTAndShapeFromAbTensor(abstract_tensor, data_type, shape_vector); +} +} // namespace infer +} // namespace mindspore diff --git a/mindspore/lite/src/extendrt/graph_compiler/anfnode_tensor_adapter.h b/mindspore/lite/src/extendrt/graph_compiler/anfnode_tensor_adapter.h new file mode 100644 index 00000000000..e113a80f3a2 --- /dev/null +++ b/mindspore/lite/src/extendrt/graph_compiler/anfnode_tensor_adapter.h @@ -0,0 +1,87 @@ +/** + * Copyright 2023 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_ANFNODE_TENSOR_ADAPTER_H_ +#define MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_ANFNODE_TENSOR_ADAPTER_H_ +#include +#include +#include +#include +#include "src/extendrt/tensor.h" +#include "abstract/abstract_value.h" +#include "ir/anf.h" +#include "include/api/status.h" + +namespace mindspore { +namespace infer { +class TensorAdapter; +using TensorAdapterPtr = std::shared_ptr; +class TensorAdapter { + public: + TensorAdapter() = default; + virtual ~TensorAdapter() { + if (own_data_) { + free(data_); + } + } + + Tensor *ToTensor(const std::string &tensor_name = ""); + + static TensorAdapterPtr Create(const ParameterPtr ¶m_node); + static TensorAdapterPtr Create(const ValueNodePtr &value_node); + static TensorAdapterPtr Create(const mindspore::abstract::AbstractTensorPtr &abstract); + static TensorAdapterPtr Create(const mindspore::abstract::AbstractBasePtr &abstract); + + static Tensor *Convert2Tensor(const ParameterPtr ¶m_node, const std::string &tensor_name = ""); + static Tensor *Convert2Tensor(const ValueNodePtr &value_node, const std::string &tensor_name = ""); + static Tensor *Convert2Tensor(const mindspore::abstract::AbstractTensorPtr &abstract, + const std::string &tensor_name = ""); + static Tensor *Convert2Tensor(const mindspore::abstract::AbstractBasePtr &abstract, + const std::string &tensor_name = ""); + + private: + static StatusCode GetDTAndShapeFromAbTensor(const mindspore::abstract::AbstractTensorPtr &abstract, TypeId *data_type, + ShapeVector *shape_vector); + + static StatusCode GetDTAndShapeFromParameter(const ParameterPtr ¶m_node, TypeId *data_type, + ShapeVector *shape_vector); + + static TensorAdapterPtr CreateFromTensorValueNode(const ValueNodePtr &value_node); + + static TensorAdapterPtr CreateFromInt32ImmValue(const ValueNodePtr &value_node); + + static TensorAdapterPtr CreateFromInt64ImmValue(const ValueNodePtr &value_node); + + static TensorAdapterPtr CreateFromBoolImmValue(const ValueNodePtr &value_node); + + static TensorAdapterPtr CreateFromNumberTypeValue(const ValueNodePtr &value_node); + + static TensorAdapterPtr CreateFromIntSequenceValue(const ValueNodePtr &value_node); + + public: + Format format_{DEFAULT_FORMAT}; + TensorCompressionType compress_type_{kNoCompression}; + TypeId data_type_{kTypeUnknown}; + bool is_const_{false}; + std::vector shape_{}; + void *data_{nullptr}; + size_t data_len_{0}; + bool own_data_{true}; +}; +} // namespace infer +} // namespace mindspore + +#endif diff --git a/mindspore/lite/src/extendrt/graph_compiler/compile_result.cc b/mindspore/lite/src/extendrt/graph_compiler/compile_result.cc new file mode 100644 index 00000000000..c1d015946aa --- /dev/null +++ b/mindspore/lite/src/extendrt/graph_compiler/compile_result.cc @@ -0,0 +1,376 @@ +/** + * Copyright 2023 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/extendrt/graph_compiler/compile_result.h" +#include +#include +#include +#include +#include +#include +#include +#include "src/extendrt/tensor.h" +#include "ops/base_operator.h" +#include "utils/hash_map.h" +#include "include/api/status.h" +#include "ir/primitive.h" +#include "ops/op_name.h" +#include "ops/primitive_c.h" + +namespace mindspore { +namespace infer { +namespace { +constexpr char tab[] = " "; + +inline std::string GenIndent(int indent) { + std::ostringstream oss; + for (int i = 0; i < indent; i++) { + oss << tab; + } + return oss.str(); +} + +inline std::string DumpIntShape(const std::vector &shape) { + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < shape.size(); i++) { + oss << shape[i]; + if (i < shape.size() - 1) { + oss << ", "; + } + } + oss << "]"; + return oss.str(); +} + +inline std::string DumpTensor(const Tensor *tensor, int indent = 0) { + std::ostringstream oss; + oss << GenIndent(indent) << "Tensor tensor_name() << ", dtype:" << tensor->data_type() + << ", format:" << tensor->format() << ", shape:" << DumpIntShape(tensor->shape()) << ">"; + return oss.str(); +} +} // namespace + +CompileNode *CompileNode::Create(CNodePtr cnode) { + if (cnode == nullptr) { + return nullptr; + } + auto node = new CompileNode(cnode->fullname_with_scope()); + auto primitive = GetValueNode>(cnode->input(0)); + if (primitive == nullptr) { + MS_LOG(ERROR) << "Node has no primitive, first input of cnode(" << cnode->fullname_with_scope() + << ") is : " << cnode->input(0); + return nullptr; + } + node->type_ = primitive->name(); + ops::PrimitiveCPtr primc{nullptr}; + if (utils::isa(primitive)) { + primc = utils::cast(primitive); + } else { + static auto ops_primc_fns = ops::OpPrimCRegister::GetInstance().GetPrimCMap(); + auto primc_creator_iter = ops_primc_fns.find(node->type_); + if (primc_creator_iter == ops_primc_fns.end()) { + MS_LOG(ERROR) << "Can not find primitive_c create function for: " << node->type_; + return nullptr; + } + primc = primc_creator_iter->second(); + if (primc == nullptr) { + MS_LOG(ERROR) << "Create primitive_c failed, type: " << node->type_; + return nullptr; + } + primc->SetAttrs(primitive->attrs()); + } + static auto baseops_fns = ops::OperatorRegister::GetInstance().GetOperatorMap(); + auto baseops_creator_iter = baseops_fns.find(node->type_); + if (baseops_creator_iter == baseops_fns.end()) { + MS_LOG(ERROR) << "Can not find base-operator create function for: " << node->type_; + return nullptr; + } + auto baseops_creator = baseops_creator_iter->second; + node->base_operator_ = baseops_creator(primc); + if (node->base_operator_ == nullptr) { + MS_LOG(ERROR) << "Create base-operator failed, type: " << node->type_; + return nullptr; + } + node->cnode_ = std::move(cnode); + return node; +} + +void CompileNode::AppendInputTensor(Tensor *tensor) { + if (tensor->tensor_name().empty()) { + tensor->set_tensor_name(this->name_ + "_in_" + std::to_string(this->inputs_.size())); + } + this->inputs_.emplace_back(tensor); +} + +void CompileNode::AppendOutputTensor(Tensor *tensor) { + if (tensor->tensor_name().empty()) { + tensor->set_tensor_name(this->name_ + "_out_" + std::to_string(this->inputs_.size())); + } + this->outputs_.emplace_back(tensor); +} + +std::string CompileNode::Dump(int indent) const { + constexpr int kNumberTwo = 2; + std::ostringstream oss; + oss << GenIndent(indent) << "CompileNode {" << std::endl; + oss << GenIndent(indent + 1) << "inputs: [" << std::endl; + for (auto &input : inputs_) { + oss << DumpTensor(input, indent + kNumberTwo) << std::endl; + } + oss << GenIndent(indent + 1) << "]" << std::endl; + oss << GenIndent(indent + 1) << "outputs: [" << std::endl; + for (auto &output : outputs_) { + oss << DumpTensor(output, indent + kNumberTwo) << std::endl; + } + oss << GenIndent(indent + 1) << "]" << std::endl; + oss << GenIndent(indent) << "}"; + return oss.str(); +} + +void CompileNode::ReplaceInputTensor(Tensor *dst, Tensor *src) { + std::replace_if( + inputs_.begin(), inputs_.end(), [&src](Tensor *ele) { return ele == src; }, dst); +} + +CompileNode *CompileResult::GetNode(const std::string &name) { + auto iter = node_map_.find(name); + if (iter == node_map_.end()) { + return nullptr; + } else { + return iter->second; + } +} + +CompileNode *CompileResult::GetArgNode(const std::string &name) { + auto iter = arg_node_map_.find(name); + if (iter == arg_node_map_.end()) { + return nullptr; + } else { + return iter->second; + } +} + +std::vector &CompileResult::GetMutableNodes() { + if (assembled_) { + MS_LOG(EXCEPTION) << "CompileResult not mutable after build."; + } + return nodes_; +} +std::vector &CompileResult::GetMutableInputs() { + if (assembled_) { + MS_LOG(EXCEPTION) << "CompileResult not mutable after build."; + } + return inputs_; +} + +StatusCode CompileResult::AppendNode(CompileNode *node) { + if (assembled_) { + MS_LOG(EXCEPTION) << "CompileResult not mutable after build."; + } + if (node == nullptr) { + MS_LOG(ERROR) << "Input node is nullptr"; + return kLiteInputParamInvalid; + } + const std::string &node_name = node->GetName(); + auto iter = node_map_.find(node_name); + if (iter != node_map_.end()) { + MS_LOG(ERROR) << "Duplicated node name : " << node_name; + return kLiteError; + } + node_map_[node_name] = node; + nodes_.emplace_back(node); + return kSuccess; +} + +StatusCode CompileResult::AppendArgNode(CompileNode *node) { + if (assembled_) { + MS_LOG(EXCEPTION) << "CompileResult not mutable after build."; + } + if (node == nullptr) { + MS_LOG(ERROR) << "Input node is nullptr"; + return kLiteInputParamInvalid; + } + const std::string &node_name = node->GetName(); + auto iter = arg_node_map_.find(node_name); + if (iter != arg_node_map_.end()) { + MS_LOG(ERROR) << "Duplicated node name : " << node_name; + return kLiteError; + } + arg_node_map_[node_name] = node; + arg_nodes_.emplace_back(node); + return kSuccess; +} + +StatusCode CompileResult::AppendTensor(Tensor *tensor) { + if (assembled_) { + MS_LOG(EXCEPTION) << "CompileResult not mutable after build."; + } + if (tensor == nullptr) { + MS_LOG(ERROR) << "Input tensor is nullptr"; + return kLiteInputParamInvalid; + } + const auto &tensor_name = tensor->tensor_name(); + auto iter = tensor_map_.find(tensor_name); + if (iter != tensor_map_.end()) { + MS_LOG(ERROR) << "Duplicated tensor name : " << tensor_name; + return kLiteError; + } + tensors_.emplace_back(tensor); + tensor_map_[tensor_name] = tensor; + + return kSuccess; +} + +StatusCode CompileResult::AppendInputTensor(Tensor *tensor, bool is_borrow) { + if (assembled_) { + MS_LOG(EXCEPTION) << "CompileResult not mutable after build."; + } + if (tensor == nullptr) { + MS_LOG(ERROR) << "Input tensor is nullptr"; + return kLiteInputParamInvalid; + } + if (tensor->tensor_name().empty()) { + MS_LOG(ERROR) << "Input tensor has no name"; + return kLiteError; + } + if (!is_borrow) { + return AppendTensor(tensor); + } + inputs_.emplace_back(tensor); + return kSuccess; +} + +StatusCode CompileResult::AppendOutputTensor(Tensor *tensor, bool is_borrow) { + if (assembled_) { + MS_LOG(EXCEPTION) << "CompileResult not mutable after build."; + } + if (tensor == nullptr) { + MS_LOG(ERROR) << "Input tensor is nullptr"; + return kLiteInputParamInvalid; + } + if (tensor->tensor_name().empty()) { + tensor->set_tensor_name("graph_out_" + std::to_string(this->outputs_.size())); + } + if (!is_borrow) { + return AppendTensor(tensor); + } + outputs_.emplace_back(tensor); + return kSuccess; +} + +StatusCode CompileResult::AppendNodeInputTensor(const CompileNode *compile_node, Tensor *tensor, bool is_borrow) { + if (compile_node == nullptr) { + MS_LOG(ERROR) << "Input compile_node is nullptr"; + return kLiteInputParamInvalid; + } + return AppendNodeInputTensor(compile_node->GetName(), tensor, is_borrow); +} + +StatusCode CompileResult::AppendNodeInputTensor(const std::string &node_name, Tensor *tensor, bool is_borrow) { + if (assembled_) { + MS_LOG(EXCEPTION) << "CompileResult not mutable after build."; + } + if (tensor == nullptr) { + MS_LOG(ERROR) << "Input tensor is nullptr"; + return kLiteInputParamInvalid; + } + + auto iter = node_map_.find(node_name); + if (iter == node_map_.end()) { + MS_LOG(ERROR) << "CompileNode not belong to this graph, node: " << node_name; + return kLiteError; + } + iter->second->AppendInputTensor(tensor); + if (!is_borrow) { + return AppendTensor(tensor); + } + return kSuccess; +} + +StatusCode CompileResult::AppendNodeOutputTensor(const CompileNode *compile_node, Tensor *tensor, bool is_borrow) { + if (compile_node == nullptr) { + MS_LOG(ERROR) << "Input compile_node is nullptr"; + return kLiteInputParamInvalid; + } + return AppendNodeOutputTensor(compile_node->GetName(), tensor, is_borrow); +} + +StatusCode CompileResult::AppendNodeOutputTensor(const std::string &node_name, Tensor *tensor, bool is_borrow) { + if (assembled_) { + MS_LOG(EXCEPTION) << "CompileResult not mutable after build."; + } + if (tensor == nullptr) { + MS_LOG(ERROR) << "Input tensor is nullptr"; + return kLiteInputParamInvalid; + } + + auto iter = node_map_.find(node_name); + if (iter == node_map_.end()) { + MS_LOG(ERROR) << "CompileNode not belong to this graph, node: " << node_name; + return kLiteError; + } + iter->second->AppendOutputTensor(tensor); + if (!is_borrow) { + return AppendTensor(tensor); + } + return kSuccess; +} + +std::string CompileResult::Dump(int indent) const { + constexpr int kNumberTwo = 2; + std::ostringstream oss; + oss << GenIndent(indent) << "CompileResult {" << std::endl; + oss << GenIndent(indent + 1) << "nodes: [" << std::endl; + for (auto &node : nodes_) { + oss << node->Dump(indent + kNumberTwo) << std::endl; + } + oss << GenIndent(indent + 1) << "]" << std::endl; + oss << GenIndent(indent + 1) << "inputs: [" << std::endl; + for (auto &input : inputs_) { + oss << DumpTensor(input, indent + kNumberTwo) << std::endl; + } + oss << GenIndent(indent + 1) << "]" << std::endl; + oss << GenIndent(indent + 1) << "outputs: [" << std::endl; + for (auto &output : outputs_) { + oss << DumpTensor(output, indent + kNumberTwo) << std::endl; + } + oss << GenIndent(indent + 1) << "]" << std::endl; + oss << GenIndent(indent + 1) << "tensors: [" << std::endl; + for (auto &tensor : tensors_) { + oss << DumpTensor(tensor, indent + kNumberTwo) << std::endl; + } + oss << GenIndent(indent + 1) << "]" << std::endl; + oss << GenIndent(indent) << "}" << std::endl; + return oss.str(); +} + +CompileResult::~CompileResult() { + for (auto &node : nodes_) { + delete (node); + } + nodes_.clear(); + for (auto &node : arg_nodes_) { + delete (node); + } + arg_nodes_.clear(); + for (auto &tensor : tensors_) { + delete (tensor); + } + tensors_.clear(); +} +} // namespace infer +} // namespace mindspore diff --git a/mindspore/lite/src/extendrt/graph_compiler/compile_result.h b/mindspore/lite/src/extendrt/graph_compiler/compile_result.h new file mode 100644 index 00000000000..db63262449c --- /dev/null +++ b/mindspore/lite/src/extendrt/graph_compiler/compile_result.h @@ -0,0 +1,125 @@ +/** + * Copyright 2023 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_COMPILE_RESULT_H_ +#define MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_COMPILE_RESULT_H_ +#include +#include +#include +#include +#include +#include +#include "ir/anf.h" +#include "src/extendrt/tensor.h" +#include "include/model.h" +#include "ops/base_operator.h" +#include "utils/hash_map.h" +#include "include/api/status.h" + +namespace mindspore { +namespace infer { +class CompileNode { + public: + explicit CompileNode(std::string name) : name_(std::move(name)) {} + static CompileNode *Create(CNodePtr cnode); + + virtual ~CompileNode() = default; + + std::string GetName() const { return name_; } + std::string GetType() const { return type_; } + std::shared_ptr GetBaseOperator() const { return base_operator_; } + CNodePtr GetCNode() const { return cnode_; } + const std::vector &GetInputs() const { return inputs_; } + Tensor *GetInput(size_t i) const { return inputs_.at(i); } + size_t InputSize() const { return inputs_.size(); } + const std::vector &GetOutputs() const { return outputs_; } + Tensor *GetOutput(size_t i) const { return outputs_.at(i); } + size_t OutputSize() const { return outputs_.size(); } + + void SetName(const std::string &name) { name_ = name; } + void AppendInputTensor(Tensor *tensor); + void AppendOutputTensor(Tensor *tensor); + void ReplaceInputTensor(Tensor *dst, Tensor *src); + + std::string Dump(int indent = 0) const; + + private: + std::string name_{}; + std::string type_{}; + std::shared_ptr base_operator_{nullptr}; + CNodePtr cnode_{nullptr}; + std::vector inputs_{}; + std::vector outputs_{}; +}; + +class CompileResult { + public: + explicit CompileResult(Format format) : base_format_(format) {} + + virtual ~CompileResult(); + + Format GetFormat() const { return base_format_; } + CompileNode *GetNode(const std::string &name); + CompileNode *GetArgNode(const std::string &name); + const std::vector &GetNodes() const { return nodes_; } + size_t NodeSize() const { return nodes_.size(); } + const std::vector &GetTensors() const { return tensors_; } + size_t TensorSize() const { return tensors_.size(); } + const std::vector &GetInputs() const { return inputs_; } + Tensor *GetInput(size_t i) const { return inputs_.at(i); } + size_t InputSize() const { return inputs_.size(); } + const std::vector &GetOutputs() const { return outputs_; } + Tensor *GetOutput(size_t i) const { return outputs_.at(i); } + size_t OutputSize() const { return outputs_.size(); } + const std::vector &GetParamNodes() const { return param_nodes_; } + const std::vector &GetReturnNodes() const { return return_nodes_; } + + std::vector &GetMutableNodes(); + std::vector &GetMutableInputs(); + StatusCode AppendNode(CompileNode *node); + StatusCode AppendArgNode(CompileNode *node); + StatusCode AppendTensor(Tensor *tensor); + StatusCode AppendInputTensor(Tensor *tensor, bool is_borrow = false); + StatusCode AppendOutputTensor(Tensor *tensor, bool is_borrow = false); + + StatusCode AppendNodeInputTensor(const CompileNode *compile_node, Tensor *tensor, bool is_borrow = false); + StatusCode AppendNodeInputTensor(const std::string &node_name, Tensor *tensor, bool is_borrow = false); + StatusCode AppendNodeOutputTensor(const CompileNode *compile_node, Tensor *tensor, bool is_borrow = false); + StatusCode AppendNodeOutputTensor(const std::string &node_name, Tensor *tensor, bool is_borrow = false); + + void Assemble() { this->assembled_ = true; } + + std::string Dump(int indent = 0) const; + + private: + bool assembled_ = false; + std::vector nodes_{}; + std::vector tensors_{}; + std::vector inputs_{}; + std::vector outputs_{}; + HashMap node_map_{}; + HashMap tensor_map_{}; + std::vector param_nodes_{}; + std::vector return_nodes_{}; + std::vector arg_nodes_{}; + HashMap arg_node_map_{}; + Format base_format_{DEFAULT_FORMAT}; +}; +using CompileResultPtr = std::shared_ptr; +} // namespace infer +} // namespace mindspore + +#endif diff --git a/mindspore/lite/src/extendrt/graph_compiler/compile_result_builder.cc b/mindspore/lite/src/extendrt/graph_compiler/compile_result_builder.cc new file mode 100644 index 00000000000..54d7f8ec345 --- /dev/null +++ b/mindspore/lite/src/extendrt/graph_compiler/compile_result_builder.cc @@ -0,0 +1,464 @@ +/** + * Copyright 2023 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/extendrt/graph_compiler/compile_result_builder.h" +#include +#include "src/extendrt/graph_compiler/anfnode_tensor_adapter.h" +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "ir/primitive.h" +#include "ops/core_ops.h" +#include "ops/op_name.h" +#include "ops/primitive_c.h" + +using AbstractBasePtr = mindspore::abstract::AbstractBasePtr; +using AbstractTensorPtr = mindspore::abstract::AbstractTensorPtr; +using AbstractSequencePtr = mindspore::abstract::AbstractSequencePtr; + +namespace mindspore { +namespace infer { +StatusCode CompileResultBuilder::BuildInputs(const AnfNodePtrList &inputs) { + MS_ASSERT(graph_ != nullptr); + if (graph_->InputSize() > 0) { + MS_LOG(ERROR) << "Please don't call BuildOutputs twice."; + return kLiteError; + } + std::vector results; + for (auto &input : inputs) { + results.clear(); + auto ret = CreateTensorsFromAbstract(input->abstract(), &results); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Create tensors from abstract of segments input failed, input : " + << input->fullname_with_scope(); + return ret; + } + auto arg_node = new CompileNode(input->fullname_with_scope()); + ret = graph_->AppendArgNode(arg_node); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Append input lite-node to graph failed, input : " << input->fullname_with_scope(); + return ret; + } + for (auto &result : results) { + arg_node->AppendOutputTensor(result); + ret = graph_->AppendInputTensor(result); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Append output tensor to argument node failed, node: " << input->fullname_with_scope(); + return ret; + } + } + } + return kSuccess; +} + +StatusCode CompileResultBuilder::BuildNodes(const GraphSegmentPtr &graph_segment) { + MS_ASSERT(graph_ != nullptr); + if (graph_->NodeSize() > 0) { + MS_LOG(ERROR) << "Please don't call BuildNodes twice."; + return kLiteError; + } + for (auto &node : graph_segment->nodes_) { + if (!utils::isa(node)) { + continue; + } + auto ret = CreateAndAppendNode(utils::cast(node)); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Create compile node from cnode failed : " << node; + return ret; + } + } + return kSuccess; +} + +StatusCode CompileResultBuilder::BuildOutputs(const AnfNodePtrList &outputs) { + MS_ASSERT(graph_ != nullptr); + if (graph_->OutputSize() > 0) { + MS_LOG(ERROR) << "Please don't call BuildOutputs twice."; + return kLiteError; + } + for (auto &output : outputs) { + auto out_cnode = utils::cast(output); + if (out_cnode == nullptr) { + MS_LOG(ERROR) << "Outputs should be a CNode vector, but got " << output->Type() << " type element."; + return kLiteError; + } + for (auto &input : out_cnode->inputs()) { + if (!utils::isa(input)) { + continue; + } + auto compile_node = graph_->GetNode(input->fullname_with_scope()); + if (compile_node == nullptr) { + continue; + } + for (auto &tensor : compile_node->GetOutputs()) { + auto ret = graph_->AppendOutputTensor(tensor, true); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Append output tensor to graph failed, output: " << out_cnode->fullname_with_scope(); + return ret; + } + } + } + } + return kSuccess; +} + +void CompileResultBuilder::IsolateTensor(Tensor *dst_tensor, const CompileNode *node, size_t index) { + if (node == nullptr || index >= node->OutputSize()) { + return; + } + auto *src_tensor = node->GetOutput(index); + // used as inputs of other node + auto &nodes = graph_->GetMutableNodes(); + for (auto &compile_node : nodes) { + if (compile_node == nullptr) { + continue; + } + compile_node->ReplaceInputTensor(dst_tensor, src_tensor); + } + // used as outputs of graph + auto &inputs = graph_->GetMutableInputs(); + std::replace_if( + inputs.begin(), inputs.end(), [&src_tensor](Tensor *ele) { return ele == src_tensor; }, dst_tensor); +} + +StatusCode CompileResultBuilder::RemoveMakeSeqNode() { + auto &nodes = graph_->GetMutableNodes(); + for (auto iter = nodes.begin(); iter != nodes.end();) { + auto &node = *iter; + if (node->GetType() != prim::kMakeTuple && node->GetType() != prim::kMakeList) { + iter++; + continue; + } + MS_LOG(INFO) << "Handling make sequence node: " << node->GetName(); + auto tensor_number = node->InputSize(); + if (tensor_number != node->OutputSize()) { + MS_LOG(ERROR) << "MakeSequence node should has same number of inputs and outputs, but got " << tensor_number + << " inputs and " << node->OutputSize() << " outputs."; + return kLiteError; + } + for (size_t i = 0; i < tensor_number; i++) { + IsolateTensor(node->GetInput(i), node, i); + } + iter = nodes.erase(iter); + } + return kSuccess; +} + +StatusCode CompileResultBuilder::RemoveDependNode() { + auto &nodes = graph_->GetMutableNodes(); + for (auto iter = nodes.begin(); iter != nodes.end();) { + auto &node = *iter; + if (node->GetType() != prim::kDepend) { + iter++; + continue; + } + MS_LOG(INFO) << "Handling Depend node: " << node->GetName(); + constexpr int kSize2 = 2; + if (node->InputSize() != kSize2) { + MS_LOG(ERROR) << "Depend node should has 2 inputs, but got " << node->InputSize(); + return kLiteError; + } + if (node->OutputSize() != 1) { + MS_LOG(ERROR) << "Depend node should has 1 outputs, but got " << node->OutputSize(); + return kLiteError; + } + IsolateTensor(node->GetInput(0), node, 0); + iter = nodes.erase(iter); + } + return kSuccess; +} + +StatusCode CompileResultBuilder::RemoveSeqGetItemNode() { + auto &nodes = graph_->GetMutableNodes(); + for (auto iter = nodes.begin(); iter != nodes.end();) { + auto &node = *iter; + if (node->GetType() != prim::kTupleGetItem && node->GetType() != prim::kListGetItem && + node->GetType() != "array_getitem" && node->GetType() != prim::kSliceGetItem) { + iter++; + continue; + } + MS_LOG(INFO) << "Handling GetItem node: " << node->GetName(); + constexpr int kSize2 = 2; + if (node->InputSize() != kSize2) { + MS_LOG(ERROR) << "GetItem node should has 2 inputs, but got " << node->InputSize(); + return kLiteError; + } + if (node->OutputSize() != 1) { + MS_LOG(ERROR) << "GetItem node should has 1 outputs, but got " << node->OutputSize(); + return kLiteError; + } + auto index_tensor = node->GetInput(1); + if (index_tensor->data() == nullptr) { + MS_LOG(ERROR) << "`index_tensor` of GetItem should be a const tensor, but has no data."; + return kLiteError; + } + if (index_tensor->data_type() == kNumberTypeInt32) { + auto idx = reinterpret_cast(index_tensor->data())[0]; + IsolateTensor(node->GetInput(idx), node, 0); + } else if (index_tensor->data_type() == kNumberTypeInt64) { + auto idx = reinterpret_cast(index_tensor->data())[0]; + IsolateTensor(node->GetInput(idx), node, 0); + } else { + MS_LOG(ERROR) << "`index_tensor` of GetItem should be a const tensor with int data type, but got " + << index_tensor->data_type(); + return kLiteError; + } + iter = nodes.erase(iter); + } + return kSuccess; +} + +StatusCode CompileResultBuilder::OptimizeGraph() { + MS_ASSERT(graph_ != nullptr); + auto ret = RemoveDependNode(); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Handle Depend node failed"; + return ret; + } + ret = RemoveMakeSeqNode(); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Handle Make Sequence node failed"; + return ret; + } + ret = RemoveSeqGetItemNode(); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Handle Sequence-Getitem node failed"; + return ret; + } + return kSuccess; +} + +CompileResultPtr CompileResultBuilder::Build(const GraphSegmentPtr &graph_segment, const AnfNodePtrList &inputs, + const AnfNodePtrList &outputs) { + graph_ = std::make_shared(graph_format_); + if (BuildInputs(inputs) != kSuccess) { + MS_LOG(ERROR) << "Build graph inputs failed"; + return nullptr; + } + if (BuildNodes(graph_segment) != kSuccess) { + MS_LOG(ERROR) << "Build graph nodes failed"; + return nullptr; + } + if (BuildOutputs(outputs) != kSuccess) { + MS_LOG(ERROR) << "Build graph outputs failed"; + return nullptr; + } + if (OptimizeGraph() != kSuccess) { + MS_LOG(ERROR) << "Optimize graph failed"; + return nullptr; + } + graph_->Assemble(); + return graph_; +} + +StatusCode CompileResultBuilder::AppendInputCNodeToInputs(const CNodePtr &cnode, const CompileNode *compile_node) { + if (cnode == nullptr) { + MS_LOG(ERROR) << "Input cnode is nullptr."; + return kLiteInputParamInvalid; + } + if (compile_node == nullptr) { + MS_LOG(ERROR) << "Input compile_node is nullptr."; + return kLiteInputParamInvalid; + } + auto input_node = graph_->GetNode(cnode->fullname_with_scope()); + if (input_node == nullptr) { + input_node = graph_->GetArgNode(cnode->fullname_with_scope()); + } + if (input_node == nullptr) { + MS_LOG(ERROR) << "Can not find input lite-node in graph, node: " << cnode->fullname_with_scope(); + return kLiteError; + } + for (auto &input_node_output : input_node->GetOutputs()) { + auto ret = graph_->AppendNodeInputTensor(compile_node, input_node_output, true); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Append input tensor for node failed, node: " << compile_node->GetName(); + return ret; + } + } + return kSuccess; +} + +StatusCode CompileResultBuilder::AppendInputParameterToInputs(const ParameterPtr ¶m_node, + const CompileNode *compile_node) { + if (param_node == nullptr) { + MS_LOG(ERROR) << "Input param_node is nullptr."; + return kLiteInputParamInvalid; + } + if (compile_node == nullptr) { + MS_LOG(ERROR) << "Input compile_node is nullptr."; + return kLiteInputParamInvalid; + } + auto arg_node = graph_->GetArgNode(param_node->fullname_with_scope()); + if (arg_node != nullptr) { + for (auto &output : arg_node->GetOutputs()) { + auto ret = graph_->AppendNodeInputTensor(compile_node, output, true); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Append input tensor for node failed, node: " << compile_node->GetName(); + return ret; + } + } + return kSuccess; + } + auto tensor = TensorAdapter::Convert2Tensor(param_node); + if (tensor == nullptr) { + MS_LOG(ERROR) << "Create tensor from Parameter failed."; + return kLiteError; + } + auto format_value = compile_node->GetBaseOperator()->GetAttr(mindspore::ops::kFormat); + if (format_value != nullptr) { + tensor->set_format(static_cast(GetValue(format_value))); + } else { + tensor->set_format(graph_format_); + } + auto ret = graph_->AppendNodeInputTensor(compile_node, tensor); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Append input tensor for node failed, node: " << compile_node->GetName(); + return ret; + } + return kSuccess; +} + +StatusCode CompileResultBuilder::AppendInputValueNodeToInputs(const ValueNodePtr &value_node, + const CompileNode *compile_node) { + if (value_node == nullptr) { + MS_LOG(ERROR) << "Input value_node is nullptr."; + return kLiteInputParamInvalid; + } + if (compile_node == nullptr) { + MS_LOG(ERROR) << "Input compile_node is nullptr."; + return kLiteInputParamInvalid; + } + auto tensor = TensorAdapter::Convert2Tensor(value_node); + if (tensor == nullptr) { + MS_LOG(ERROR) << "Create tensor from ValueNode failed."; + return kLiteError; + } + auto format_value = compile_node->GetBaseOperator()->GetAttr(mindspore::ops::kFormat); + if (format_value != nullptr) { + tensor->set_format(static_cast(GetValue(format_value))); + } else { + tensor->set_format(graph_format_); + } + auto ret = graph_->AppendNodeInputTensor(compile_node, tensor); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Append input tensor for node failed, node: " << compile_node->GetName(); + return ret; + } + return kSuccess; +} + +StatusCode CompileResultBuilder::CreateAndAppendNode(const CNodePtr &cnode) { + auto compile_node = CompileNode::Create(cnode); + if (compile_node == nullptr) { + MS_LOG(ERROR) << "Create compile node failed, cnode: " << cnode->fullname_with_scope(); + return kLiteError; + } + auto ret = graph_->AppendNode(compile_node); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Append compile_node to graph failed, node: " << compile_node->GetName(); + return ret; + } + // inputs + for (size_t i = 1; i < cnode->size(); i++) { + auto &input = cnode->input(i); + if (utils::isa(input)) { + ret = this->AppendInputCNodeToInputs(utils::cast(input), compile_node); + } else if (utils::isa(input)) { + ret = this->AppendInputParameterToInputs(utils::cast(input), compile_node); + } else if (utils::isa(input)) { + ret = this->AppendInputValueNodeToInputs(utils::cast(input), compile_node); + } else { + MS_LOG(ERROR) << "Unsupported input node of cnode: " << input + << ", current cnode: " << cnode->fullname_with_scope(); + ret = kLiteNotSupport; + } + if (ret != kSuccess) { + MS_LOG(ERROR) << "Create input tensor for cnode failed, cnode: " << cnode->fullname_with_scope(); + return ret; + } + } + // outputs + ret = BuildNodeOutputTensor(cnode, compile_node); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Create output tensors of cnode failed, cnode: " << cnode; + return ret; + } + return kSuccess; +} + +StatusCode CompileResultBuilder::CreateTensorsFromAbstract(const AbstractBasePtr &abstract, + std::vector *results) { + if (results == nullptr) { + MS_LOG(ERROR) << "Result is nullptr."; + return kLiteInputParamInvalid; + } + results->clear(); + // multi output abstract + if (utils::isa(abstract)) { + auto elements = utils::cast(abstract)->elements(); + for (auto &element : elements) { + auto tensor = TensorAdapter::Convert2Tensor(element); + if (tensor == nullptr) { + MS_LOG(ERROR) << "Create tensor from abstract failed, abstract : " << element; + return kLiteError; + } + results->emplace_back(tensor); + } + return kSuccess; + } + // single output abstract + if (utils::isa(abstract)) { + auto tensor = TensorAdapter::Convert2Tensor(abstract); + if (tensor == nullptr) { + MS_LOG(ERROR) << "Create tensor from abstract failed, abstract : " << abstract; + return kLiteError; + } + results->emplace_back(tensor); + return kSuccess; + } + MS_LOG(ERROR) << "Unsupported abstract: " << abstract; + return kLiteNotSupport; +} + +StatusCode CompileResultBuilder::BuildNodeOutputTensor(const CNodePtr &cnode, const CompileNode *compile_node) { + if (cnode == nullptr) { + MS_LOG(ERROR) << "Input cnode is nullptr."; + return kLiteInputParamInvalid; + } + if (compile_node == nullptr) { + MS_LOG(ERROR) << "Input compile_node is nullptr."; + return kLiteInputParamInvalid; + } + std::vector results; + auto ret = CreateTensorsFromAbstract(cnode->abstract(), &results); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Create tensors from output abstract of cnode failed, cnode : " << cnode->fullname_with_scope(); + return ret; + } + if (compile_node->OutputSize() > 0) { + MS_LOG(ERROR) << "Build node output twice, node : " << compile_node->GetName(); + return kLiteError; + } + for (auto &result : results) { + ret = graph_->AppendNodeOutputTensor(compile_node, result); + if (ret != kSuccess) { + MS_LOG(ERROR) << "Append output tensor to node failed, node: " << compile_node->GetName(); + return ret; + } + } + return kSuccess; +} +} // namespace infer +} // namespace mindspore diff --git a/mindspore/lite/src/extendrt/graph_compiler/compile_result_builder.h b/mindspore/lite/src/extendrt/graph_compiler/compile_result_builder.h new file mode 100644 index 00000000000..efcbcd359a1 --- /dev/null +++ b/mindspore/lite/src/extendrt/graph_compiler/compile_result_builder.h @@ -0,0 +1,69 @@ +/** + * Copyright 2023 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_COMPILE_RESULT_BUILDER_H_ +#define MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_COMPILE_RESULT_BUILDER_H_ +#include +#include +#include +#include +#include +#include "src/extendrt/graph_compiler/compile_result.h" +#include "src/extendrt/tensor.h" +#include "abstract/abstract_value.h" +#include "ir/anf.h" +#include "include/api/status.h" + +namespace mindspore { +namespace infer { +class CompileResultBuilder { + public: + explicit CompileResultBuilder(Format graph_format) : graph_format_(graph_format) {} + ~CompileResultBuilder() = default; + CompileResultPtr Build(const GraphSegmentPtr &graph_segment, const AnfNodePtrList &inputs, + const AnfNodePtrList &outputs); + + private: + // build + StatusCode BuildInputs(const AnfNodePtrList &inputs); + StatusCode BuildNodes(const GraphSegmentPtr &graph_segment); + StatusCode BuildOutputs(const AnfNodePtrList &outputs); + StatusCode OptimizeGraph(); + // methods about node + StatusCode CreateAndAppendNode(const CNodePtr &cnode); + StatusCode AppendInputCNodeToInputs(const CNodePtr &cnode, const CompileNode *compile_node); + StatusCode AppendInputParameterToInputs(const ParameterPtr ¶m_node, const CompileNode *compile_node); + StatusCode AppendInputValueNodeToInputs(const ValueNodePtr &value_node, const CompileNode *compile_node); + // methods about tensor + static StatusCode CreateTensorsFromAbstract(const mindspore::abstract::AbstractBasePtr &abstract, + std::vector *results); + StatusCode BuildNodeOutputTensor(const CNodePtr &cnode, const CompileNode *compile_node); + // methods about optimize + StatusCode RemoveSeqGetItemNode(); + StatusCode RemoveMakeSeqNode(); + StatusCode RemoveDependNode(); + // Replace `index`th output tensor of `node` by `dst_tensor` tensor. + void IsolateTensor(Tensor *dst_tensor, const CompileNode *node, size_t index); + + private: + CompileResultPtr graph_ = nullptr; + Format graph_format_{DEFAULT_FORMAT}; + std::set input_names_{}; +}; +} // namespace infer +} // namespace mindspore + +#endif diff --git a/mindspore/lite/src/extendrt/graph_compiler/default_graph_compiler.cc b/mindspore/lite/src/extendrt/graph_compiler/default_graph_compiler.cc index 36cd846be9e..8905c3ddadb 100644 --- a/mindspore/lite/src/extendrt/graph_compiler/default_graph_compiler.cc +++ b/mindspore/lite/src/extendrt/graph_compiler/default_graph_compiler.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019-2021 Huawei Technologies Co., Ltd + * Copyright 2023 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/mindspore/lite/src/extendrt/graph_compiler/default_graph_compiler.h b/mindspore/lite/src/extendrt/graph_compiler/default_graph_compiler.h index e5e2cb2fb3f..8a6454e5a95 100644 --- a/mindspore/lite/src/extendrt/graph_compiler/default_graph_compiler.h +++ b/mindspore/lite/src/extendrt/graph_compiler/default_graph_compiler.h @@ -1,5 +1,5 @@ /** - * Copyright 2019-2021 Huawei Technologies Co., Ltd + * Copyright 2023 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,15 +33,20 @@ class DefaultGraphCompiler : public mindspore::infer::abstract::GraphCompiler { std::shared_ptr Compile(FuncGraphPtr graph) override; + infer::abstract::ExecutionFlowPtr Compile(const GraphSegmentPtr &segment, const AnfNodePtrList &inputs, + const AnfNodePtrList &outputs) override { + return nullptr; + } + protected: virtual std::vector Partition(const FuncGraphPtr &graph); virtual std::shared_ptr Schedule(const std::vector &graph_segments, - FuncGraphPtr func_graph); + FuncGraphPtr func_graph); virtual std::shared_ptr Schedule(const GraphSegmentPtr &graph_segment, - const std::vector &inputs, - const std::vector &outputs); + const std::vector &inputs, + const std::vector &outputs); private: infer::abstract::Tensor *CreateTensor(AnfNodePtr node); diff --git a/mindspore/lite/src/extendrt/graph_compiler/main.cc b/mindspore/lite/src/extendrt/graph_compiler/main.cc new file mode 100644 index 00000000000..a977350cc6c --- /dev/null +++ b/mindspore/lite/src/extendrt/graph_compiler/main.cc @@ -0,0 +1,133 @@ +/** + * Copyright 2023 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "abstract/abstract_value.h" +#include "ops/core_ops.h" +#include "include/api/context.h" +#include "src/litert/inner_context.h" +#include "src/litert/cxx_api/converters.h" +#include "backend/graph_compiler/graph_partition.h" +#include "backend/graph_compiler/segment_runner.h" +#include "src/extendrt/graph_compiler/single_graph_compiler.h" + +namespace mindspore { +FuncGraphPtr CreateFuncGraph() { + std::vector shp{2, 2}; + abstract::AbstractTensorPtr abs; + + // Func graph. + auto func_graph = std::make_shared(); + + // Parameter. + auto abstract_x = std::make_shared(kFloat32, shp); + auto parameter_x = func_graph->add_parameter(); + parameter_x->set_name("input_x"); + parameter_x->set_abstract(abstract_x); + + auto abstract_y = std::make_shared(kFloat32, shp); + auto parameter_y = func_graph->add_parameter(); + parameter_y->set_name("input_y"); + parameter_y->set_abstract(abstract_y); + auto parameters = func_graph->parameters(); + + // Add. + std::vector add_inputs{NewValueNode(prim::kPrimAdd), parameters[0], parameters[1]}; + auto add_node = func_graph->NewCNode(add_inputs); + abs = std::make_shared(kFloat32, shp); + add_node->set_abstract(abs); + + // Reshape. + std::vector reshape_inputs{NewValueNode(prim::kPrimReshape), add_node}; + auto reshape_node = func_graph->NewCNode(reshape_inputs); + abs = std::make_shared(kFloat32, shp); + reshape_node->set_abstract(abs); + + // sub. + std::vector sub_inputs{NewValueNode(prim::kPrimSub), reshape_node, parameters[0]}; + auto sub_node = func_graph->NewCNode(sub_inputs); + abs = std::make_shared(kFloat32, shp); + sub_node->set_abstract(abs); + + // Return. + std::vector return_inputs{NewValueNode(prim::kPrimReturn), sub_node}; + auto return_node = func_graph->NewCNode(return_inputs); + func_graph->set_return(return_node); + +#ifdef USE_REAL_MODEL + const std::string model_path = ""; // /path/to/mobilenet.mindir + MindIRLoader mindir_loader; + func_graph = mindir_loader.LoadMindIR(model_path); +#endif + return func_graph; +} + +std::tuple CreateSegment() { + auto func_graph = CreateFuncGraph(); + std::cout << "============================================= func_graph inputs:" << std::endl; + for (auto &input : func_graph->get_inputs()) { + std::cout << input << ":" << input->fullname_with_scope() << std::endl; + } + auto new_manager = MakeManager({func_graph}); + MS_EXCEPTION_IF_NULL(new_manager); + new_manager->AddFuncGraph(func_graph); + func_graph->set_manager(new_manager); + + static const std::vector ms_nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, + prim::kPrimSwitch, prim::kPrimMakeTuple, + prim::kPrimBpropCut, prim::kPrimSwitchLayer}; + auto graph_partition = std::make_shared(ms_nonlinear_ops, kMsConvert); + bool multi_target = false; + auto segments = graph_partition->Partition(func_graph, &multi_target); + auto segment = segments[0]; + FuncGraphPtr fg; + AnfNodePtrList inputs; + AnfNodePtrList outputs; + std::tie(fg, inputs, outputs) = compile::TransformSegmentToAnfGraph(segment->nodes_); + std::cout << "============================================= segment:" << std::endl; + std::cout << "--------------------------------------------- nodes:" << std::endl; + for (auto &node : segment->nodes_) { + std::cout << node << ":" << node->fullname_with_scope() << std::endl; + } + std::cout << "--------------------------------------------- inputs:" << std::endl; + for (auto &input : inputs) { + std::cout << input << ":" << input->fullname_with_scope() << std::endl; + } + std::cout << "--------------------------------------------- outputs:" << std::endl; + for (auto &output : outputs) { + std::cout << output << ":" << output->fullname_with_scope() << std::endl; + } + return std::make_tuple(segment, inputs, outputs); +} +} // namespace mindspore + +int main() { + mindspore::GraphSegmentPtr segment; + mindspore::AnfNodePtrList inputs; + mindspore::AnfNodePtrList outputs; + std::tie(segment, inputs, outputs) = mindspore::CreateSegment(); + auto context = std::make_shared(); + auto device_info = std::make_shared(); + context->MutableDeviceInfo().emplace_back(device_info); + auto inner_context = std::shared_ptr(mindspore::ContextUtils::Convert(context.get())); + auto compiler = std::make_shared(inner_context); + compiler->Compile(segment, inputs, outputs); + return 0; +} diff --git a/mindspore/lite/src/extendrt/graph_compiler/single_graph_compiler.cc b/mindspore/lite/src/extendrt/graph_compiler/single_graph_compiler.cc new file mode 100644 index 00000000000..438a66aa8e9 --- /dev/null +++ b/mindspore/lite/src/extendrt/graph_compiler/single_graph_compiler.cc @@ -0,0 +1,50 @@ +/** + * Copyright 2023 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/extendrt/graph_compiler/single_graph_compiler.h" +#include +#include +#include "src/extendrt/execution_flow.h" +#include "src/extendrt/graph_compiler/compile_result_builder.h" + +namespace mindspore { +namespace infer { +CompileResultPtr SingleGraphCompiler::Build(const GraphSegmentPtr &segment, const AnfNodePtrList &inputs, + const AnfNodePtrList &outputs) { + // assume graph_format is NHWC + auto graph_format = NHWC; + auto builder = std::make_shared(graph_format); + return builder->Build(segment, inputs, outputs); +} + +abstract::ExecutionFlowPtr SingleGraphCompiler::Schedule(const CompileResultPtr &node_list) { + // infershape + // choose lite-kernel else choose kernel-mod + // tensor data type/format transform + if (MS_UNLIKELY(scheduler_ == nullptr)) { + scheduler_ = std::make_shared(this->context_); + } + return scheduler_->Schedule(node_list); +} + +abstract::ExecutionFlowPtr SingleGraphCompiler::Compile(const GraphSegmentPtr &segment, const AnfNodePtrList &inputs, + const AnfNodePtrList &outputs) { + auto node_list = this->Build(segment, inputs, outputs); + std::cout << node_list->Dump() << std::endl; + return this->Schedule(node_list); +} +} // namespace infer +} // namespace mindspore diff --git a/mindspore/lite/src/extendrt/graph_compiler/single_graph_compiler.h b/mindspore/lite/src/extendrt/graph_compiler/single_graph_compiler.h new file mode 100644 index 00000000000..87c4715d3b0 --- /dev/null +++ b/mindspore/lite/src/extendrt/graph_compiler/single_graph_compiler.h @@ -0,0 +1,53 @@ +/** + * Copyright 2023 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_SINGLE_GRAPH_COMPILER_H_ +#define MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_SINGLE_GRAPH_COMPILER_H_ +#include +#include +#include +#include +#include +#include "src/litert/inner_context.h" +#include "mindapi/ir/abstract.h" +#include "ir/anf.h" +#include "src/extendrt/graph_compiler/compile_result.h" +#include "src/extendrt/tensor.h" +#include "src/infer/execution_flow.h" +#include "src/infer/graph_compiler.h" +#include "src/extendrt/graph_compiler/single_graph_scheduler.h" + +namespace mindspore { +namespace infer { +class SingleGraphCompiler : public abstract::GraphCompiler { + public: + explicit SingleGraphCompiler(std::shared_ptr context) : context_(std::move(context)) {} + ~SingleGraphCompiler() override = default; + abstract::ExecutionPlanPtr Compile(FuncGraphPtr graph) override { return nullptr; } + abstract::ExecutionFlowPtr Compile(const GraphSegmentPtr &segment, const AnfNodePtrList &inputs, + const AnfNodePtrList &outputs) override; + + private: + CompileResultPtr Build(const GraphSegmentPtr &segment, const AnfNodePtrList &inputs, const AnfNodePtrList &outputs); + abstract::ExecutionFlowPtr Schedule(const CompileResultPtr &node_list); + + SingleGraphSchedulerPtr scheduler_{nullptr}; + std::shared_ptr context_{nullptr}; +}; +} // namespace infer +} // namespace mindspore + +#endif diff --git a/mindspore/lite/src/extendrt/graph_compiler/single_graph_scheduler.cc b/mindspore/lite/src/extendrt/graph_compiler/single_graph_scheduler.cc new file mode 100644 index 00000000000..e912c1fcf4a --- /dev/null +++ b/mindspore/lite/src/extendrt/graph_compiler/single_graph_scheduler.cc @@ -0,0 +1,43 @@ +/** + * Copyright 2023 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/extendrt/graph_compiler/single_graph_scheduler.h" + +namespace mindspore { +namespace infer { +ExecutionFlowPtr SingleGraphScheduler::Schedule(const CompileResultPtr node_list) { + // no need to infer, infer is invoked in converter and delived to here + // select kernel + // fp16/fp32 weight, transpose weight + // append kernel with transpose + // optimize transpose + // infershape + return nullptr; +} + +KernelType SingleGraphScheduler::SelectKernel(const CompileNode *compile_node) { return kKernelTypeUnknown; } + +bool SingleGraphScheduler::HandleWeightForKernel(const CompileNode *compile_node, const KernelType kernel_type) { + return false; +} + +bool SingleGraphScheduler::AppendKernelToPlan(kernel::KernelExec *kernel) { return false; } + +bool SingleGraphScheduler::OptimizeTranspose(const std::vector &kernels) { return false; } + +bool InferShape(const std::vector &kernels) { return false; } +} // namespace infer +} // namespace mindspore diff --git a/mindspore/lite/src/extendrt/graph_compiler/single_graph_scheduler.h b/mindspore/lite/src/extendrt/graph_compiler/single_graph_scheduler.h new file mode 100644 index 00000000000..3cf7be5b186 --- /dev/null +++ b/mindspore/lite/src/extendrt/graph_compiler/single_graph_scheduler.h @@ -0,0 +1,53 @@ +/** + * Copyright 2023 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_SINGLE_GRAPH_SCHEDULER_H_ +#define MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_SINGLE_GRAPH_SCHEDULER_H_ +#include +#include +#include +#include "src/extendrt/graph_compiler/compile_result.h" +#include "src/extendrt/execution_flow.h" +#include "src/litert/inner_context.h" + +namespace mindspore { +namespace infer { +enum KernelType { + kKernelTypeUnknown = 0, + kKernelTypeLite = 1, + kKernelTypeCloud = 2, +}; + +class SingleGraphScheduler { + public: + explicit SingleGraphScheduler(std::shared_ptr context) : context_(std::move(context)) {} + virtual ~SingleGraphScheduler() = default; + ExecutionFlowPtr Schedule(const CompileResultPtr node_list); + + private: + KernelType SelectKernel(const CompileNode *compile_node); + bool HandleWeightForKernel(const CompileNode *compile_node, const KernelType kernel_type); + bool AppendKernelToPlan(kernel::KernelExec *kernel); + bool OptimizeTranspose(const std::vector &kernels); + bool InferShape(const std::vector &kernels); + + private: + std::shared_ptr context_; + ExecutionFlowPtr execution_plan_{nullptr}; +}; +using SingleGraphSchedulerPtr = std::shared_ptr; +} // namespace infer +} // namespace mindspore +#endif diff --git a/mindspore/lite/src/extendrt/graph_scheduler.cc b/mindspore/lite/src/extendrt/graph_scheduler.cc index a4604b82838..e9e89bd377c 100644 --- a/mindspore/lite/src/extendrt/graph_scheduler.cc +++ b/mindspore/lite/src/extendrt/graph_scheduler.cc @@ -15,27 +15,9 @@ */ #include "extendrt/graph_scheduler.h" #include "extendrt/graph_compiler.h" -ExcutionPlan GraphCompiler::Schedule(const CompileResult &compile_result) { - ExcutionPlan execplan; - for (auto subgraph_ : compile_result.graphs_) { - if (KernelGraphUtils::Instance()->IsControlNode(subgraph_)) { - // generate control kernel - CNode &node = subgraph_->nodes[0]; - KernelInfo kernelInfo; - kernelInfo.kernel_ = CreateControlKernel(node); - execplan.emplace_back(kernelInfo); - } else if (KernelGraphUtils::Instance()->IsSingleNode(subgraph_)) { - // generate kernel - KernelInfo kernelInfo; - CNode &node = subgraph_->nodes[0]; - kernelInfo.kernel_ = CreateKernel(node); - execplan.emplace_back(kernelInfo); - } else if (KernelGraphUtils::Instance()->IsDAG(subgraph_)) { - // generate subgraph kernel - KernelInfo kernelInfo; - kernelInfo.kernel_ = CreateSubgraphKernel(subgraph_); - execplan.emplace_back(kernelInfo); - } - } - return execplan; -} + +namespace mindspore { +namespace infer { +ExcutionPlan GraphScheduler::Schedule(const CompileResult &compile_result) { return {}; } +} // namespace infer +} // namespace mindspore diff --git a/mindspore/lite/src/extendrt/graph_scheduler.h b/mindspore/lite/src/extendrt/graph_scheduler.h index e89fa55352f..948dd68d2d1 100644 --- a/mindspore/lite/src/extendrt/graph_scheduler.h +++ b/mindspore/lite/src/extendrt/graph_scheduler.h @@ -25,10 +25,21 @@ #include "include/api/status.h" #include "include/common/utils/utils.h" #include "ir/func_graph.h" +#include "src/litert/kernel_exec.h" + namespace mindspore { namespace infer { struct ScheduleStrategy {}; struct GraphCompilerInfo; + +struct Tensor {}; +struct ExcutionPlan { + std::vector kernels_; + std::vector tensors_; + std::vector inputs_; + std::vector outputs_; +}; + class GraphScheduler : public std::enable_shared_from_this { public: explicit GraphScheduler(const ScheduleStrategy &strategy); diff --git a/mindspore/lite/src/extendrt/tensor.h b/mindspore/lite/src/extendrt/tensor.h new file mode 100644 index 00000000000..1519398b1ae --- /dev/null +++ b/mindspore/lite/src/extendrt/tensor.h @@ -0,0 +1,29 @@ +/** + * Copyright 2023 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_TENSOR_H_ +#define MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_TENSOR_H_ + +#include "src/infer/tensor.h" + +namespace mindspore { +namespace infer { +// implemented tensor for extendrt +using Tensor = abstract::Tensor; +} // namespace infer +} // namespace mindspore + +#endif // MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_TENSOR_H_ diff --git a/mindspore/lite/src/infer/execution_flow.h b/mindspore/lite/src/infer/execution_flow.h index ec03b2373f8..ffb2dd8cf5f 100644 --- a/mindspore/lite/src/infer/execution_flow.h +++ b/mindspore/lite/src/infer/execution_flow.h @@ -105,6 +105,7 @@ class ExecutionFlow : public std::enable_shared_from_this { /// \return Kernel pointer. virtual Kernel *ConstructFusionKernel() = 0; }; +using ExecutionFlowPtr = std::shared_ptr; } // namespace mindspore::infer::abstract #endif // MINDSPORE_LITE_INFER_EXECUTION_FLOW_H_ diff --git a/mindspore/lite/src/infer/execution_plan.h b/mindspore/lite/src/infer/execution_plan.h index cc05e570098..6d62c76992f 100644 --- a/mindspore/lite/src/infer/execution_plan.h +++ b/mindspore/lite/src/infer/execution_plan.h @@ -124,6 +124,7 @@ class ExecutionPlan : public std::enable_shared_from_this { /// \return Kernel List virtual std::vector ToKernelList() = 0; }; +using ExecutionPlanPtr = std::shared_ptr; } // namespace mindspore::infer::abstract #endif // MINDSPORE_LITE_INFER_EXECUTION_PLAN_H_ diff --git a/mindspore/lite/src/infer/graph_compiler.h b/mindspore/lite/src/infer/graph_compiler.h index 3cb4db892da..5e3479ba44f 100644 --- a/mindspore/lite/src/infer/graph_compiler.h +++ b/mindspore/lite/src/infer/graph_compiler.h @@ -19,6 +19,7 @@ #include #include "infer/execution_plan.h" +#include "infer/execution_flow.h" namespace mindspore::infer::abstract { class GraphCompiler : public std::enable_shared_from_this { @@ -30,7 +31,10 @@ class GraphCompiler : public std::enable_shared_from_this { /// \param[in] graph FuncGraph need to compile. /// /// \return ExecutionPlan pointer. - virtual std::shared_ptr Compile(FuncGraphPtr graph) = 0; + virtual ExecutionPlanPtr Compile(FuncGraphPtr graph) = 0; + + virtual ExecutionFlowPtr Compile(const GraphSegmentPtr &segment, const AnfNodePtrList &inputs, + const AnfNodePtrList &outputs) = 0; }; } // namespace mindspore::infer::abstract diff --git a/mindspore/lite/src/litert/kernel_exec.h b/mindspore/lite/src/litert/kernel_exec.h index 8bea496064e..1c39bba9a1f 100644 --- a/mindspore/lite/src/litert/kernel_exec.h +++ b/mindspore/lite/src/litert/kernel_exec.h @@ -375,6 +375,7 @@ class KernelExec { protected: std::shared_ptr kernel_ = nullptr; KernelKey desc_; + std::string op_type_{}; // merge op_type_ and desc_.type into type-wrapper while redesign infer-kernel // tensor will free in ~lite_session() std::vector in_kernels_; std::vector out_kernels_;