Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

mindinfer: add graph compiler

This commit is contained in:
hangangqiang 2023-02-27 15:46:41 +08:00
parent 232aede207
commit a25d2007db
25 changed files with 1897 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
mindspore/lite/src/extendrt/CMakeLists.txt
View File

@ -60,6 +60,11 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE OR MSLITE_ENABLE_CLOUD_INFERENCE)
${CMAKE_CURRENT_SOURCE_DIR}/graph_executor/mindrt_graph_executor.cc
${CMAKE_CURRENT_SOURCE_DIR}/execution_flow.cc
${CMAKE_CURRENT_SOURCE_DIR}/execution_plan.cc
${CMAKE_CURRENT_SOURCE_DIR}/graph_compiler/compile_result.cc
${CMAKE_CURRENT_SOURCE_DIR}/graph_compiler/single_graph_compiler.cc
${CMAKE_CURRENT_SOURCE_DIR}/graph_compiler/single_graph_scheduler.cc
${CMAKE_CURRENT_SOURCE_DIR}/graph_compiler/compile_result_builder.cc
${CMAKE_CURRENT_SOURCE_DIR}/graph_compiler/anfnode_tensor_adapter.cc
)
if(MSLITE_ENABLE_BFC_MEMORY)
set(MSLITE_EXTEND_RUNTIME_SRC ${MSLITE_EXTEND_RUNTIME_SRC}
@ -140,9 +145,10 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE OR MSLITE_ENABLE_CLOUD_INFERENCE)
${CCSRC_DIR}/utils/signal_util.cc
${CORE_DIR}/utils/status.cc
)
add_library(mindspore-kernel-graph OBJECT ${KERNEL_GRAPH_SRC})
add_library(mindspore-extendrt SHARED ${MSLITE_EXTEND_RUNTIME_SRC})
add_library(mindspore-extendrt SHARED ${MSLITE_EXTEND_RUNTIME_SRC} ${MSLITE_EXTEND_CPU_RUNTIME_SRC})
add_dependencies(mindspore-extendrt fbs_inner_src)
add_dependencies(mindspore-extendrt mindspore-infer-anfalgo)
add_dependencies(mindspore-extendrt mindspore-kernel-graph)
@ -260,6 +266,9 @@ if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE OR MSLITE_ENABLE_CLOUD_INFERENCE)
add_dependencies(cloud_infer_test mindspore-extendrt)
target_link_libraries(cloud_infer_test mindspore-extendrt)
endif()
add_executable(unify-cpu-infer graph_compiler/main.cc)
add_dependencies(unify-cpu-infer mindspore-extendrt)
target_link_libraries(unify-cpu-infer mindspore-extendrt)
else()
set(MSLITE_EXTEND_RUNTIME_SRC ${MODEL_LOADER_FRAMEWORK_SRC})
add_library(mindspore-extendrt OBJECT ${MSLITE_EXTEND_RUNTIME_SRC})
@ -269,4 +278,3 @@ else()
endif()
set_target_properties(mindspore-extendrt PROPERTIES OUTPUT_NAME "mindspore-lite")

23
mindspore/lite/src/extendrt/execution_flow.cc
View File

@ -16,28 +16,19 @@
#include <vector>
#include "extendrt/execution_flow.h"
#include "src/extendrt/execution_flow.h"
#include "litert/lite_kernel.h"
#include "litert/kernel_exec_util.h"
#include "litert/sub_graph_kernel.h"
#include "src/extendrt/tensor.h"
#include "src/litert/kernel_exec_util.h"
#include "src/litert/sub_graph_kernel.h"
namespace mindspore::infer {
ExecutionFlow::~ExecutionFlow() {
for (auto tensor : inputs_) {
if (tensor != nullptr) {
delete tensor;
}
}
for (auto tensor : outputs_) {
if (tensor != nullptr) {
delete tensor;
}
for (auto tensor : tensors_) {
delete tensor;
}
for (auto kernel : kernels_) {
if (kernel != nullptr) {
delete kernel;
}
delete kernel;
}
}

6
mindspore/lite/src/extendrt/execution_flow.h
View File

@ -26,11 +26,11 @@ namespace mindspore::infer {
class ExecutionFlow : public abstract::ExecutionFlow {
public:
ExecutionFlow() = default;
virtual ~ExecutionFlow();
~ExecutionFlow() override;
std::vector<abstract::Kernel *> GetKernels() override { return kernels_; }
void SetKernels(std::vector<abstract::Kernel *> kernels) { kernels_ = kernels; }
void SetKernels(const std::vector<abstract::Kernel *> &kernels) override { kernels_ = kernels; }
std::vector<abstract::Tensor *> GetInputs() override { return inputs_; }
@ -58,10 +58,12 @@ class ExecutionFlow : public abstract::ExecutionFlow {
std::vector<abstract::Kernel *> kernels_;
std::vector<abstract::Tensor *> inputs_;
std::vector<abstract::Tensor *> outputs_;
std::vector<abstract::Tensor *> tensors_;
std::shared_ptr<abstract::Context> context_;
abstract::KernelCallBack before_;
abstract::KernelCallBack after_;
};
using ExecutionFlowPtr = std::shared_ptr<ExecutionFlow>;
} // namespace mindspore::infer
#endif // MINDSPORE_LITE_SRC_EXTENDRT_EXECUTION_FLOW_H_

24
mindspore/lite/src/extendrt/graph_compiler.cc
View File

@ -17,28 +17,8 @@
namespace mindspore {
namespace infer {
ExcutionPlan GraphCompiler::Compile(FuncGraphPtr func_graph) {
// Split graph to segments.
const auto &segments = graph_partition_->Partition(func_graph, &contain_multi_target);
MS_LOG(INFO) << "Compile graph: " << func_graph->ToString() << ", Split segments size:" << segments.size();
// Foreach the segments to compile graph.
for (const auto &segment : segments) {
CompileSegment(segment);
}
CompileResult result = LinkSegments();
return Schedule(result);
}
GraphId GraphCompiler::CompileSegment(const GraphSegmentPtr &segment) {
// Generate kernel graph.
auto nodes = segment->nodes_;
KernelGraphPtr graph = KernelGraphUtils::Instance()->ConstructKernelGraph(nodes, outputs, device_terget);
MS_EXCEPTION_IF_NULL(graph);
opt::EliminateIllegalDataTypePass(graph);
SetGraphDependency(graph, segment);
return graph->GraphId();
}
ExcutionPlan GraphCompiler::Compile(FuncGraphPtr func_graph) { return {}; }
GraphId GraphCompiler::CompileSegment(const GraphSegmentPtr segment) { return -1; }
CompileResult GraphCompiler::LinkSegment() { return CompileResult(); }
ExcutionPlan GraphCompiler::Schedule(const CompileResult &compile_result) {
return scheduler_.Schedule(compile_result);

27
mindspore/lite/src/extendrt/graph_compiler.h
View File

@ -23,45 +23,34 @@
#include "include/api/model.h"
#include "include/api/graph.h"
#include "include/api/status.h"
#include "include/api/kernel.h"
#include "include/common/utils/utils.h"
#include "ir/func_graph.h"
#include "src/extendrt/graph_scheduler.h"
namespace mindspore {
namespace infer {
using GraphId = uint32_t;
struct CompileOptions {
int optimize_level_;
};
struct CompileResult {
GraphId root_;
std::vector<GraphId> control_nodes_;
std::vector<KernelGraphPtr> graphs_;
};
struct KernelInfo {
KernelPtr kernel_;
bool isSubgraphKernel;
std::vector<int64_t> inputs_;
std::vector<int64_t> outputs_;
};
struct ExcutionPlan {
std::vector<KernelInfo> kernels_;
std::vector<int64_t> inputs_;
std::vector<int64_t> outputs_;
};
struct CompileResult {};
class GraphCompiler : public std::enable_shared_from_this<GraphCompiler> {
public:
explicit GraphCompiler(const CompileOptions &opts);
virtual ~GraphCompiler();
ExcutionPlan Compile(FuncGraphPtr graph);
ExcutionPlan Compile(GraphSegmentPtr segment);
protected:
ExcutionPlan Schedule(const CompileResult &);
GraphId CompileSegment(const GraphSegmentPtr &segment);
GraphId CompileSegment(const GraphSegmentPtr segment);
CompileResult LinkSegment();
protected:
GraphPartition graph_partition_;
GraphScheduler scheduler_;
CompileOptions options_;
mindspore::HashMap<GraphId, std::shared_ptr<KernelGraph>> graphs_;
};
} // namespace infer
} // namespace mindspore

349
mindspore/lite/src/extendrt/graph_compiler/anfnode_tensor_adapter.cc Normal file
View File

@ -0,0 +1,349 @@
/**
* Copyright 2023 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "src/extendrt/graph_compiler/anfnode_tensor_adapter.h"
#include "src/extendrt/graph_compiler/compile_result_builder.h"
#include "ir/anf.h"
#include "ir/func_graph.h"
#include "ir/primitive.h"
#include "utils/ms_utils_secure.h"
using ShapePtr = mindspore::abstract::ShapePtr;
using AbstractBasePtr = mindspore::abstract::AbstractBasePtr;
using AbstractTensorPtr = mindspore::abstract::AbstractTensorPtr;
namespace mindspore {
namespace infer {
Tensor *TensorAdapter::Convert2Tensor(const ParameterPtr &param_node, const std::string &tensor_name) {
auto adapter = TensorAdapter::Create(param_node);
if (adapter == nullptr) {
MS_LOG(ERROR) << "Create tensor-adapter from parameter failed, parameter : " << param_node;
return nullptr;
}
return adapter->ToTensor(tensor_name);
}
Tensor *TensorAdapter::Convert2Tensor(const ValueNodePtr &value_node, const std::string &tensor_name) {
auto adapter = TensorAdapter::Create(value_node);
if (adapter == nullptr) {
MS_LOG(ERROR) << "Create tensor-adapter from value-node failed, value-node : " << value_node;
return nullptr;
}
return adapter->ToTensor(tensor_name);
}
Tensor *TensorAdapter::Convert2Tensor(const AbstractTensorPtr &abstract, const std::string &tensor_name) {
auto adapter = TensorAdapter::Create(abstract);
if (adapter == nullptr) {
MS_LOG(ERROR) << "Create tensor-adapter from abstracttensor failed, abstract : " << abstract;
return nullptr;
}
return adapter->ToTensor(tensor_name);
}
Tensor *TensorAdapter::Convert2Tensor(const AbstractBasePtr &abstract, const std::string &tensor_name) {
auto adapter = TensorAdapter::Create(abstract);
if (adapter == nullptr) {
MS_LOG(ERROR) << "Create tensor-adapter from abstractbase failed, abstract : " << abstract;
return nullptr;
}
return adapter->ToTensor(tensor_name);
}
Tensor *TensorAdapter::ToTensor(const std::string &tensor_name) {
std::vector<int32_t> int32_shape(shape_.size());
for (size_t i = 0; i < shape_.size(); i++) {
int32_shape[i] = static_cast<int32_t>(shape_[i]);
}
auto tensor = Tensor::CreateTensor(tensor_name, data_type_, int32_shape, data_, data_len_);
if (tensor == nullptr) {
return nullptr;
}
// move data to tensor
tensor->set_own_data(own_data_);
own_data_ = false;
return tensor;
}
TensorAdapterPtr TensorAdapter::Create(const ParameterPtr &param_node) {
if (param_node == nullptr) {
MS_LOG(ERROR) << "Input parameter is nullptr.";
return nullptr;
}
ShapeVector shape_vector;
TypeId data_type = kTypeUnknown;
auto status = GetDTAndShapeFromParameter(param_node, &data_type, &shape_vector);
if (status != kSuccess) {
MS_LOG(ERROR) << "Get data type and shape from param node failed.";
return nullptr;
}
if (data_type == kObjectTypeString) {
MS_LOG(ERROR) << "Not support kObjectTypeString type DefaultParam.";
return nullptr;
}
auto adapter = std::make_shared<TensorAdapter>();
adapter->data_type_ = data_type;
adapter->shape_ = shape_vector;
adapter->format_ = NHWC;
adapter->is_const_ = param_node->has_default();
if (!adapter->is_const_) {
return adapter;
}
auto tensor_info = std::dynamic_pointer_cast<tensor::Tensor>(param_node->default_param());
if (tensor_info == nullptr) {
MS_LOG(ERROR) << "Cast default-param to tensor failed.";
return nullptr;
}
adapter->compress_type_ = tensor_info->compression_type();
adapter->data_ = tensor_info->data_c();
adapter->data_len_ = tensor_info->Size();
adapter->own_data_ = false;
return adapter;
}
TensorAdapterPtr TensorAdapter::CreateFromTensorValueNode(const ValueNodePtr &value_node) {
auto value_abstract = value_node->abstract();
if (value_abstract == nullptr) {
MS_LOG(ERROR) << "Abstract of value is nullptr";
return nullptr;
}
auto adapter = TensorAdapter::Create(value_abstract);
if (adapter == nullptr) {
MS_LOG(ERROR) << "Create tensor adapter from abstract of valuenode failed, valuenode: "
<< value_node->fullname_with_scope();
return nullptr;
}
adapter->is_const_ = true;
auto value = value_node->value();
if (value == nullptr) {
MS_LOG(ERROR) << "Value of value-node is nullptr, " << value_node->fullname_with_scope();
return nullptr;
}
auto data = value->cast<tensor::TensorPtr>();
if (data == nullptr) {
MS_LOG(ERROR) << "Value of tensor-type value-node is not a Tensor, " << value_node->fullname_with_scope();
return nullptr;
}
adapter->data_ = data->data_c();
adapter->data_len_ = data->Size();
adapter->own_data_ = false;
return adapter;
}
TensorAdapterPtr TensorAdapter::CreateFromInt32ImmValue(const ValueNodePtr &value_node) {
MS_ASSERT(value_node != nullptr);
auto adapter = std::make_shared<TensorAdapter>();
adapter->is_const_ = true;
adapter->data_type_ = kNumberTypeInt32;
adapter->shape_ = {1};
auto value = value_node->value();
if (value == nullptr) {
MS_LOG(ERROR) << "Value of value-node is nullptr, " << value_node->fullname_with_scope();
return nullptr;
}
auto data = GetValue<int32_t>(value);
adapter->data_ = &data;
adapter->data_len_ = sizeof(int32_t);
adapter->own_data_ = false;
return adapter;
}
TensorAdapterPtr TensorAdapter::CreateFromInt64ImmValue(const ValueNodePtr &value_node) {
MS_ASSERT(value_node != nullptr);
auto adapter = std::make_shared<TensorAdapter>();
adapter->is_const_ = true;
adapter->data_type_ = kNumberTypeInt64;
adapter->shape_ = {1};
auto value = value_node->value();
if (value == nullptr) {
MS_LOG(ERROR) << "Value of value-node is nullptr, " << value_node->fullname_with_scope();
return nullptr;
}
auto data = GetValue<int64_t>(value);
adapter->data_ = &data;
adapter->data_len_ = sizeof(int64_t);
adapter->own_data_ = false;
return adapter;
}
TensorAdapterPtr TensorAdapter::CreateFromBoolImmValue(const ValueNodePtr &value_node) {
MS_ASSERT(value_node != nullptr);
auto adapter = std::make_shared<TensorAdapter>();
adapter->is_const_ = true;
adapter->data_type_ = kNumberTypeBool;
adapter->shape_ = {1};
auto value = value_node->value();
if (value == nullptr) {
MS_LOG(ERROR) << "Value of value-node is nullptr, " << value_node->fullname_with_scope();
return nullptr;
}
auto data = value->cast<mindspore::BoolImmPtr>();
if (data == nullptr) {
MS_LOG(ERROR) << "BoolImm Value of cast to BoolImmPtr failed, " << value_node->fullname_with_scope();
return nullptr;
}
auto data_value = data->value();
adapter->data_ = &data_value;
adapter->data_len_ = sizeof(bool);
adapter->own_data_ = false;
return adapter;
}
TensorAdapterPtr TensorAdapter::CreateFromNumberTypeValue(const ValueNodePtr &value_node) {
MS_ASSERT(value_node != nullptr);
auto adapter = std::make_shared<TensorAdapter>();
adapter->is_const_ = true;
adapter->data_type_ = kNumberTypeInt32;
adapter->shape_ = {1};
auto data = utils::cast<NumberPtr>(value_node->value());
if (data == nullptr) {
MS_LOG(ERROR) << "Value of Number type value-node is not a NumberPtr, " << value_node->fullname_with_scope();
return nullptr;
}
TypeId number_type = data->number_type();
static const std::unordered_map<TypeId, TypeId> TypeToTypeMap = {
{kNumberTypeInt, kNumberTypeInt32}, {kNumberTypeUInt, kNumberTypeUInt32}, {kNumberTypeFloat, kNumberTypeFloat32}};
if (TypeToTypeMap.find(number_type) != TypeToTypeMap.end()) {
number_type = TypeToTypeMap.at(number_type);
}
auto number_data = static_cast<int32_t>(number_type);
adapter->data_ = &number_data;
adapter->data_len_ = sizeof(int32_t);
adapter->own_data_ = false;
return adapter;
}
TensorAdapterPtr TensorAdapter::CreateFromIntSequenceValue(const ValueNodePtr &value_node) {
MS_ASSERT(value_node != nullptr);
auto value_seq = utils::cast<ValueSequencePtr>(value_node->value());
if (value_seq == nullptr) {
MS_LOG(ERROR) << "Value of Sequence type value-node is not a ValueSequencePtr, "
<< value_node->fullname_with_scope();
return nullptr;
}
auto adapter = std::make_shared<TensorAdapter>();
adapter->is_const_ = true;
if (!value_seq->value().empty()) {
if (value_seq->value().front()->type()->number_type() == kNumberTypeInt32 ||
value_seq->value().front()->type()->number_type() == kNumberTypeInt) {
adapter->data_type_ = kNumberTypeInt32;
auto data = GetValue<std::vector<int32_t>>(value_seq);
adapter->shape_ = {static_cast<int64_t>(data.size())};
adapter->data_ = data.data();
adapter->data_len_ = data.size() * sizeof(int32_t);
adapter->own_data_ = false;
} else if (value_seq->value().front()->type()->number_type() == kNumberTypeInt64) {
adapter->data_type_ = kNumberTypeInt64;
auto data = GetValue<std::vector<int64_t>>(value_seq);
adapter->shape_ = {static_cast<int64_t>(data.size())};
adapter->data_ = data.data();
adapter->data_len_ = data.size() * sizeof(int64_t);
adapter->own_data_ = false;
} else {
MS_LOG(ERROR) << "only support integer value ValueSequence.";
return nullptr;
}
}
return adapter;
}
TensorAdapterPtr TensorAdapter::Create(const ValueNodePtr &value_node) {
MS_ASSERT(value_node != nullptr);
auto value = value_node->value();
if (value->isa<tensor::Tensor>()) {
return CreateFromTensorValueNode(value_node);
} else if (value->isa<mindspore::Int32Imm>()) {
return CreateFromInt32ImmValue(value_node);
} else if (value->isa<mindspore::Int64Imm>()) {
return CreateFromInt64ImmValue(value_node);
} else if (value->isa<mindspore::BoolImm>()) {
return CreateFromBoolImmValue(value_node);
} else if (value->isa<mindspore::ValueSequence>()) {
return CreateFromIntSequenceValue(value_node);
} else if (value->isa<Number>()) {
return CreateFromNumberTypeValue(value_node);
} else {
MS_LOG(ERROR) << "Not support value type: " << value->type();
return nullptr;
}
}
TensorAdapterPtr TensorAdapter::Create(const AbstractBasePtr &abs) {
auto abs_tensor = utils::cast<AbstractTensorPtr>(abs);
if (abs_tensor == nullptr) {
MS_LOG(ERROR) << "Input abstract is not a AbstractTensor.";
return nullptr;
}
return TensorAdapter::Create(abs_tensor);
}
TensorAdapterPtr TensorAdapter::Create(const AbstractTensorPtr &abs_tensor) {
if (abs_tensor == nullptr) {
MS_LOG(ERROR) << "Input abstract is not a AbstractTensor.";
return nullptr;
}
ShapeVector shape_vector;
TypeId data_type = kTypeUnknown;
auto ret = GetDTAndShapeFromAbTensor(abs_tensor, &data_type, &shape_vector);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Get data type and shape from value node failed.";
return nullptr;
}
auto adapter = std::make_shared<TensorAdapter>();
adapter->data_type_ = data_type;
adapter->shape_ = shape_vector;
return adapter;
}
StatusCode TensorAdapter::GetDTAndShapeFromAbTensor(const AbstractTensorPtr &abstract, TypeId *data_type,
ShapeVector *shape_vector) {
MS_ASSERT(abstract != nullptr && data_type != nullptr && shape_vector != nullptr);
if (abstract->element() == nullptr) {
MS_LOG(ERROR) << "`element` of abstract is nullptr";
return kLiteError;
}
auto type_ptr = abstract->element()->GetTypeTrack();
if (type_ptr == nullptr) {
MS_LOG(ERROR) << "Type of abstract is nullptr";
return kLiteError;
}
*data_type = type_ptr->type_id();
if (!utils::isa<ShapePtr>(abstract->BuildShape())) {
MS_LOG(ERROR) << "Shape of Abstract of parameter should be ShapePtr";
return kLiteError;
}
*shape_vector = utils::cast<ShapePtr>(abstract->BuildShape())->shape();
return kSuccess;
}
StatusCode TensorAdapter::GetDTAndShapeFromParameter(const ParameterPtr &param_node, TypeId *data_type,
ShapeVector *shape_vector) {
MS_ASSERT(param_node != nullptr && data_type != nullptr && shape_vector != nullptr);
auto abstract_base = param_node->abstract();
if (abstract_base == nullptr) {
MS_LOG(ERROR) << "Abstract of parameter is nullptr, " << param_node->name();
return kLiteError;
}
auto abstract_tensor = utils::cast<AbstractTensorPtr>(abstract_base);
if (abstract_tensor == nullptr) {
MS_LOG(ERROR) << "Abstract of parameter should be abstract tensor, " << param_node->name();
return kLiteError;
}
return GetDTAndShapeFromAbTensor(abstract_tensor, data_type, shape_vector);
}
} // namespace infer
} // namespace mindspore

87
mindspore/lite/src/extendrt/graph_compiler/anfnode_tensor_adapter.h Normal file
View File

@ -0,0 +1,87 @@
/**
* Copyright 2023 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_ANFNODE_TENSOR_ADAPTER_H_
#define MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_ANFNODE_TENSOR_ADAPTER_H_
#include <string>
#include <memory>
#include <unordered_map>
#include <vector>
#include "src/extendrt/tensor.h"
#include "abstract/abstract_value.h"
#include "ir/anf.h"
#include "include/api/status.h"
namespace mindspore {
namespace infer {
class TensorAdapter;
using TensorAdapterPtr = std::shared_ptr<TensorAdapter>;
class TensorAdapter {
public:
TensorAdapter() = default;
virtual ~TensorAdapter() {
if (own_data_) {
free(data_);
}
}
Tensor *ToTensor(const std::string &tensor_name = "");
static TensorAdapterPtr Create(const ParameterPtr &param_node);
static TensorAdapterPtr Create(const ValueNodePtr &value_node);
static TensorAdapterPtr Create(const mindspore::abstract::AbstractTensorPtr &abstract);
static TensorAdapterPtr Create(const mindspore::abstract::AbstractBasePtr &abstract);
static Tensor *Convert2Tensor(const ParameterPtr &param_node, const std::string &tensor_name = "");
static Tensor *Convert2Tensor(const ValueNodePtr &value_node, const std::string &tensor_name = "");
static Tensor *Convert2Tensor(const mindspore::abstract::AbstractTensorPtr &abstract,
const std::string &tensor_name = "");
static Tensor *Convert2Tensor(const mindspore::abstract::AbstractBasePtr &abstract,
const std::string &tensor_name = "");
private:
static StatusCode GetDTAndShapeFromAbTensor(const mindspore::abstract::AbstractTensorPtr &abstract, TypeId *data_type,
ShapeVector *shape_vector);
static StatusCode GetDTAndShapeFromParameter(const ParameterPtr &param_node, TypeId *data_type,
ShapeVector *shape_vector);
static TensorAdapterPtr CreateFromTensorValueNode(const ValueNodePtr &value_node);
static TensorAdapterPtr CreateFromInt32ImmValue(const ValueNodePtr &value_node);
static TensorAdapterPtr CreateFromInt64ImmValue(const ValueNodePtr &value_node);
static TensorAdapterPtr CreateFromBoolImmValue(const ValueNodePtr &value_node);
static TensorAdapterPtr CreateFromNumberTypeValue(const ValueNodePtr &value_node);
static TensorAdapterPtr CreateFromIntSequenceValue(const ValueNodePtr &value_node);
public:
Format format_{DEFAULT_FORMAT};
TensorCompressionType compress_type_{kNoCompression};
TypeId data_type_{kTypeUnknown};
bool is_const_{false};
std::vector<int64_t> shape_{};
void *data_{nullptr};
size_t data_len_{0};
bool own_data_{true};
};
} // namespace infer
} // namespace mindspore
#endif

376
mindspore/lite/src/extendrt/graph_compiler/compile_result.cc Normal file
View File

@ -0,0 +1,376 @@
/**
* Copyright 2023 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "src/extendrt/graph_compiler/compile_result.h"
#include <string>
#include <memory>
#include <sstream>
#include <unordered_map>
#include <utility>
#include <vector>
#include <algorithm>
#include "src/extendrt/tensor.h"
#include "ops/base_operator.h"
#include "utils/hash_map.h"
#include "include/api/status.h"
#include "ir/primitive.h"
#include "ops/op_name.h"
#include "ops/primitive_c.h"
namespace mindspore {
namespace infer {
namespace {
constexpr char tab[] = " ";
inline std::string GenIndent(int indent) {
std::ostringstream oss;
for (int i = 0; i < indent; i++) {
oss << tab;
}
return oss.str();
}
inline std::string DumpIntShape(const std::vector<int> &shape) {
std::ostringstream oss;
oss << "[";
for (size_t i = 0; i < shape.size(); i++) {
oss << shape[i];
if (i < shape.size() - 1) {
oss << ", ";
}
}
oss << "]";
return oss.str();
}
inline std::string DumpTensor(const Tensor *tensor, int indent = 0) {
std::ostringstream oss;
oss << GenIndent(indent) << "Tensor <name:" << tensor->tensor_name() << ", dtype:" << tensor->data_type()
<< ", format:" << tensor->format() << ", shape:" << DumpIntShape(tensor->shape()) << ">";
return oss.str();
}
} // namespace
CompileNode *CompileNode::Create(CNodePtr cnode) {
if (cnode == nullptr) {
return nullptr;
}
auto node = new CompileNode(cnode->fullname_with_scope());
auto primitive = GetValueNode<std::shared_ptr<Primitive>>(cnode->input(0));
if (primitive == nullptr) {
MS_LOG(ERROR) << "Node has no primitive, first input of cnode(" << cnode->fullname_with_scope()
<< ") is : " << cnode->input(0);
return nullptr;
}
node->type_ = primitive->name();
ops::PrimitiveCPtr primc{nullptr};
if (utils::isa<ops::PrimitiveCPtr>(primitive)) {
primc = utils::cast<ops::PrimitiveCPtr>(primitive);
} else {
static auto ops_primc_fns = ops::OpPrimCRegister::GetInstance().GetPrimCMap();
auto primc_creator_iter = ops_primc_fns.find(node->type_);
if (primc_creator_iter == ops_primc_fns.end()) {
MS_LOG(ERROR) << "Can not find primitive_c create function for: " << node->type_;
return nullptr;
}
primc = primc_creator_iter->second();
if (primc == nullptr) {
MS_LOG(ERROR) << "Create primitive_c failed, type: " << node->type_;
return nullptr;
}
primc->SetAttrs(primitive->attrs());
}
static auto baseops_fns = ops::OperatorRegister::GetInstance().GetOperatorMap();
auto baseops_creator_iter = baseops_fns.find(node->type_);
if (baseops_creator_iter == baseops_fns.end()) {
MS_LOG(ERROR) << "Can not find base-operator create function for: " << node->type_;
return nullptr;
}
auto baseops_creator = baseops_creator_iter->second;
node->base_operator_ = baseops_creator(primc);
if (node->base_operator_ == nullptr) {
MS_LOG(ERROR) << "Create base-operator failed, type: " << node->type_;
return nullptr;
}
node->cnode_ = std::move(cnode);
return node;
}
void CompileNode::AppendInputTensor(Tensor *tensor) {
if (tensor->tensor_name().empty()) {
tensor->set_tensor_name(this->name_ + "_in_" + std::to_string(this->inputs_.size()));
}
this->inputs_.emplace_back(tensor);
}
void CompileNode::AppendOutputTensor(Tensor *tensor) {
if (tensor->tensor_name().empty()) {
tensor->set_tensor_name(this->name_ + "_out_" + std::to_string(this->inputs_.size()));
}
this->outputs_.emplace_back(tensor);
}
std::string CompileNode::Dump(int indent) const {
constexpr int kNumberTwo = 2;
std::ostringstream oss;
oss << GenIndent(indent) << "CompileNode <name:" << name_ << ", type:" << type_ << "> {" << std::endl;
oss << GenIndent(indent + 1) << "inputs: [" << std::endl;
for (auto &input : inputs_) {
oss << DumpTensor(input, indent + kNumberTwo) << std::endl;
}
oss << GenIndent(indent + 1) << "]" << std::endl;
oss << GenIndent(indent + 1) << "outputs: [" << std::endl;
for (auto &output : outputs_) {
oss << DumpTensor(output, indent + kNumberTwo) << std::endl;
}
oss << GenIndent(indent + 1) << "]" << std::endl;
oss << GenIndent(indent) << "}";
return oss.str();
}
void CompileNode::ReplaceInputTensor(Tensor *dst, Tensor *src) {
std::replace_if(
inputs_.begin(), inputs_.end(), [&src](Tensor *ele) { return ele == src; }, dst);
}
CompileNode *CompileResult::GetNode(const std::string &name) {
auto iter = node_map_.find(name);
if (iter == node_map_.end()) {
return nullptr;
} else {
return iter->second;
}
}
CompileNode *CompileResult::GetArgNode(const std::string &name) {
auto iter = arg_node_map_.find(name);
if (iter == arg_node_map_.end()) {
return nullptr;
} else {
return iter->second;
}
}
std::vector<CompileNode *> &CompileResult::GetMutableNodes() {
if (assembled_) {
MS_LOG(EXCEPTION) << "CompileResult not mutable after build.";
}
return nodes_;
}
std::vector<Tensor *> &CompileResult::GetMutableInputs() {
if (assembled_) {
MS_LOG(EXCEPTION) << "CompileResult not mutable after build.";
}
return inputs_;
}
StatusCode CompileResult::AppendNode(CompileNode *node) {
if (assembled_) {
MS_LOG(EXCEPTION) << "CompileResult not mutable after build.";
}
if (node == nullptr) {
MS_LOG(ERROR) << "Input node is nullptr";
return kLiteInputParamInvalid;
}
const std::string &node_name = node->GetName();
auto iter = node_map_.find(node_name);
if (iter != node_map_.end()) {
MS_LOG(ERROR) << "Duplicated node name : " << node_name;
return kLiteError;
}
node_map_[node_name] = node;
nodes_.emplace_back(node);
return kSuccess;
}
StatusCode CompileResult::AppendArgNode(CompileNode *node) {
if (assembled_) {
MS_LOG(EXCEPTION) << "CompileResult not mutable after build.";
}
if (node == nullptr) {
MS_LOG(ERROR) << "Input node is nullptr";
return kLiteInputParamInvalid;
}
const std::string &node_name = node->GetName();
auto iter = arg_node_map_.find(node_name);
if (iter != arg_node_map_.end()) {
MS_LOG(ERROR) << "Duplicated node name : " << node_name;
return kLiteError;
}
arg_node_map_[node_name] = node;
arg_nodes_.emplace_back(node);
return kSuccess;
}
StatusCode CompileResult::AppendTensor(Tensor *tensor) {
if (assembled_) {
MS_LOG(EXCEPTION) << "CompileResult not mutable after build.";
}
if (tensor == nullptr) {
MS_LOG(ERROR) << "Input tensor is nullptr";
return kLiteInputParamInvalid;
}
const auto &tensor_name = tensor->tensor_name();
auto iter = tensor_map_.find(tensor_name);
if (iter != tensor_map_.end()) {
MS_LOG(ERROR) << "Duplicated tensor name : " << tensor_name;
return kLiteError;
}
tensors_.emplace_back(tensor);
tensor_map_[tensor_name] = tensor;
return kSuccess;
}
StatusCode CompileResult::AppendInputTensor(Tensor *tensor, bool is_borrow) {
if (assembled_) {
MS_LOG(EXCEPTION) << "CompileResult not mutable after build.";
}
if (tensor == nullptr) {
MS_LOG(ERROR) << "Input tensor is nullptr";
return kLiteInputParamInvalid;
}
if (tensor->tensor_name().empty()) {
MS_LOG(ERROR) << "Input tensor has no name";
return kLiteError;
}
if (!is_borrow) {
return AppendTensor(tensor);
}
inputs_.emplace_back(tensor);
return kSuccess;
}
StatusCode CompileResult::AppendOutputTensor(Tensor *tensor, bool is_borrow) {
if (assembled_) {
MS_LOG(EXCEPTION) << "CompileResult not mutable after build.";
}
if (tensor == nullptr) {
MS_LOG(ERROR) << "Input tensor is nullptr";
return kLiteInputParamInvalid;
}
if (tensor->tensor_name().empty()) {
tensor->set_tensor_name("graph_out_" + std::to_string(this->outputs_.size()));
}
if (!is_borrow) {
return AppendTensor(tensor);
}
outputs_.emplace_back(tensor);
return kSuccess;
}
StatusCode CompileResult::AppendNodeInputTensor(const CompileNode *compile_node, Tensor *tensor, bool is_borrow) {
if (compile_node == nullptr) {
MS_LOG(ERROR) << "Input compile_node is nullptr";
return kLiteInputParamInvalid;
}
return AppendNodeInputTensor(compile_node->GetName(), tensor, is_borrow);
}
StatusCode CompileResult::AppendNodeInputTensor(const std::string &node_name, Tensor *tensor, bool is_borrow) {
if (assembled_) {
MS_LOG(EXCEPTION) << "CompileResult not mutable after build.";
}
if (tensor == nullptr) {
MS_LOG(ERROR) << "Input tensor is nullptr";
return kLiteInputParamInvalid;
}
auto iter = node_map_.find(node_name);
if (iter == node_map_.end()) {
MS_LOG(ERROR) << "CompileNode not belong to this graph, node: " << node_name;
return kLiteError;
}
iter->second->AppendInputTensor(tensor);
if (!is_borrow) {
return AppendTensor(tensor);
}
return kSuccess;
}
StatusCode CompileResult::AppendNodeOutputTensor(const CompileNode *compile_node, Tensor *tensor, bool is_borrow) {
if (compile_node == nullptr) {
MS_LOG(ERROR) << "Input compile_node is nullptr";
return kLiteInputParamInvalid;
}
return AppendNodeOutputTensor(compile_node->GetName(), tensor, is_borrow);
}
StatusCode CompileResult::AppendNodeOutputTensor(const std::string &node_name, Tensor *tensor, bool is_borrow) {
if (assembled_) {
MS_LOG(EXCEPTION) << "CompileResult not mutable after build.";
}
if (tensor == nullptr) {
MS_LOG(ERROR) << "Input tensor is nullptr";
return kLiteInputParamInvalid;
}
auto iter = node_map_.find(node_name);
if (iter == node_map_.end()) {
MS_LOG(ERROR) << "CompileNode not belong to this graph, node: " << node_name;
return kLiteError;
}
iter->second->AppendOutputTensor(tensor);
if (!is_borrow) {
return AppendTensor(tensor);
}
return kSuccess;
}
std::string CompileResult::Dump(int indent) const {
constexpr int kNumberTwo = 2;
std::ostringstream oss;
oss << GenIndent(indent) << "CompileResult {" << std::endl;
oss << GenIndent(indent + 1) << "nodes: [" << std::endl;
for (auto &node : nodes_) {
oss << node->Dump(indent + kNumberTwo) << std::endl;
}
oss << GenIndent(indent + 1) << "]" << std::endl;
oss << GenIndent(indent + 1) << "inputs: [" << std::endl;
for (auto &input : inputs_) {
oss << DumpTensor(input, indent + kNumberTwo) << std::endl;
}
oss << GenIndent(indent + 1) << "]" << std::endl;
oss << GenIndent(indent + 1) << "outputs: [" << std::endl;
for (auto &output : outputs_) {
oss << DumpTensor(output, indent + kNumberTwo) << std::endl;
}
oss << GenIndent(indent + 1) << "]" << std::endl;
oss << GenIndent(indent + 1) << "tensors: [" << std::endl;
for (auto &tensor : tensors_) {
oss << DumpTensor(tensor, indent + kNumberTwo) << std::endl;
}
oss << GenIndent(indent + 1) << "]" << std::endl;
oss << GenIndent(indent) << "}" << std::endl;
return oss.str();
}
CompileResult::~CompileResult() {
for (auto &node : nodes_) {
delete (node);
}
nodes_.clear();
for (auto &node : arg_nodes_) {
delete (node);
}
arg_nodes_.clear();
for (auto &tensor : tensors_) {
delete (tensor);
}
tensors_.clear();
}
} // namespace infer
} // namespace mindspore

125
mindspore/lite/src/extendrt/graph_compiler/compile_result.h Normal file
View File

@ -0,0 +1,125 @@
/**
* Copyright 2023 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_COMPILE_RESULT_H_
#define MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_COMPILE_RESULT_H_
#include <string>
#include <memory>
#include <sstream>
#include <unordered_map>
#include <utility>
#include <vector>
#include "ir/anf.h"
#include "src/extendrt/tensor.h"
#include "include/model.h"
#include "ops/base_operator.h"
#include "utils/hash_map.h"
#include "include/api/status.h"
namespace mindspore {
namespace infer {
class CompileNode {
public:
explicit CompileNode(std::string name) : name_(std::move(name)) {}
static CompileNode *Create(CNodePtr cnode);
virtual ~CompileNode() = default;
std::string GetName() const { return name_; }
std::string GetType() const { return type_; }
std::shared_ptr<ops::BaseOperator> GetBaseOperator() const { return base_operator_; }
CNodePtr GetCNode() const { return cnode_; }
const std::vector<Tensor *> &GetInputs() const { return inputs_; }
Tensor *GetInput(size_t i) const { return inputs_.at(i); }
size_t InputSize() const { return inputs_.size(); }
const std::vector<Tensor *> &GetOutputs() const { return outputs_; }
Tensor *GetOutput(size_t i) const { return outputs_.at(i); }
size_t OutputSize() const { return outputs_.size(); }
void SetName(const std::string &name) { name_ = name; }
void AppendInputTensor(Tensor *tensor);
void AppendOutputTensor(Tensor *tensor);
void ReplaceInputTensor(Tensor *dst, Tensor *src);
std::string Dump(int indent = 0) const;
private:
std::string name_{};
std::string type_{};
std::shared_ptr<ops::BaseOperator> base_operator_{nullptr};
CNodePtr cnode_{nullptr};
std::vector<Tensor *> inputs_{};
std::vector<Tensor *> outputs_{};
};
class CompileResult {
public:
explicit CompileResult(Format format) : base_format_(format) {}
virtual ~CompileResult();
Format GetFormat() const { return base_format_; }
CompileNode *GetNode(const std::string &name);
CompileNode *GetArgNode(const std::string &name);
const std::vector<CompileNode *> &GetNodes() const { return nodes_; }
size_t NodeSize() const { return nodes_.size(); }
const std::vector<Tensor *> &GetTensors() const { return tensors_; }
size_t TensorSize() const { return tensors_.size(); }
const std::vector<Tensor *> &GetInputs() const { return inputs_; }
Tensor *GetInput(size_t i) const { return inputs_.at(i); }
size_t InputSize() const { return inputs_.size(); }
const std::vector<Tensor *> &GetOutputs() const { return outputs_; }
Tensor *GetOutput(size_t i) const { return outputs_.at(i); }
size_t OutputSize() const { return outputs_.size(); }
const std::vector<CompileNode *> &GetParamNodes() const { return param_nodes_; }
const std::vector<CompileNode *> &GetReturnNodes() const { return return_nodes_; }
std::vector<CompileNode *> &GetMutableNodes();
std::vector<Tensor *> &GetMutableInputs();
StatusCode AppendNode(CompileNode *node);
StatusCode AppendArgNode(CompileNode *node);
StatusCode AppendTensor(Tensor *tensor);
StatusCode AppendInputTensor(Tensor *tensor, bool is_borrow = false);
StatusCode AppendOutputTensor(Tensor *tensor, bool is_borrow = false);
StatusCode AppendNodeInputTensor(const CompileNode *compile_node, Tensor *tensor, bool is_borrow = false);
StatusCode AppendNodeInputTensor(const std::string &node_name, Tensor *tensor, bool is_borrow = false);
StatusCode AppendNodeOutputTensor(const CompileNode *compile_node, Tensor *tensor, bool is_borrow = false);
StatusCode AppendNodeOutputTensor(const std::string &node_name, Tensor *tensor, bool is_borrow = false);
void Assemble() { this->assembled_ = true; }
std::string Dump(int indent = 0) const;
private:
bool assembled_ = false;
std::vector<CompileNode *> nodes_{};
std::vector<Tensor *> tensors_{};
std::vector<Tensor *> inputs_{};
std::vector<Tensor *> outputs_{};
HashMap<std::string, CompileNode *> node_map_{};
HashMap<std::string, Tensor *> tensor_map_{};
std::vector<CompileNode *> param_nodes_{};
std::vector<CompileNode *> return_nodes_{};
std::vector<CompileNode *> arg_nodes_{};
HashMap<std::string, CompileNode *> arg_node_map_{};
Format base_format_{DEFAULT_FORMAT};
};
using CompileResultPtr = std::shared_ptr<CompileResult>;
} // namespace infer
} // namespace mindspore
#endif

464
mindspore/lite/src/extendrt/graph_compiler/compile_result_builder.cc Normal file
View File

@ -0,0 +1,464 @@
/**
* Copyright 2023 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "src/extendrt/graph_compiler/compile_result_builder.h"
#include <algorithm>
#include "src/extendrt/graph_compiler/anfnode_tensor_adapter.h"
#include "ir/anf.h"
#include "ir/func_graph.h"
#include "ir/primitive.h"
#include "ops/core_ops.h"
#include "ops/op_name.h"
#include "ops/primitive_c.h"
using AbstractBasePtr = mindspore::abstract::AbstractBasePtr;
using AbstractTensorPtr = mindspore::abstract::AbstractTensorPtr;
using AbstractSequencePtr = mindspore::abstract::AbstractSequencePtr;
namespace mindspore {
namespace infer {
StatusCode CompileResultBuilder::BuildInputs(const AnfNodePtrList &inputs) {
MS_ASSERT(graph_ != nullptr);
if (graph_->InputSize() > 0) {
MS_LOG(ERROR) << "Please don't call BuildOutputs twice.";
return kLiteError;
}
std::vector<Tensor *> results;
for (auto &input : inputs) {
results.clear();
auto ret = CreateTensorsFromAbstract(input->abstract(), &results);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Create tensors from abstract of segments input failed, input : "
<< input->fullname_with_scope();
return ret;
}
auto arg_node = new CompileNode(input->fullname_with_scope());
ret = graph_->AppendArgNode(arg_node);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Append input lite-node to graph failed, input : " << input->fullname_with_scope();
return ret;
}
for (auto &result : results) {
arg_node->AppendOutputTensor(result);
ret = graph_->AppendInputTensor(result);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Append output tensor to argument node failed, node: " << input->fullname_with_scope();
return ret;
}
}
}
return kSuccess;
}
StatusCode CompileResultBuilder::BuildNodes(const GraphSegmentPtr &graph_segment) {
MS_ASSERT(graph_ != nullptr);
if (graph_->NodeSize() > 0) {
MS_LOG(ERROR) << "Please don't call BuildNodes twice.";
return kLiteError;
}
for (auto &node : graph_segment->nodes_) {
if (!utils::isa<CNodePtr>(node)) {
continue;
}
auto ret = CreateAndAppendNode(utils::cast<CNodePtr>(node));
if (ret != kSuccess) {
MS_LOG(ERROR) << "Create compile node from cnode failed : " << node;
return ret;
}
}
return kSuccess;
}
StatusCode CompileResultBuilder::BuildOutputs(const AnfNodePtrList &outputs) {
MS_ASSERT(graph_ != nullptr);
if (graph_->OutputSize() > 0) {
MS_LOG(ERROR) << "Please don't call BuildOutputs twice.";
return kLiteError;
}
for (auto &output : outputs) {
auto out_cnode = utils::cast<CNodePtr>(output);
if (out_cnode == nullptr) {
MS_LOG(ERROR) << "Outputs should be a CNode vector, but got " << output->Type() << " type element.";
return kLiteError;
}
for (auto &input : out_cnode->inputs()) {
if (!utils::isa<CNodePtr>(input)) {
continue;
}
auto compile_node = graph_->GetNode(input->fullname_with_scope());
if (compile_node == nullptr) {
continue;
}
for (auto &tensor : compile_node->GetOutputs()) {
auto ret = graph_->AppendOutputTensor(tensor, true);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Append output tensor to graph failed, output: " << out_cnode->fullname_with_scope();
return ret;
}
}
}
}
return kSuccess;
}
void CompileResultBuilder::IsolateTensor(Tensor *dst_tensor, const CompileNode *node, size_t index) {
if (node == nullptr || index >= node->OutputSize()) {
return;
}
auto *src_tensor = node->GetOutput(index);
// used as inputs of other node
auto &nodes = graph_->GetMutableNodes();
for (auto &compile_node : nodes) {
if (compile_node == nullptr) {
continue;
}
compile_node->ReplaceInputTensor(dst_tensor, src_tensor);
}
// used as outputs of graph
auto &inputs = graph_->GetMutableInputs();
std::replace_if(
inputs.begin(), inputs.end(), [&src_tensor](Tensor *ele) { return ele == src_tensor; }, dst_tensor);
}
StatusCode CompileResultBuilder::RemoveMakeSeqNode() {
auto &nodes = graph_->GetMutableNodes();
for (auto iter = nodes.begin(); iter != nodes.end();) {
auto &node = *iter;
if (node->GetType() != prim::kMakeTuple && node->GetType() != prim::kMakeList) {
iter++;
continue;
}
MS_LOG(INFO) << "Handling make sequence node: " << node->GetName();
auto tensor_number = node->InputSize();
if (tensor_number != node->OutputSize()) {
MS_LOG(ERROR) << "MakeSequence node should has same number of inputs and outputs, but got " << tensor_number
<< " inputs and " << node->OutputSize() << " outputs.";
return kLiteError;
}
for (size_t i = 0; i < tensor_number; i++) {
IsolateTensor(node->GetInput(i), node, i);
}
iter = nodes.erase(iter);
}
return kSuccess;
}
StatusCode CompileResultBuilder::RemoveDependNode() {
auto &nodes = graph_->GetMutableNodes();
for (auto iter = nodes.begin(); iter != nodes.end();) {
auto &node = *iter;
if (node->GetType() != prim::kDepend) {
iter++;
continue;
}
MS_LOG(INFO) << "Handling Depend node: " << node->GetName();
constexpr int kSize2 = 2;
if (node->InputSize() != kSize2) {
MS_LOG(ERROR) << "Depend node should has 2 inputs, but got " << node->InputSize();
return kLiteError;
}
if (node->OutputSize() != 1) {
MS_LOG(ERROR) << "Depend node should has 1 outputs, but got " << node->OutputSize();
return kLiteError;
}
IsolateTensor(node->GetInput(0), node, 0);
iter = nodes.erase(iter);
}
return kSuccess;
}
StatusCode CompileResultBuilder::RemoveSeqGetItemNode() {
auto &nodes = graph_->GetMutableNodes();
for (auto iter = nodes.begin(); iter != nodes.end();) {
auto &node = *iter;
if (node->GetType() != prim::kTupleGetItem && node->GetType() != prim::kListGetItem &&
node->GetType() != "array_getitem" && node->GetType() != prim::kSliceGetItem) {
iter++;
continue;
}
MS_LOG(INFO) << "Handling GetItem node: " << node->GetName();
constexpr int kSize2 = 2;
if (node->InputSize() != kSize2) {
MS_LOG(ERROR) << "GetItem node should has 2 inputs, but got " << node->InputSize();
return kLiteError;
}
if (node->OutputSize() != 1) {
MS_LOG(ERROR) << "GetItem node should has 1 outputs, but got " << node->OutputSize();
return kLiteError;
}
auto index_tensor = node->GetInput(1);
if (index_tensor->data() == nullptr) {
MS_LOG(ERROR) << "`index_tensor` of GetItem should be a const tensor, but has no data.";
return kLiteError;
}
if (index_tensor->data_type() == kNumberTypeInt32) {
auto idx = reinterpret_cast<int32_t *>(index_tensor->data())[0];
IsolateTensor(node->GetInput(idx), node, 0);
} else if (index_tensor->data_type() == kNumberTypeInt64) {
auto idx = reinterpret_cast<int64_t *>(index_tensor->data())[0];
IsolateTensor(node->GetInput(idx), node, 0);
} else {
MS_LOG(ERROR) << "`index_tensor` of GetItem should be a const tensor with int data type, but got "
<< index_tensor->data_type();
return kLiteError;
}
iter = nodes.erase(iter);
}
return kSuccess;
}
StatusCode CompileResultBuilder::OptimizeGraph() {
MS_ASSERT(graph_ != nullptr);
auto ret = RemoveDependNode();
if (ret != kSuccess) {
MS_LOG(ERROR) << "Handle Depend node failed";
return ret;
}
ret = RemoveMakeSeqNode();
if (ret != kSuccess) {
MS_LOG(ERROR) << "Handle Make Sequence node failed";
return ret;
}
ret = RemoveSeqGetItemNode();
if (ret != kSuccess) {
MS_LOG(ERROR) << "Handle Sequence-Getitem node failed";
return ret;
}
return kSuccess;
}
CompileResultPtr CompileResultBuilder::Build(const GraphSegmentPtr &graph_segment, const AnfNodePtrList &inputs,
const AnfNodePtrList &outputs) {
graph_ = std::make_shared<CompileResult>(graph_format_);
if (BuildInputs(inputs) != kSuccess) {
MS_LOG(ERROR) << "Build graph inputs failed";
return nullptr;
}
if (BuildNodes(graph_segment) != kSuccess) {
MS_LOG(ERROR) << "Build graph nodes failed";
return nullptr;
}
if (BuildOutputs(outputs) != kSuccess) {
MS_LOG(ERROR) << "Build graph outputs failed";
return nullptr;
}
if (OptimizeGraph() != kSuccess) {
MS_LOG(ERROR) << "Optimize graph failed";
return nullptr;
}
graph_->Assemble();
return graph_;
}
StatusCode CompileResultBuilder::AppendInputCNodeToInputs(const CNodePtr &cnode, const CompileNode *compile_node) {
if (cnode == nullptr) {
MS_LOG(ERROR) << "Input cnode is nullptr.";
return kLiteInputParamInvalid;
}
if (compile_node == nullptr) {
MS_LOG(ERROR) << "Input compile_node is nullptr.";
return kLiteInputParamInvalid;
}
auto input_node = graph_->GetNode(cnode->fullname_with_scope());
if (input_node == nullptr) {
input_node = graph_->GetArgNode(cnode->fullname_with_scope());
}
if (input_node == nullptr) {
MS_LOG(ERROR) << "Can not find input lite-node in graph, node: " << cnode->fullname_with_scope();
return kLiteError;
}
for (auto &input_node_output : input_node->GetOutputs()) {
auto ret = graph_->AppendNodeInputTensor(compile_node, input_node_output, true);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Append input tensor for node failed, node: " << compile_node->GetName();
return ret;
}
}
return kSuccess;
}
StatusCode CompileResultBuilder::AppendInputParameterToInputs(const ParameterPtr &param_node,
const CompileNode *compile_node) {
if (param_node == nullptr) {
MS_LOG(ERROR) << "Input param_node is nullptr.";
return kLiteInputParamInvalid;
}
if (compile_node == nullptr) {
MS_LOG(ERROR) << "Input compile_node is nullptr.";
return kLiteInputParamInvalid;
}
auto arg_node = graph_->GetArgNode(param_node->fullname_with_scope());
if (arg_node != nullptr) {
for (auto &output : arg_node->GetOutputs()) {
auto ret = graph_->AppendNodeInputTensor(compile_node, output, true);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Append input tensor for node failed, node: " << compile_node->GetName();
return ret;
}
}
return kSuccess;
}
auto tensor = TensorAdapter::Convert2Tensor(param_node);
if (tensor == nullptr) {
MS_LOG(ERROR) << "Create tensor from Parameter failed.";
return kLiteError;
}
auto format_value = compile_node->GetBaseOperator()->GetAttr(mindspore::ops::kFormat);
if (format_value != nullptr) {
tensor->set_format(static_cast<Format>(GetValue<int64_t>(format_value)));
} else {
tensor->set_format(graph_format_);
}
auto ret = graph_->AppendNodeInputTensor(compile_node, tensor);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Append input tensor for node failed, node: " << compile_node->GetName();
return ret;
}
return kSuccess;
}
StatusCode CompileResultBuilder::AppendInputValueNodeToInputs(const ValueNodePtr &value_node,
const CompileNode *compile_node) {
if (value_node == nullptr) {
MS_LOG(ERROR) << "Input value_node is nullptr.";
return kLiteInputParamInvalid;
}
if (compile_node == nullptr) {
MS_LOG(ERROR) << "Input compile_node is nullptr.";
return kLiteInputParamInvalid;
}
auto tensor = TensorAdapter::Convert2Tensor(value_node);
if (tensor == nullptr) {
MS_LOG(ERROR) << "Create tensor from ValueNode failed.";
return kLiteError;
}
auto format_value = compile_node->GetBaseOperator()->GetAttr(mindspore::ops::kFormat);
if (format_value != nullptr) {
tensor->set_format(static_cast<Format>(GetValue<int64_t>(format_value)));
} else {
tensor->set_format(graph_format_);
}
auto ret = graph_->AppendNodeInputTensor(compile_node, tensor);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Append input tensor for node failed, node: " << compile_node->GetName();
return ret;
}
return kSuccess;
}
StatusCode CompileResultBuilder::CreateAndAppendNode(const CNodePtr &cnode) {
auto compile_node = CompileNode::Create(cnode);
if (compile_node == nullptr) {
MS_LOG(ERROR) << "Create compile node failed, cnode: " << cnode->fullname_with_scope();
return kLiteError;
}
auto ret = graph_->AppendNode(compile_node);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Append compile_node to graph failed, node: " << compile_node->GetName();
return ret;
}
// inputs
for (size_t i = 1; i < cnode->size(); i++) {
auto &input = cnode->input(i);
if (utils::isa<CNodePtr>(input)) {
ret = this->AppendInputCNodeToInputs(utils::cast<CNodePtr>(input), compile_node);
} else if (utils::isa<Parameter>(input)) {
ret = this->AppendInputParameterToInputs(utils::cast<ParameterPtr>(input), compile_node);
} else if (utils::isa<ValueNode>(input)) {
ret = this->AppendInputValueNodeToInputs(utils::cast<ValueNodePtr>(input), compile_node);
} else {
MS_LOG(ERROR) << "Unsupported input node of cnode: " << input
<< ", current cnode: " << cnode->fullname_with_scope();
ret = kLiteNotSupport;
}
if (ret != kSuccess) {
MS_LOG(ERROR) << "Create input tensor for cnode failed, cnode: " << cnode->fullname_with_scope();
return ret;
}
}
// outputs
ret = BuildNodeOutputTensor(cnode, compile_node);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Create output tensors of cnode failed, cnode: " << cnode;
return ret;
}
return kSuccess;
}
StatusCode CompileResultBuilder::CreateTensorsFromAbstract(const AbstractBasePtr &abstract,
std::vector<Tensor *> *results) {
if (results == nullptr) {
MS_LOG(ERROR) << "Result is nullptr.";
return kLiteInputParamInvalid;
}
results->clear();
// multi output abstract
if (utils::isa<AbstractSequencePtr>(abstract)) {
auto elements = utils::cast<AbstractSequencePtr>(abstract)->elements();
for (auto &element : elements) {
auto tensor = TensorAdapter::Convert2Tensor(element);
if (tensor == nullptr) {
MS_LOG(ERROR) << "Create tensor from abstract failed, abstract : " << element;
return kLiteError;
}
results->emplace_back(tensor);
}
return kSuccess;
}
// single output abstract
if (utils::isa<AbstractTensorPtr>(abstract)) {
auto tensor = TensorAdapter::Convert2Tensor(abstract);
if (tensor == nullptr) {
MS_LOG(ERROR) << "Create tensor from abstract failed, abstract : " << abstract;
return kLiteError;
}
results->emplace_back(tensor);
return kSuccess;
}
MS_LOG(ERROR) << "Unsupported abstract: " << abstract;
return kLiteNotSupport;
}
StatusCode CompileResultBuilder::BuildNodeOutputTensor(const CNodePtr &cnode, const CompileNode *compile_node) {
if (cnode == nullptr) {
MS_LOG(ERROR) << "Input cnode is nullptr.";
return kLiteInputParamInvalid;
}
if (compile_node == nullptr) {
MS_LOG(ERROR) << "Input compile_node is nullptr.";
return kLiteInputParamInvalid;
}
std::vector<Tensor *> results;
auto ret = CreateTensorsFromAbstract(cnode->abstract(), &results);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Create tensors from output abstract of cnode failed, cnode : " << cnode->fullname_with_scope();
return ret;
}
if (compile_node->OutputSize() > 0) {
MS_LOG(ERROR) << "Build node output twice, node : " << compile_node->GetName();
return kLiteError;
}
for (auto &result : results) {
ret = graph_->AppendNodeOutputTensor(compile_node, result);
if (ret != kSuccess) {
MS_LOG(ERROR) << "Append output tensor to node failed, node: " << compile_node->GetName();
return ret;
}
}
return kSuccess;
}
} // namespace infer
} // namespace mindspore

69
mindspore/lite/src/extendrt/graph_compiler/compile_result_builder.h Normal file
View File

@ -0,0 +1,69 @@
/**
* Copyright 2023 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_COMPILE_RESULT_BUILDER_H_
#define MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_COMPILE_RESULT_BUILDER_H_
#include <string>
#include <memory>
#include <set>
#include <unordered_map>
#include <vector>
#include "src/extendrt/graph_compiler/compile_result.h"
#include "src/extendrt/tensor.h"
#include "abstract/abstract_value.h"
#include "ir/anf.h"
#include "include/api/status.h"
namespace mindspore {
namespace infer {
class CompileResultBuilder {
public:
explicit CompileResultBuilder(Format graph_format) : graph_format_(graph_format) {}
~CompileResultBuilder() = default;
CompileResultPtr Build(const GraphSegmentPtr &graph_segment, const AnfNodePtrList &inputs,
const AnfNodePtrList &outputs);
private:
// build
StatusCode BuildInputs(const AnfNodePtrList &inputs);
StatusCode BuildNodes(const GraphSegmentPtr &graph_segment);
StatusCode BuildOutputs(const AnfNodePtrList &outputs);
StatusCode OptimizeGraph();
// methods about node
StatusCode CreateAndAppendNode(const CNodePtr &cnode);
StatusCode AppendInputCNodeToInputs(const CNodePtr &cnode, const CompileNode *compile_node);
StatusCode AppendInputParameterToInputs(const ParameterPtr &param_node, const CompileNode *compile_node);
StatusCode AppendInputValueNodeToInputs(const ValueNodePtr &value_node, const CompileNode *compile_node);
// methods about tensor
static StatusCode CreateTensorsFromAbstract(const mindspore::abstract::AbstractBasePtr &abstract,
std::vector<Tensor *> *results);
StatusCode BuildNodeOutputTensor(const CNodePtr &cnode, const CompileNode *compile_node);
// methods about optimize
StatusCode RemoveSeqGetItemNode();
StatusCode RemoveMakeSeqNode();
StatusCode RemoveDependNode();
// Replace `index`th output tensor of `node` by `dst_tensor` tensor.
void IsolateTensor(Tensor *dst_tensor, const CompileNode *node, size_t index);
private:
CompileResultPtr graph_ = nullptr;
Format graph_format_{DEFAULT_FORMAT};
std::set<std::string> input_names_{};
};
} // namespace infer
} // namespace mindspore
#endif

2
mindspore/lite/src/extendrt/graph_compiler/default_graph_compiler.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019-2021 Huawei Technologies Co., Ltd
* Copyright 2023 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

13
mindspore/lite/src/extendrt/graph_compiler/default_graph_compiler.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019-2021 Huawei Technologies Co., Ltd
* Copyright 2023 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -33,15 +33,20 @@ class DefaultGraphCompiler : public mindspore::infer::abstract::GraphCompiler {
std::shared_ptr<infer::abstract::ExecutionPlan> Compile(FuncGraphPtr graph) override;
infer::abstract::ExecutionFlowPtr Compile(const GraphSegmentPtr &segment, const AnfNodePtrList &inputs,
const AnfNodePtrList &outputs) override {
return nullptr;
}
protected:
virtual std::vector<GraphSegmentPtr> Partition(const FuncGraphPtr &graph);
virtual std::shared_ptr<infer::abstract::ExecutionPlan> Schedule(const std::vector<GraphSegmentPtr> &graph_segments,
FuncGraphPtr func_graph);
FuncGraphPtr func_graph);
virtual std::shared_ptr<infer::abstract::ExecutionFlow> Schedule(const GraphSegmentPtr &graph_segment,
const std::vector<AnfNodePtr> &inputs,
const std::vector<AnfNodePtr> &outputs);
const std::vector<AnfNodePtr> &inputs,
const std::vector<AnfNodePtr> &outputs);
private:
infer::abstract::Tensor *CreateTensor(AnfNodePtr node);

133
mindspore/lite/src/extendrt/graph_compiler/main.cc Normal file
View File

@ -0,0 +1,133 @@
/**
* Copyright 2023 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <vector>
#include <cstdlib>
#include <iostream>
#include "ir/anf.h"
#include "ir/func_graph.h"
#include "abstract/abstract_value.h"
#include "ops/core_ops.h"
#include "include/api/context.h"
#include "src/litert/inner_context.h"
#include "src/litert/cxx_api/converters.h"
#include "backend/graph_compiler/graph_partition.h"
#include "backend/graph_compiler/segment_runner.h"
#include "src/extendrt/graph_compiler/single_graph_compiler.h"
namespace mindspore {
FuncGraphPtr CreateFuncGraph() {
std::vector<int64_t> shp{2, 2};
abstract::AbstractTensorPtr abs;
// Func graph.
auto func_graph = std::make_shared<FuncGraph>();
// Parameter.
auto abstract_x = std::make_shared<abstract::AbstractTensor>(kFloat32, shp);
auto parameter_x = func_graph->add_parameter();
parameter_x->set_name("input_x");
parameter_x->set_abstract(abstract_x);
auto abstract_y = std::make_shared<abstract::AbstractTensor>(kFloat32, shp);
auto parameter_y = func_graph->add_parameter();
parameter_y->set_name("input_y");
parameter_y->set_abstract(abstract_y);
auto parameters = func_graph->parameters();
// Add.
std::vector<AnfNodePtr> add_inputs{NewValueNode(prim::kPrimAdd), parameters[0], parameters[1]};
auto add_node = func_graph->NewCNode(add_inputs);
abs = std::make_shared<abstract::AbstractTensor>(kFloat32, shp);
add_node->set_abstract(abs);
// Reshape.
std::vector<AnfNodePtr> reshape_inputs{NewValueNode(prim::kPrimReshape), add_node};
auto reshape_node = func_graph->NewCNode(reshape_inputs);
abs = std::make_shared<abstract::AbstractTensor>(kFloat32, shp);
reshape_node->set_abstract(abs);
// sub.
std::vector<AnfNodePtr> sub_inputs{NewValueNode(prim::kPrimSub), reshape_node, parameters[0]};
auto sub_node = func_graph->NewCNode(sub_inputs);
abs = std::make_shared<abstract::AbstractTensor>(kFloat32, shp);
sub_node->set_abstract(abs);
// Return.
std::vector<AnfNodePtr> return_inputs{NewValueNode(prim::kPrimReturn), sub_node};
auto return_node = func_graph->NewCNode(return_inputs);
func_graph->set_return(return_node);
#ifdef USE_REAL_MODEL
const std::string model_path = ""; // /path/to/mobilenet.mindir
MindIRLoader mindir_loader;
func_graph = mindir_loader.LoadMindIR(model_path);
#endif
return func_graph;
}
std::tuple<GraphSegmentPtr, AnfNodePtrList, AnfNodePtrList> CreateSegment() {
auto func_graph = CreateFuncGraph();
std::cout << "============================================= func_graph inputs:" << std::endl;
for (auto &input : func_graph->get_inputs()) {
std::cout << input << ":" << input->fullname_with_scope() << std::endl;
}
auto new_manager = MakeManager({func_graph});
MS_EXCEPTION_IF_NULL(new_manager);
new_manager->AddFuncGraph(func_graph);
func_graph->set_manager(new_manager);
static const std::vector<PrimitivePtr> ms_nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial,
prim::kPrimSwitch, prim::kPrimMakeTuple,
prim::kPrimBpropCut, prim::kPrimSwitchLayer};
auto graph_partition = std::make_shared<compile::GraphPartition>(ms_nonlinear_ops, kMsConvert);
bool multi_target = false;
auto segments = graph_partition->Partition(func_graph, &multi_target);
auto segment = segments[0];
FuncGraphPtr fg;
AnfNodePtrList inputs;
AnfNodePtrList outputs;
std::tie(fg, inputs, outputs) = compile::TransformSegmentToAnfGraph(segment->nodes_);
std::cout << "============================================= segment:" << std::endl;
std::cout << "--------------------------------------------- nodes:" << std::endl;
for (auto &node : segment->nodes_) {
std::cout << node << ":" << node->fullname_with_scope() << std::endl;
}
std::cout << "--------------------------------------------- inputs:" << std::endl;
for (auto &input : inputs) {
std::cout << input << ":" << input->fullname_with_scope() << std::endl;
}
std::cout << "--------------------------------------------- outputs:" << std::endl;
for (auto &output : outputs) {
std::cout << output << ":" << output->fullname_with_scope() << std::endl;
}
return std::make_tuple(segment, inputs, outputs);
}
} // namespace mindspore
int main() {
mindspore::GraphSegmentPtr segment;
mindspore::AnfNodePtrList inputs;
mindspore::AnfNodePtrList outputs;
std::tie(segment, inputs, outputs) = mindspore::CreateSegment();
auto context = std::make_shared<mindspore::Context>();
auto device_info = std::make_shared<mindspore::CPUDeviceInfo>();
context->MutableDeviceInfo().emplace_back(device_info);
auto inner_context = std::shared_ptr<mindspore::lite::InnerContext>(mindspore::ContextUtils::Convert(context.get()));
auto compiler = std::make_shared<mindspore::infer::SingleGraphCompiler>(inner_context);
compiler->Compile(segment, inputs, outputs);
return 0;
}

50
mindspore/lite/src/extendrt/graph_compiler/single_graph_compiler.cc Normal file
View File

@ -0,0 +1,50 @@
/**
* Copyright 2023 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "src/extendrt/graph_compiler/single_graph_compiler.h"
#include <memory>
#include <iostream>
#include "src/extendrt/execution_flow.h"
#include "src/extendrt/graph_compiler/compile_result_builder.h"
namespace mindspore {
namespace infer {
CompileResultPtr SingleGraphCompiler::Build(const GraphSegmentPtr &segment, const AnfNodePtrList &inputs,
const AnfNodePtrList &outputs) {
// assume graph_format is NHWC
auto graph_format = NHWC;
auto builder = std::make_shared<CompileResultBuilder>(graph_format);
return builder->Build(segment, inputs, outputs);
}
abstract::ExecutionFlowPtr SingleGraphCompiler::Schedule(const CompileResultPtr &node_list) {
// infershape
// choose lite-kernel else choose kernel-mod
// tensor data type/format transform
if (MS_UNLIKELY(scheduler_ == nullptr)) {
scheduler_ = std::make_shared<SingleGraphScheduler>(this->context_);
}
return scheduler_->Schedule(node_list);
}
abstract::ExecutionFlowPtr SingleGraphCompiler::Compile(const GraphSegmentPtr &segment, const AnfNodePtrList &inputs,
const AnfNodePtrList &outputs) {
auto node_list = this->Build(segment, inputs, outputs);
std::cout << node_list->Dump() << std::endl;
return this->Schedule(node_list);
}
} // namespace infer
} // namespace mindspore

53
mindspore/lite/src/extendrt/graph_compiler/single_graph_compiler.h Normal file
View File

@ -0,0 +1,53 @@
/**
* Copyright 2023 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_SINGLE_GRAPH_COMPILER_H_
#define MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_SINGLE_GRAPH_COMPILER_H_
#include <string>
#include <memory>
#include <unordered_map>
#include <utility>
#include <vector>
#include "src/litert/inner_context.h"
#include "mindapi/ir/abstract.h"
#include "ir/anf.h"
#include "src/extendrt/graph_compiler/compile_result.h"
#include "src/extendrt/tensor.h"
#include "src/infer/execution_flow.h"
#include "src/infer/graph_compiler.h"
#include "src/extendrt/graph_compiler/single_graph_scheduler.h"
namespace mindspore {
namespace infer {
class SingleGraphCompiler : public abstract::GraphCompiler {
public:
explicit SingleGraphCompiler(std::shared_ptr<lite::InnerContext> context) : context_(std::move(context)) {}
~SingleGraphCompiler() override = default;
abstract::ExecutionPlanPtr Compile(FuncGraphPtr graph) override { return nullptr; }
abstract::ExecutionFlowPtr Compile(const GraphSegmentPtr &segment, const AnfNodePtrList &inputs,
const AnfNodePtrList &outputs) override;
private:
CompileResultPtr Build(const GraphSegmentPtr &segment, const AnfNodePtrList &inputs, const AnfNodePtrList &outputs);
abstract::ExecutionFlowPtr Schedule(const CompileResultPtr &node_list);
SingleGraphSchedulerPtr scheduler_{nullptr};
std::shared_ptr<lite::InnerContext> context_{nullptr};
};
} // namespace infer
} // namespace mindspore
#endif

43
mindspore/lite/src/extendrt/graph_compiler/single_graph_scheduler.cc Normal file
View File

@ -0,0 +1,43 @@
/**
* Copyright 2023 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "src/extendrt/graph_compiler/single_graph_scheduler.h"
namespace mindspore {
namespace infer {
ExecutionFlowPtr SingleGraphScheduler::Schedule(const CompileResultPtr node_list) {
// no need to infer, infer is invoked in converter and delived to here
// select kernel
// fp16/fp32 weight, transpose weight
// append kernel with transpose
// optimize transpose
// infershape
return nullptr;
}
KernelType SingleGraphScheduler::SelectKernel(const CompileNode *compile_node) { return kKernelTypeUnknown; }
bool SingleGraphScheduler::HandleWeightForKernel(const CompileNode *compile_node, const KernelType kernel_type) {
return false;
}
bool SingleGraphScheduler::AppendKernelToPlan(kernel::KernelExec *kernel) { return false; }
bool SingleGraphScheduler::OptimizeTranspose(const std::vector<kernel::KernelExec *> &kernels) { return false; }
bool InferShape(const std::vector<kernel::KernelExec *> &kernels) { return false; }
} // namespace infer
} // namespace mindspore

53
mindspore/lite/src/extendrt/graph_compiler/single_graph_scheduler.h Normal file
View File

@ -0,0 +1,53 @@
/**
* Copyright 2023 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_SINGLE_GRAPH_SCHEDULER_H_
#define MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_SINGLE_GRAPH_SCHEDULER_H_
#include <memory>
#include <utility>
#include <vector>
#include "src/extendrt/graph_compiler/compile_result.h"
#include "src/extendrt/execution_flow.h"
#include "src/litert/inner_context.h"
namespace mindspore {
namespace infer {
enum KernelType {
kKernelTypeUnknown = 0,
kKernelTypeLite = 1,
kKernelTypeCloud = 2,
};
class SingleGraphScheduler {
public:
explicit SingleGraphScheduler(std::shared_ptr<lite::InnerContext> context) : context_(std::move(context)) {}
virtual ~SingleGraphScheduler() = default;
ExecutionFlowPtr Schedule(const CompileResultPtr node_list);
private:
KernelType SelectKernel(const CompileNode *compile_node);
bool HandleWeightForKernel(const CompileNode *compile_node, const KernelType kernel_type);
bool AppendKernelToPlan(kernel::KernelExec *kernel);
bool OptimizeTranspose(const std::vector<kernel::KernelExec *> &kernels);
bool InferShape(const std::vector<kernel::KernelExec *> &kernels);
private:
std::shared_ptr<lite::InnerContext> context_;
ExecutionFlowPtr execution_plan_{nullptr};
};
using SingleGraphSchedulerPtr = std::shared_ptr<SingleGraphScheduler>;
} // namespace infer
} // namespace mindspore
#endif

30
mindspore/lite/src/extendrt/graph_scheduler.cc
View File

@ -15,27 +15,9 @@
*/
#include "extendrt/graph_scheduler.h"
#include "extendrt/graph_compiler.h"
ExcutionPlan GraphCompiler::Schedule(const CompileResult &compile_result) {
ExcutionPlan execplan;
for (auto subgraph_ : compile_result.graphs_) {
if (KernelGraphUtils::Instance()->IsControlNode(subgraph_)) {
// generate control kernel
CNode &node = subgraph_->nodes[0];
KernelInfo kernelInfo;
kernelInfo.kernel_ = CreateControlKernel(node);
execplan.emplace_back(kernelInfo);
} else if (KernelGraphUtils::Instance()->IsSingleNode(subgraph_)) {
// generate kernel
KernelInfo kernelInfo;
CNode &node = subgraph_->nodes[0];
kernelInfo.kernel_ = CreateKernel(node);
execplan.emplace_back(kernelInfo);
} else if (KernelGraphUtils::Instance()->IsDAG(subgraph_)) {
// generate subgraph kernel
KernelInfo kernelInfo;
kernelInfo.kernel_ = CreateSubgraphKernel(subgraph_);
execplan.emplace_back(kernelInfo);
}
}
return execplan;
}
namespace mindspore {
namespace infer {
ExcutionPlan GraphScheduler::Schedule(const CompileResult &compile_result) { return {}; }
} // namespace infer
} // namespace mindspore

11
mindspore/lite/src/extendrt/graph_scheduler.h
View File

@ -25,10 +25,21 @@
#include "include/api/status.h"
#include "include/common/utils/utils.h"
#include "ir/func_graph.h"
#include "src/litert/kernel_exec.h"
namespace mindspore {
namespace infer {
struct ScheduleStrategy {};
struct GraphCompilerInfo;
struct Tensor {};
struct ExcutionPlan {
std::vector<kernel::KernelExec> kernels_;
std::vector<Tensor> tensors_;
std::vector<int64_t> inputs_;
std::vector<int64_t> outputs_;
};
class GraphScheduler : public std::enable_shared_from_this<GraphScheduler> {
public:
explicit GraphScheduler(const ScheduleStrategy &strategy);

29
mindspore/lite/src/extendrt/tensor.h Normal file
View File

@ -0,0 +1,29 @@
/**
* Copyright 2023 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_TENSOR_H_
#define MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_TENSOR_H_
#include "src/infer/tensor.h"
namespace mindspore {
namespace infer {
// implemented tensor for extendrt
using Tensor = abstract::Tensor;
} // namespace infer
} // namespace mindspore
#endif // MINDSPORE_LITE_EXTENDRT_GRAPH_COMPILER_TENSOR_H_

1
mindspore/lite/src/infer/execution_flow.h
View File

@ -105,6 +105,7 @@ class ExecutionFlow : public std::enable_shared_from_this<ExecutionFlow> {
/// \return Kernel pointer.
virtual Kernel *ConstructFusionKernel() = 0;
};
using ExecutionFlowPtr = std::shared_ptr<ExecutionFlow>;
} // namespace mindspore::infer::abstract
#endif // MINDSPORE_LITE_INFER_EXECUTION_FLOW_H_

1
mindspore/lite/src/infer/execution_plan.h
View File

@ -124,6 +124,7 @@ class ExecutionPlan : public std::enable_shared_from_this<ExecutionPlan> {
/// \return Kernel List
virtual std::vector<Kernel *> ToKernelList() = 0;
};
using ExecutionPlanPtr = std::shared_ptr<ExecutionPlan>;
} // namespace mindspore::infer::abstract
#endif // MINDSPORE_LITE_INFER_EXECUTION_PLAN_H_

6
mindspore/lite/src/infer/graph_compiler.h
View File

@ -19,6 +19,7 @@
#include <memory>
#include "infer/execution_plan.h"
#include "infer/execution_flow.h"
namespace mindspore::infer::abstract {
class GraphCompiler : public std::enable_shared_from_this<GraphCompiler> {
@ -30,7 +31,10 @@ class GraphCompiler : public std::enable_shared_from_this<GraphCompiler> {
/// \param[in] graph FuncGraph need to compile.
///
/// \return ExecutionPlan pointer.
virtual std::shared_ptr<ExecutionPlan> Compile(FuncGraphPtr graph) = 0;
virtual ExecutionPlanPtr Compile(FuncGraphPtr graph) = 0;
virtual ExecutionFlowPtr Compile(const GraphSegmentPtr &segment, const AnfNodePtrList &inputs,
const AnfNodePtrList &outputs) = 0;
};
} // namespace mindspore::infer::abstract

1
mindspore/lite/src/litert/kernel_exec.h
View File

@ -375,6 +375,7 @@ class KernelExec {
protected:
std::shared_ptr<Kernel> kernel_ = nullptr;
KernelKey desc_;
std::string op_type_{}; // merge op_type_ and desc_.type into type-wrapper while redesign infer-kernel
// tensor will free in ~lite_session()
std::vector<KernelExec *> in_kernels_;
std::vector<KernelExec *> out_kernels_;