Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!28615 MSLITE] fix bugs for diverse networks compatibility in tensorrt delegate for audio video edit 

Merge pull request !28615 from Liu_Xuu/trt_1229_av_edit
This commit is contained in:
i-robot 2022-01-06 11:57:09 +00:00 committed by Gitee
parent 64bbece74c 78a7171959
commit de34e90e4d
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
23 changed files with 273 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
mindspore/lite/src/delegate/tensorrt/op/concate_tensorrt.cc
View File

@ -46,13 +46,15 @@ int ConcateTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
return RET_ERROR;
}
if (tensorrt_in_tensors_.size() != in_tensors_.size()) {
MS_LOG(ERROR) << "concate_op in tensor is invalid";
MS_LOG(ERROR) << "concate_op in tensor is invalid, trt tensor has " << tensorrt_in_tensors_.size()
<< ", but origin ms tensor has " << in_tensors_.size();
return RET_ERROR;
}
nvinfer1::ITensor *trt_input_tensors[tensorrt_in_tensors_.size()];
int input_nbDims = tensorrt_in_tensors_[0].trt_tensor_->getDimensions().nbDims;
Format out_format = tensorrt_in_tensors_[0].format_;
bool same_format = tensorrt_in_tensors_[0].same_format_;
for (size_t i = 0; i < tensorrt_in_tensors_.size(); i++) {
if (tensorrt_in_tensors_[i].trt_tensor_->getDimensions().nbDims != input_nbDims) {
@ -78,6 +80,7 @@ int ConcateTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
return RET_ERROR;
}
trt_input_tensors[i] = transpose_layer->getOutput(0);
same_format = true;
MS_LOG(DEBUG) << "concate input " << GetTensorFormat(trt_input_tensors[i], Format::NHWC, true);
}
}
@ -92,10 +95,14 @@ int ConcateTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
if (axis == -1) {
axis = input_nbDims - 1;
}
if (trt_input_tensors[0]->getDimensions().nbDims == DIMENSION_4D && out_format == Format::NCHW) {
// when inputs all NCHW, change axis
axis = ConvertAxisFromNHWC2NCHW(axis);
MS_LOG(DEBUG) << "concate axis change to " << axis << " when using NCHW format.";
if (!same_format) {
if (trt_input_tensors[0]->getDimensions().nbDims == DIMENSION_4D && out_format == Format::NCHW) {
// when inputs all NCHW, change axis
axis = ConvertAxisFromNHWC2NCHW(axis);
MS_LOG(DEBUG) << "concate axis change to " << axis << " when using NCHW format.";
} else {
MS_LOG(WARNING) << "input tensor format needs check, convert concat axis failed for " << op_name_;
}
}
nvinfer1::IConcatenationLayer *concate_layer =
@ -110,8 +117,6 @@ int ConcateTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
}
concate_layer->setName(op_name_.c_str());
concate_layer->getOutput(0)->setName((op_name_ + "_output").c_str());
bool same_format = SameDims(trt_input_tensors[0]->getDimensions(), in_tensors_[0].Shape()) &&
SameDims(concate_layer->getOutput(0)->getDimensions(), out_tensors_[0].Shape());
this->AddInnerOutTensors(ITensorHelper{concate_layer->getOutput(0), out_format, same_format});
return RET_OK;
}

4
mindspore/lite/src/delegate/tensorrt/op/convolution_tensorrt.cc
View File

@ -123,9 +123,7 @@ int ConvolutionTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
activation_layer->setName((op_name_ + "_activation").c_str());
}
activation_layer->getOutput(0)->setName((op_name_ + "_output").c_str());
bool same_format = SameDims(activation_layer->getOutput(0)->getDimensions(), out_tensors_[0].Shape()) &&
SameDims(tensorrt_in_tensors_[0].trt_tensor_->getDimensions(), in_tensors_[0].Shape());
this->AddInnerOutTensors(ITensorHelper{activation_layer->getOutput(0), Format::NCHW, same_format});
this->AddInnerOutTensors(ITensorHelper{activation_layer->getOutput(0), Format::NCHW, false});
return RET_OK;
}

4
mindspore/lite/src/delegate/tensorrt/op/deconvolution_tensorrt.cc
View File

@ -120,9 +120,7 @@ int DeconvolutionTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
activation_layer->setName((op_name_ + "_activation").c_str());
}
activation_layer->getOutput(0)->setName((op_name_ + "_output").c_str());
bool same_format = SameDims(activation_layer->getOutput(0)->getDimensions(), out_tensors_[0].Shape()) &&
SameDims(tensorrt_in_tensors_[0].trt_tensor_->getDimensions(), in_tensors_[0].Shape());
this->AddInnerOutTensors(ITensorHelper{activation_layer->getOutput(0), Format::NCHW, same_format});
this->AddInnerOutTensors(ITensorHelper{activation_layer->getOutput(0), Format::NCHW, false});
return RET_OK;
}

85
mindspore/lite/src/delegate/tensorrt/op/elementwise_tensorrt.cc
View File

@ -82,37 +82,15 @@ int ElementWiseTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
MS_LOG(ERROR) << "network or input tensor size is invalid";
return RET_ERROR;
}
input_x_index_ = SameTensor(tensorrt_in_tensors_[0].trt_tensor_, &in_tensors_[0]) ? 0 : 1;
if (this->tensorrt_in_tensors_.size() != INPUT_SIZE2) {
int ret = AddConstTensor(network);
if (ret != RET_OK) {
return ret;
}
ITensorHelper x_input;
ITensorHelper y_input;
int ret = PreprocessInputTensors(network, &x_input, &y_input);
if (ret != RET_OK) {
MS_LOG(ERROR) << "PreprocessInputTensors failed.";
return RET_ERROR;
}
MS_LOG(DEBUG) << "before transpose " << GetTensorFormat(tensorrt_in_tensors_[input_x_index_]);
MS_LOG(DEBUG) << "before transpose " << GetTensorFormat(tensorrt_in_tensors_[1 - input_x_index_]);
if (tensorrt_in_tensors_[0].trt_tensor_->getDimensions().nbDims == DIMENSION_4D &&
tensorrt_in_tensors_[0].format_ != tensorrt_in_tensors_[1].format_) {
// when inputs format are different, change to NHWC
int transpose_input_tensor = tensorrt_in_tensors_[0].format_ == Format::NCHW ? 0 : 1;
nvinfer1::IShuffleLayer *transpose_layer =
NCHW2NHWC(network, *tensorrt_in_tensors_[transpose_input_tensor].trt_tensor_);
if (transpose_layer == nullptr) {
MS_LOG(ERROR) << "op action convert failed";
return RET_ERROR;
}
transpose_layer->setName((op_name_ + "_input_transpose2NHWC").c_str());
tensorrt_in_tensors_[transpose_input_tensor].trt_tensor_ = transpose_layer->getOutput(0);
tensorrt_in_tensors_[transpose_input_tensor].format_ = Format::NHWC;
}
MS_LOG(DEBUG) << "after transpose " << GetTensorFormat(tensorrt_in_tensors_[input_x_index_]);
MS_LOG(DEBUG) << "after transpose " << GetTensorFormat(tensorrt_in_tensors_[1 - input_x_index_]);
nvinfer1::IElementWiseLayer *cal_layer =
network->addElementWise(*tensorrt_in_tensors_[input_x_index_].trt_tensor_,
*tensorrt_in_tensors_[1 - input_x_index_].trt_tensor_, element_wise_op_);
network->addElementWise(*x_input.trt_tensor_, *y_input.trt_tensor_, element_wise_op_);
if (cal_layer == nullptr) {
MS_LOG(ERROR) << "addElementWise failed for TensorRT.";
@ -143,12 +121,57 @@ int ElementWiseTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
}
}
op_out_tensor->setName((op_name_ + "_output").c_str());
this->AddInnerOutTensors(
ITensorHelper{op_out_tensor, tensorrt_in_tensors_[1].format_, tensorrt_in_tensors_[1].same_format_});
this->AddInnerOutTensors(ITensorHelper{op_out_tensor, x_input.format_, x_input.same_format_});
MS_LOG(DEBUG) << "output " << GetTensorFormat(tensorrt_out_tensors_[0]);
return RET_OK;
}
int ElementWiseTensorRT::PreprocessInputTensors(nvinfer1::INetworkDefinition *network, ITensorHelper *x_input,
ITensorHelper *y_input) {
int input_x_index = SameTensor(tensorrt_in_tensors_[0].trt_tensor_, &in_tensors_[0]) ? 0 : 1;
if (this->tensorrt_in_tensors_.size() != INPUT_SIZE2) {
int ret = AddConstTensor(network);
if (ret != RET_OK) {
return ret;
}
}
*x_input = tensorrt_in_tensors_[input_x_index];
*y_input = tensorrt_in_tensors_[1 - input_x_index];
MS_LOG(DEBUG) << "before transpose " << GetTensorFormat(*x_input);
MS_LOG(DEBUG) << "before transpose " << GetTensorFormat(*y_input);
if (x_input->trt_tensor_->getDimensions().nbDims == DIMENSION_4D && x_input->format_ != y_input->format_) {
// when inputs format are different, change to NHWC
auto need_trans = x_input->format_ == Format::NCHW ? x_input : y_input;
nvinfer1::IShuffleLayer *transpose_layer = NCHW2NHWC(network, *need_trans->trt_tensor_);
if (transpose_layer == nullptr) {
MS_LOG(ERROR) << "op action convert failed";
return RET_ERROR;
}
transpose_layer->setName((op_name_ + "_input_transpose2NHWC").c_str());
need_trans->trt_tensor_ = transpose_layer->getOutput(0);
need_trans->format_ = Format::NHWC;
need_trans->same_format_ = true;
}
MS_LOG(DEBUG) << "after transpose " << GetTensorFormat(*x_input);
MS_LOG(DEBUG) << "after transpose " << GetTensorFormat(*y_input);
if (GetDimsVolume(x_input->trt_tensor_->getDimensions()) == GetDimsVolume(y_input->trt_tensor_->getDimensions()) &&
x_input->trt_tensor_->getDimensions().nbDims != y_input->trt_tensor_->getDimensions().nbDims) {
bool x_large = x_input->trt_tensor_->getDimensions().nbDims > y_input->trt_tensor_->getDimensions().nbDims;
auto input_tensor = x_large ? y_input : x_input;
auto output_dim = x_large ? x_input->trt_tensor_->getDimensions() : y_input->trt_tensor_->getDimensions();
auto reshape_layer = network->addShuffle(*input_tensor->trt_tensor_);
if (reshape_layer == nullptr) {
MS_LOG(ERROR) << "add reshape failed for " << op_name_;
return RET_ERROR;
}
reshape_layer->setReshapeDimensions(output_dim);
input_tensor->trt_tensor_ = reshape_layer->getOutput(0);
}
return RET_OK;
}
nvinfer1::ITensor *ElementWiseTensorRT::AddActivation(nvinfer1::INetworkDefinition *network,
nvinfer1::ITensor *in_tensor) {
schema::ActivationType activation = schema::ActivationType::ActivationType_NO_ACTIVATION;

5
mindspore/lite/src/delegate/tensorrt/op/elementwise_tensorrt.h
View File

@ -41,10 +41,9 @@ class ElementWiseTensorRT : public TensorRTOp {
bool SameTensor(nvinfer1::ITensor *trt_tensor, mindspore::MSTensor *ms_tensor);
nvinfer1::ElementWiseOperation element_wise_op_;
int PreprocessInputTensors(nvinfer1::INetworkDefinition *network, ITensorHelper *x_input, ITensorHelper *y_input);
// index of first input MSTensor in the trt input tensor vector
size_t input_x_index_ = 0;
nvinfer1::ElementWiseOperation element_wise_op_;
};
} // namespace mindspore::lite
#endif // MINDSPORE_LITE_SRC_DELEGATE_TENSORRT_OP_ELEMENTWISE_TENSORRT_H_

21
mindspore/lite/src/delegate/tensorrt/op/gather_tensorrt.cc
View File

@ -60,16 +60,25 @@ int GatherTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
MS_LOG(ERROR) << "add const input tensor failed for " << op_name_;
return RET_ERROR;
}
tensorrt_in_tensors_.push_back(ITensorHelper{const_input, Format::NHWC, true});
tensorrt_in_tensors_.push_back(ITensorHelper{const_input});
}
int indices_tensor_index = tensorrt_in_tensors_[0].trt_tensor_->getType() == nvinfer1::DataType::kINT32 ? 0 : 1;
nvinfer1::ITensor *gather_input = PreprocessInputs2SameDim(network, tensorrt_in_tensors_[1 - indices_tensor_index]);
nvinfer1::ITensor *indices_tensor = PreprocessInputs2SameDim(network, tensorrt_in_tensors_[indices_tensor_index]);
ITensorHelper gather_input;
int ret = PreprocessInputs2SameDim(network, tensorrt_in_tensors_[1 - indices_tensor_index], &gather_input);
if (ret != RET_OK || gather_input.trt_tensor_ == nullptr) {
MS_LOG(ERROR) << "PreprocessInputs2SameDim gather failed for " << op_name_;
return RET_ERROR;
}
ITensorHelper indices_tensor;
ret = PreprocessInputs2SameDim(network, tensorrt_in_tensors_[indices_tensor_index], &indices_tensor);
if (ret != RET_OK || indices_tensor.trt_tensor_ == nullptr) {
MS_LOG(ERROR) << "PreprocessInputs2SameDim indices failed for " << op_name_;
return RET_ERROR;
}
nvinfer1::IGatherLayer *gather_layer =
network->addGather(*gather_input, *indices_tensor /* indices */, axis_ /* axis */);
network->addGather(*gather_input.trt_tensor_, *indices_tensor.trt_tensor_ /* indices */, axis_ /* axis */);
if (gather_layer == nullptr) {
MS_LOG(ERROR) << "addGather failed for TensorRT.";
return RET_ERROR;
@ -91,7 +100,7 @@ int GatherTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
op_output = squeeze->getOutput(0);
}
op_output->setName((op_name_ + "_output").c_str());
this->AddInnerOutTensors(ITensorHelper{op_output, Format::NHWC, true});
this->AddInnerOutTensors(ITensorHelper{op_output, gather_input.format_, gather_input.same_format_});
return RET_OK;
}
} // namespace mindspore::lite

100
mindspore/lite/src/delegate/tensorrt/op/matmul_tensorrt.cc
View File

@ -52,37 +52,20 @@ int MatMulTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
transpose_b_ = nvinfer1::MatrixOperation::kTRANSPOSE;
}
nvinfer1::ITensor *matmul_input = tensorrt_in_tensors_[0].trt_tensor_;
if (tensorrt_in_tensors_[0].trt_tensor_->getDimensions().nbDims == DIMENSION_4D &&
tensorrt_in_tensors_[0].format_ == Format::NCHW) {
// transpose: NCHW->NHWC
nvinfer1::IShuffleLayer *transpose_layer_in = NCHW2NHWC(network, *tensorrt_in_tensors_[0].trt_tensor_);
if (transpose_layer_in == nullptr) {
MS_LOG(ERROR) << "op action convert failed";
return RET_ERROR;
}
transpose_layer_in->setName((op_name_ + "_transpose2NHWC").c_str());
matmul_input = transpose_layer_in->getOutput(0);
}
ITensorHelper matmul_a;
ITensorHelper matmul_b;
nvinfer1::ITensor *weight = nullptr;
if (in_tensors_[1].Shape().size() < static_cast<size_t>(matmul_input->getDimensions().nbDims)) {
weight = ConvertTensorWithExpandDims(network, in_tensors_[1], in_tensors_[0].Shape().size(), op_name_);
} else if (in_tensors_[1].Shape().size() == static_cast<size_t>(matmul_input->getDimensions().nbDims)) {
weight = ConvertConstantTensor(network, in_tensors_[1], op_name_);
} else {
MS_LOG(ERROR) << "input tensor shape is invalid for " << op_name_;
return RET_ERROR;
}
if (weight == nullptr) {
MS_LOG(ERROR) << "create constant weight tensor failed for " << op_name_;
int ret = PreprocessInputs(network, &matmul_a, &matmul_b);
if (ret != RET_OK || matmul_a.trt_tensor_ == nullptr || matmul_b.trt_tensor_ == nullptr) {
MS_LOG(ERROR) << "PreprocessInputs matmul failed for " << op_name_;
return RET_ERROR;
}
MS_LOG(DEBUG) << "matmul input a " << GetTensorFormat(matmul_input);
MS_LOG(DEBUG) << "matmul input b " << GetTensorFormat(weight);
MS_LOG(DEBUG) << "matmul input a " << GetTensorFormat(matmul_a);
MS_LOG(DEBUG) << "matmul input b " << GetTensorFormat(matmul_b);
auto matmul_layer = network->addMatrixMultiply(*matmul_input, transpose_a_, *weight, transpose_b_);
auto matmul_layer =
network->addMatrixMultiply(*matmul_a.trt_tensor_, transpose_a_, *matmul_b.trt_tensor_, transpose_b_);
matmul_layer->setName(op_name_.c_str());
nvinfer1::ITensor *out_tensor = matmul_layer->getOutput(0);
@ -124,7 +107,70 @@ int MatMulTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
out_tensor->setName((op_name_ + "_output").c_str());
MS_LOG(DEBUG) << "output " << GetTensorFormat(out_tensor);
this->AddInnerOutTensors(ITensorHelper{out_tensor});
this->AddInnerOutTensors(ITensorHelper{out_tensor, out_format_});
return RET_OK;
}
int MatMulTensorRT::PreprocessInputs(nvinfer1::INetworkDefinition *network, ITensorHelper *matmul_a,
ITensorHelper *matmul_b) {
int ret;
if (tensorrt_in_tensors_.size() == INPUT_SIZE2) {
int a_index =
GetDimsVolume(tensorrt_in_tensors_[0].trt_tensor_->getDimensions()) == GetDimsVolume(in_tensors_[0].Shape()) ? 0
: 1;
ret = PreprocessInputs2SameDim(network, tensorrt_in_tensors_[a_index], matmul_a);
if (ret != RET_OK || matmul_a->trt_tensor_ == nullptr) {
MS_LOG(ERROR) << "PreprocessInputs2SameDim of matmul input a failed for " << op_name_;
return RET_ERROR;
}
ret = PreprocessInputs2SameDim(network, tensorrt_in_tensors_[1 - a_index], matmul_b);
if (ret != RET_OK || matmul_b->trt_tensor_ == nullptr) {
MS_LOG(ERROR) << "PreprocessInputs2SameDim of matmul input b failed for " << op_name_;
return RET_ERROR;
}
out_format_ = matmul_a->format_;
if (matmul_a->format_ != matmul_b->format_) {
MS_LOG(WARNING) << "matmul input tensor has different format " << op_name_;
}
} else if (tensorrt_in_tensors_.size() == 1) {
nvinfer1::ITensor *weight = nullptr;
int weight_index = in_tensors_[1].Data() != nullptr ? 1 : 0;
if (in_tensors_[weight_index].Shape().size() <
static_cast<size_t>(tensorrt_in_tensors_[0].trt_tensor_->getDimensions().nbDims)) {
weight = ConvertTensorWithExpandDims(network, in_tensors_[weight_index],
in_tensors_[1 - weight_index].Shape().size(), op_name_);
} else if (in_tensors_[weight_index].Shape().size() ==
static_cast<size_t>(tensorrt_in_tensors_[0].trt_tensor_->getDimensions().nbDims)) {
weight = ConvertConstantTensor(network, in_tensors_[weight_index], op_name_);
} else {
MS_LOG(ERROR) << "input tensor shape is invalid for " << op_name_;
return RET_ERROR;
}
if (weight == nullptr) {
MS_LOG(ERROR) << "create constant weight tensor failed for " << op_name_;
return RET_ERROR;
}
if (weight_index == 1) {
matmul_b->trt_tensor_ = weight;
ret = PreprocessInputs2SameDim(network, tensorrt_in_tensors_[0], matmul_a);
if (ret != RET_OK || matmul_a->trt_tensor_ == nullptr) {
MS_LOG(ERROR) << "PreprocessInputs2SameDim of matmul input a failed for " << op_name_;
return RET_ERROR;
}
out_format_ = matmul_a->format_;
} else {
matmul_a->trt_tensor_ = weight;
ret = PreprocessInputs2SameDim(network, tensorrt_in_tensors_[0], matmul_b);
if (ret != RET_OK || matmul_b->trt_tensor_ == nullptr) {
MS_LOG(ERROR) << "PreprocessInputs2SameDim of matmul input b failed for " << op_name_;
return RET_ERROR;
}
out_format_ = matmul_b->format_;
}
} else {
MS_LOG(ERROR) << op_name_ << " tensorrt in tensor size is invalid " << tensorrt_in_tensors_.size();
return RET_ERROR;
}
return RET_OK;
}
} // namespace mindspore::lite

2
mindspore/lite/src/delegate/tensorrt/op/matmul_tensorrt.h
View File

@ -36,8 +36,10 @@ class MatMulTensorRT : public TensorRTOp {
int AddInnerOp(nvinfer1::INetworkDefinition *network) override;
private:
int PreprocessInputs(nvinfer1::INetworkDefinition *network, ITensorHelper *matmul_a, ITensorHelper *matmul_b);
nvinfer1::MatrixOperation transpose_a_ = nvinfer1::MatrixOperation::kNONE;
nvinfer1::MatrixOperation transpose_b_ = nvinfer1::MatrixOperation::kNONE;
Format out_format_;
};
} // namespace mindspore::lite
#endif // MINDSPORE_LITE_SRC_RUNTIME_DELEGATE_TENSORRT_OP_MATMUL_TENSORRT_H_

5
mindspore/lite/src/delegate/tensorrt/op/pad_tensorrt.cc
View File

@ -46,7 +46,7 @@ int PadTensorRT::IsSupport(const mindspore::schema::Primitive *primitive,
}
schema::PaddingMode padding_mode = pad_primitive->padding_mode();
if (padding_mode != schema::PaddingMode::PaddingMode_CONSTANT) {
MS_LOG(ERROR) << "Unsupported padding mode: " << pad_primitive << ", for op: " << op_name_;
MS_LOG(ERROR) << "Unsupported padding mode: " << schema::PaddingMode(padding_mode) << ", for op: " << op_name_;
return RET_ERROR;
}
if (in_tensors[0].format() != Format::NHWC && in_tensors[0].format() != Format::NCHW) {
@ -67,6 +67,8 @@ int PadTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
}
nvinfer1::ITensor *pad_input = tensorrt_in_tensors_[0].trt_tensor_;
MS_LOG(DEBUG) << "before transpose "
<< GetTensorFormat(pad_input, tensorrt_in_tensors_[0].format_, tensorrt_in_tensors_[0].same_format_);
if (tensorrt_in_tensors_[0].trt_tensor_->getDimensions().nbDims == DIMENSION_4D &&
tensorrt_in_tensors_[0].format_ == Format::NHWC) {
// transpose: NHWC->NCHW
@ -129,6 +131,7 @@ int PadTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
bool same_format = SameDims(padding_layer->getOutput(0)->getDimensions(), out_tensors_[0].Shape()) &&
SameDims(tensorrt_in_tensors_[0].trt_tensor_->getDimensions(), in_tensors_[0].Shape());
this->AddInnerOutTensors(ITensorHelper{padding_layer->getOutput(0), Format::NCHW, same_format});
MS_LOG(DEBUG) << "after transpose " << GetTensorFormat(tensorrt_out_tensors_[0]);
return RET_OK;
}
} // namespace mindspore::lite

4
mindspore/lite/src/delegate/tensorrt/op/pool_tensorrt.cc
View File

@ -95,9 +95,7 @@ int PoolTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
}
nvinfer1::ITensor *out_trt_tensor = activation_layer->getOutput(0);
out_trt_tensor->setName((op_name_ + "_output").c_str());
bool same_format = SameDims(out_trt_tensor->getDimensions(), out_tensors_[0].Shape()) &&
SameDims(tensorrt_in_tensors_[0].trt_tensor_->getDimensions(), in_tensors_[0].Shape());
this->AddInnerOutTensors(ITensorHelper{out_trt_tensor, Format::NCHW, same_format});
this->AddInnerOutTensors(ITensorHelper{out_trt_tensor, Format::NCHW, false});
MS_LOG(DEBUG) << "output " << GetTensorFormat(tensorrt_out_tensors_[0]);
return RET_OK;
}

41
mindspore/lite/src/delegate/tensorrt/op/scale_tensorrt.cc
View File

@ -56,12 +56,13 @@ int ScaleTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
schema::ActivationType activation_type = scale_op->activation_type();
// mode of scale
int64_t axis = scale_op->axis();
if (axis == -1) {
axis = static_cast<int64_t>(in_tensors_[0].Shape().size() - 1);
axis_ = scale_op->axis();
if (axis_ == -1) {
axis_ = static_cast<int64_t>(in_tensors_[0].Shape().size() - 1);
}
mode_ = GetScaleMode(axis);
mode_ = GetScaleMode(axis_);
out_format_ = tensorrt_in_tensors_[0].format_;
out_same_format_ = tensorrt_in_tensors_[0].same_format_;
MS_LOG(DEBUG) << "before transpose " << GetTensorFormat(tensorrt_in_tensors_[0]);
nvinfer1::ITensor *scale_in_tensor = PreProcessInputTensor(network);
@ -70,8 +71,7 @@ int ScaleTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
return RET_ERROR;
}
MS_LOG(DEBUG) << "after transpose "
<< GetTensorFormat(scale_in_tensor, out_format_, tensorrt_in_tensors_[0].same_format_);
MS_LOG(DEBUG) << "after transpose " << GetTensorFormat(scale_in_tensor, out_format_, out_same_format_);
bool nd = false;
// (input * scale + shift) ^ power
@ -101,7 +101,7 @@ int ScaleTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
if (nd) {
MS_LOG(WARNING) << "multi dims ScaleMode enter";
cal_layer = network->addScaleNd(*scale_in_tensor, mode_, shift, scale, power, axis);
cal_layer = network->addScaleNd(*scale_in_tensor, mode_, shift, scale, power, axis_);
} else {
cal_layer = network->addScale(*scale_in_tensor, mode_, shift, scale, power);
}
@ -131,7 +131,7 @@ int ScaleTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
op_out_tensor = AddSqueezeOp(activation_tensor, network);
}
op_out_tensor->setName((op_name_ + "_output").c_str());
this->AddInnerOutTensors(ITensorHelper{op_out_tensor, out_format_, tensorrt_in_tensors_[0].same_format_});
this->AddInnerOutTensors(ITensorHelper{op_out_tensor, out_format_, out_same_format_});
MS_LOG(DEBUG) << "output " << GetTensorFormat(tensorrt_out_tensors_[0]);
return RET_OK;
}
@ -146,17 +146,24 @@ nvinfer1::ITensor *ScaleTensorRT::PreProcessInputTensor(nvinfer1::INetworkDefini
return nullptr;
}
} else if (tensorrt_in_tensors_[0].trt_tensor_->getDimensions().nbDims == DIMENSION_4D &&
mode_ == nvinfer1::ScaleMode::kCHANNEL && tensorrt_in_tensors_[0].format_ == Format::NHWC) {
mode_ == nvinfer1::ScaleMode::kCHANNEL) {
// per channel input format should be nchw, otherwise should be same with scale nhwc
// transpose: NHWC->NCHW
nvinfer1::IShuffleLayer *transpose_layer_in = NHWC2NCHW(network, *tensorrt_in_tensors_[0].trt_tensor_);
if (transpose_layer_in == nullptr) {
MS_LOG(ERROR) << "op action convert failed";
return nullptr;
if ((tensorrt_in_tensors_[0].format_ == Format::NHWC && axis_ == kNHWC_C) ||
(tensorrt_in_tensors_[0].same_format_ == true && axis_ == kNHWC_C)) {
nvinfer1::IShuffleLayer *transpose_layer_in = NHWC2NCHW(network, *tensorrt_in_tensors_[0].trt_tensor_);
if (transpose_layer_in == nullptr) {
MS_LOG(ERROR) << "op action convert failed";
return nullptr;
}
transpose_layer_in->setName((op_name_ + "_transpose2NCHW").c_str());
scale_in_tensor = transpose_layer_in->getOutput(0);
out_format_ = Format::NCHW;
out_same_format_ = !out_same_format_;
} else if (out_format_ != Format::NCHW && axis_ != kNCHW_C) {
MS_LOG(WARNING) << op_name_ << " out format (NHWC:1, NCHW:0) infer as " << out_format_ << ", and axis is "
<< axis_;
}
transpose_layer_in->setName((op_name_ + "_transpose2NCHW").c_str());
scale_in_tensor = transpose_layer_in->getOutput(0);
out_format_ = Format::NCHW;
} else if (tensorrt_in_tensors_[0].trt_tensor_->getDimensions().nbDims == DIMENSION_4D &&
tensorrt_in_tensors_[0].format_ == Format::NCHW && mode_ == nvinfer1::ScaleMode::kELEMENTWISE) {
// transpose: NCHW->NHWC
@ -168,6 +175,7 @@ nvinfer1::ITensor *ScaleTensorRT::PreProcessInputTensor(nvinfer1::INetworkDefini
transpose_layer_in->setName((op_name_ + "_transpose2NHWC").c_str());
scale_in_tensor = transpose_layer_in->getOutput(0);
out_format_ = Format::NHWC;
out_same_format_ = true;
}
return scale_in_tensor;
}
@ -177,7 +185,6 @@ nvinfer1::ScaleMode ScaleTensorRT::GetScaleMode(int64_t axis) {
auto input_data_shape = in_tensors_[0].Shape();
auto input_weight_shape = in_tensors_[1].Shape();
int total = std::accumulate(input_data_shape.begin(), input_data_shape.end(), 1, std::multiplies<int>());
MS_LOG(DEBUG) << "input tensor element cnt: " << total;
if (input_weight_shape.size() == 0 || (input_weight_shape.size() == 1 && input_weight_shape[0] == 1)) {
mode = nvinfer1::ScaleMode::kUNIFORM;
} else if ((axis < static_cast<int64_t>(input_data_shape.size()) && input_weight_shape.size() == 1 &&

4
mindspore/lite/src/delegate/tensorrt/op/scale_tensorrt.h
View File

@ -46,7 +46,11 @@ class ScaleTensorRT : public TensorRTOp {
Format out_format_;
bool out_same_format_;
nvinfer1::ScaleMode mode_;
int64_t axis_;
};
} // namespace mindspore::lite
#endif // MINDSPORE_LITE_SRC_DELEGATE_TENSORRT_OP_SCALE_TENSORRT_H_

46
mindspore/lite/src/delegate/tensorrt/op/shuffle_tensorrt.cc
View File

@ -61,6 +61,9 @@ int ShuffleTensorRT::IsSupport(const schema::Primitive *primitive, const std::ve
return RET_ERROR;
}
dynamic_shape_params_.support_hw_dynamic_ = false;
if (in_tensors[0].Shape()[0] != out_tensors[0].Shape()[0]) {
dynamic_shape_params_.support_dynamic_ = false;
}
break;
}
case schema::PrimitiveType_Transpose:
@ -218,30 +221,20 @@ int ShuffleTensorRT::AddUnsqueezeOp(nvinfer1::IShuffleLayer *shuffle_layer) {
MS_LOG(ERROR) << "AddUnsqueezeOp convert failed";
return RET_ERROR;
}
if (in_tensors_.size() != 1) {
MS_LOG(WARNING) << "AddUnsqueezeOp size of in tensort needs check: " << in_tensors_.size();
}
// axis
auto unsqueeze_shape = shuffler_input_->getDimensions();
std::vector<int64_t> new_shape(unsqueeze_shape.d, unsqueeze_shape.d + unsqueeze_shape.nbDims);
param_axis_ = unsqueeze_op->axis();
if (param_axis_ == nullptr) {
MS_LOG(ERROR) << "axis is invalid for " << op_name_;
return RET_ERROR;
}
if (param_axis_->size() != 1) {
MS_LOG(WARNING) << op_name_ << " has unsqueeze axis size: " << param_axis_->size();
}
nvinfer1::ITensor *expand_input = shuffler_input_;
for (size_t i = 0; i < param_axis_->size(); i++) {
new_shape.insert(new_shape.begin() + param_axis_->Get(i), 1);
expand_input = ExpandDim(shuffle_layer, expand_input, param_axis_->Get(i));
}
nvinfer1::Dims unsqueeze_dims = lite::ConvertCudaDims(new_shape);
if (unsqueeze_dims.nbDims == -1) {
MS_LOG(ERROR) << "ConvertCudaDims failed for " << op_name_;
return RET_ERROR;
}
shuffle_layer->setReshapeDimensions(unsqueeze_dims);
shuffler_output_ = shuffle_layer->getOutput(0);
shuffler_output_ = expand_input;
return shuffler_output_ == nullptr ? RET_ERROR : RET_OK;
}
@ -321,7 +314,13 @@ int ShuffleTensorRT::AddExpandDimsOp(nvinfer1::IShuffleLayer *shuffle_layer) {
}
auto axis_data = static_cast<const int *>(in_tensors_[1].Data().get());
int axis = axis_data[0];
auto input_dims = shuffler_input_->getDimensions();
shuffler_output_ = ExpandDim(shuffle_layer, shuffler_input_, axis);
return shuffler_output_ == nullptr ? RET_ERROR : RET_OK;
}
nvinfer1::ITensor *ShuffleTensorRT::ExpandDim(nvinfer1::IShuffleLayer *shuffle_layer, nvinfer1::ITensor *input_tensor,
int axis) {
auto input_dims = input_tensor->getDimensions();
// if expand dim not at last dim and shape is dynamic, change to expanddim at last dim and transpose
bool special_expand = false;
for (int i = 0; i < input_dims.nbDims; i++) {
@ -338,7 +337,7 @@ int ShuffleTensorRT::AddExpandDimsOp(nvinfer1::IShuffleLayer *shuffle_layer) {
nvinfer1::Dims new_dims = ConvertCudaDims(new_shape);
if (new_dims.nbDims == -1) {
MS_LOG(ERROR) << "ConvertCudaDims failed for " << op_name_;
return RET_ERROR;
return nullptr;
}
shuffle_layer->setReshapeDimensions(new_dims);
// transpose
@ -355,10 +354,10 @@ int ShuffleTensorRT::AddExpandDimsOp(nvinfer1::IShuffleLayer *shuffle_layer) {
nvinfer1::IShuffleLayer *trans_layer = network_->addShuffle(*shuffle_layer->getOutput(0));
if (trans_layer == nullptr) {
MS_LOG(ERROR) << "add transpose layer failed for special expand dims op " << op_name_;
return RET_ERROR;
return nullptr;
}
trans_layer->setFirstTranspose(perm);
shuffler_output_ = trans_layer->getOutput(0);
return trans_layer->getOutput(0);
} else {
std::vector<int64_t> new_shape;
for (int i = 0; i < input_dims.nbDims; i++) {
@ -367,18 +366,17 @@ int ShuffleTensorRT::AddExpandDimsOp(nvinfer1::IShuffleLayer *shuffle_layer) {
}
new_shape.push_back(input_dims.d[i] == -1 ? 0 : input_dims.d[i]);
}
if (axis == -1) {
if (axis == -1 || axis == input_dims.nbDims) {
new_shape.push_back(1);
}
nvinfer1::Dims new_dims = ConvertCudaDims(new_shape);
if (new_dims.nbDims == -1) {
MS_LOG(ERROR) << "ConvertCudaDims failed for " << op_name_;
return RET_ERROR;
return nullptr;
}
shuffle_layer->setReshapeDimensions(new_dims);
shuffler_output_ = shuffle_layer->getOutput(0);
return shuffle_layer->getOutput(0);
}
return RET_OK;
}
nvinfer1::Dims ShuffleTensorRT::InferReshapeDims(const nvinfer1::Dims &input_dims,

1
mindspore/lite/src/delegate/tensorrt/op/shuffle_tensorrt.h
View File

@ -42,6 +42,7 @@ class ShuffleTensorRT : public TensorRTOp {
int AddReshapeOp(nvinfer1::IShuffleLayer *shuffle_layer);
int AddFlattenOp(nvinfer1::IShuffleLayer *shuffle_layer);
int AddExpandDimsOp(nvinfer1::IShuffleLayer *shuffle_layer);
nvinfer1::ITensor *ExpandDim(nvinfer1::IShuffleLayer *shuffle_layer, nvinfer1::ITensor *input_tensor, int axis);
nvinfer1::Dims InferReshapeDims(const nvinfer1::Dims &input_dims, const std::vector<int64_t> &ms_input_shape,
const std::vector<int64_t> &ms_output_shape);

12
mindspore/lite/src/delegate/tensorrt/op/slice_tensorrt.cc
View File

@ -52,18 +52,20 @@ int SliceTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
strides_index_ = in_tensors_.size() - 1;
axis_index_ = in_tensors_.size() == HAS_AXIS ? AXIS_INDEX : -1;
nvinfer1::ITensor *slice_input = PreprocessInputs2SameDim(network, tensorrt_in_tensors_[0]);
if (slice_input == nullptr) {
ITensorHelper slice_input;
int ret = PreprocessInputs2SameDim(network, tensorrt_in_tensors_[0], &slice_input);
if (ret != RET_OK || slice_input.trt_tensor_ == nullptr) {
MS_LOG(ERROR) << "PreprocessInputs2SameDim input tensor failed for " << op_name_;
return RET_ERROR;
}
int ret = ConvertParamsDims();
ret = ConvertParamsDims();
if (ret != RET_OK) {
MS_LOG(ERROR) << "ConvertParamsDims failed for " << op_name_;
return ret;
}
nvinfer1::ISliceLayer *slice_layer = network->addSlice(*slice_input, start_dims_, size_dims_, stride_dims_);
nvinfer1::ISliceLayer *slice_layer =
network->addSlice(*slice_input.trt_tensor_, start_dims_, size_dims_, stride_dims_);
if (slice_layer == nullptr) {
MS_LOG(ERROR) << "add Slice op failed for TensorRT: " << op_name_;
return RET_ERROR;
@ -75,7 +77,7 @@ int SliceTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
return RET_ERROR;
}
out_tensor->setName((op_name_ + "_output").c_str());
this->AddInnerOutTensors(ITensorHelper{out_tensor, Format::NHWC, true});
this->AddInnerOutTensors(ITensorHelper{out_tensor, slice_input.format_, slice_input.same_format_});
return RET_OK;
}

6
mindspore/lite/src/delegate/tensorrt/op/unary_tensorrt.h
View File

@ -38,6 +38,12 @@ class UnaryTensorRT : public TensorRTOp {
std::map<schema::PrimitiveType, nvinfer1::UnaryOperation> unary_ops_ = {
{schema::PrimitiveType_Sqrt, nvinfer1::UnaryOperation::kSQRT},
{schema::PrimitiveType_Abs, nvinfer1::UnaryOperation::kABS},
{schema::PrimitiveType_Neg, nvinfer1::UnaryOperation::kNEG},
{schema::PrimitiveType_Log, nvinfer1::UnaryOperation::kLOG},
{schema::PrimitiveType_Sin, nvinfer1::UnaryOperation::kSIN},
{schema::PrimitiveType_Cos, nvinfer1::UnaryOperation::kCOS},
{schema::PrimitiveType_Ceil, nvinfer1::UnaryOperation::kCEIL},
{schema::PrimitiveType_Floor, nvinfer1::UnaryOperation::kFLOOR},
};
nvinfer1::UnaryOperation unary_op_;
};

6
mindspore/lite/src/delegate/tensorrt/tensorrt_delegate.cc
View File

@ -119,6 +119,12 @@ Status TensorRTDelegate::Init() {
{schema::PrimitiveType_ArgMaxFusion, GetTensorRTOp<TopKTensorRT>},
{schema::PrimitiveType_Sqrt, GetTensorRTOp<UnaryTensorRT>},
{schema::PrimitiveType_Abs, GetTensorRTOp<UnaryTensorRT>},
{schema::PrimitiveType_Neg, GetTensorRTOp<UnaryTensorRT>},
{schema::PrimitiveType_Log, GetTensorRTOp<UnaryTensorRT>},
{schema::PrimitiveType_Sin, GetTensorRTOp<UnaryTensorRT>},
{schema::PrimitiveType_Cos, GetTensorRTOp<UnaryTensorRT>},
{schema::PrimitiveType_Ceil, GetTensorRTOp<UnaryTensorRT>},
{schema::PrimitiveType_Floor, GetTensorRTOp<UnaryTensorRT>},
};
int ret = lite::SetCudaDevice(device_info_);
if (ret != RET_OK) {

3
mindspore/lite/src/delegate/tensorrt/tensorrt_serializer.h
View File

@ -28,7 +28,8 @@ using mindspore::lite::RET_OK;
namespace mindspore::lite {
class TensorRTSerializer {
public:
explicit TensorRTSerializer(std::string serialize_file_path) : serialize_file_path_(std::move(serialize_file_path)) {}
explicit TensorRTSerializer(const std::string &serialize_file_path)
: serialize_file_path_(std::move(serialize_file_path)) {}
~TensorRTSerializer() = default;

33
mindspore/lite/src/delegate/tensorrt/tensorrt_subgraph.cc
View File

@ -60,6 +60,9 @@ int TensorRTSubGraph::Init(cudaStream_t stream) {
MS_LOG(ERROR) << "createOptimizationProfile failed.";
return RET_ERROR;
}
if (SetDeviceConfig(stream) != RET_OK) {
MS_LOG(WARNING) << "set tensorrt config failed.";
}
serializer_ = std::make_shared<TensorRTSerializer>(serialize_file_path_);
if (serializer_ == nullptr) {
MS_LOG(ERROR) << "create Serializer failed.";
@ -81,9 +84,6 @@ int TensorRTSubGraph::Init(cudaStream_t stream) {
input_hw_index_ = -1;
}
}
if (SetDeviceConfig(stream) != RET_OK) {
MS_LOG(WARNING) << "set tensorrt config failed.";
}
return RET_OK;
}
@ -129,33 +129,6 @@ int TensorRTSubGraph::SetDeviceConfig(cudaStream_t stream) {
return RET_OK;
}
bool TensorRTSubGraph::SupportFP16() {
int deviceCnt = 0;
cudaError ret = cudaGetDeviceCount(&deviceCnt);
if (ret != cudaSuccess) {
MS_LOG(ERROR) << "cudaGetDeviceCount failed.";
return false;
}
std::vector<std::string> supportFP16_versions{"5.3", "6.0", "6.2", "7.0", "7.2", "7.5", "8.0", "8.6"};
cudaDeviceProp prop;
std::string version;
for (int dev = 0; dev < deviceCnt; dev++) {
ret = cudaGetDeviceProperties(&prop, dev);
if (ret != cudaSuccess) {
MS_LOG(ERROR) << "cuDeviceGetAttribute failed.";
return false;
}
version = std::to_string(prop.major) + "." + std::to_string(prop.minor);
if (std::find(supportFP16_versions.begin(), supportFP16_versions.end(), version) != supportFP16_versions.end()) {
MS_LOG(INFO) << "cuda device version is: " << version << ", support FP16, set enable FP16 tag successful";
return true;
}
}
MS_LOG(WARNING) << "cuda device version is: " << version << ", don't support FP16, set enable FP16 tag failed";
return false;
}
nvinfer1::ITensor *TensorRTSubGraph::SetTensorRTNetworkInput(const mindspore::MSTensor &in_tensor) {
for (int i = 0; i < this->network_->getNbInputs(); i++) {
if (in_tensor.Name().compare(this->network_->getInput(i)->getName()) == 0) {

2
mindspore/lite/src/delegate/tensorrt/tensorrt_subgraph.h
View File

@ -85,8 +85,6 @@ class TensorRTSubGraph : public kernel::Kernel {
int SetDeviceConfig(cudaStream_t stream);
bool SupportFP16();
nvinfer1::ITensor *SetTensorRTNetworkInput(const mindspore::MSTensor &in_tensor);
ITensorHelper FindTensorRTInputs(TensorRTOp *cur_op, const mindspore::MSTensor &in_tensor);

38
mindspore/lite/src/delegate/tensorrt/tensorrt_utils.cc
View File

@ -135,7 +135,7 @@ nvinfer1::ITensor *ConvertConstantTensor(nvinfer1::INetworkDefinition *network,
}
nvinfer1::Dims dims = ConvertCudaDims(ms_tensor.Shape());
if (dims.nbDims == -1) {
MS_LOG(WARNING) << "ConvertCudaDims failed for " << op_name;
MS_LOG(WARNING) << ms_tensor.Name() << " ConvertCudaDims failed, convert as scalar.";
dims.nbDims = 1;
dims.d[0] = 1;
}
@ -461,37 +461,53 @@ nvinfer1::ReduceOperation ConvertTRTReduceMode(schema::ReduceMode mode) {
}
return trt_mode;
}
nvinfer1::ITensor *PreprocessInputs2SameDim(nvinfer1::INetworkDefinition *network,
const ITensorHelper &input_tensor_helper) {
nvinfer1::ITensor *output = input_tensor_helper.trt_tensor_;
int PreprocessInputs2SameDim(nvinfer1::INetworkDefinition *network, const ITensorHelper &input_tensor_helper,
ITensorHelper *out_tensor_helper) {
out_tensor_helper->trt_tensor_ = input_tensor_helper.trt_tensor_;
out_tensor_helper->format_ = input_tensor_helper.format_;
out_tensor_helper->same_format_ = true;
if (input_tensor_helper.trt_tensor_->getDimensions().nbDims == DIMENSION_4D && !input_tensor_helper.same_format_) {
if (input_tensor_helper.format_ == Format::NCHW) {
// transpose: NCHW->NHWC
nvinfer1::IShuffleLayer *transpose_layer_in = NCHW2NHWC(network, *input_tensor_helper.trt_tensor_);
if (transpose_layer_in == nullptr) {
MS_LOG(ERROR) << "op action convert failed";
return nullptr;
return RET_ERROR;
}
transpose_layer_in->setName((std::string(output->getName()) + "_input_transpose2NHWC").c_str());
output = transpose_layer_in->getOutput(0);
transpose_layer_in->setName(
(std::string(input_tensor_helper.trt_tensor_->getName()) + "_input_transpose2NHWC").c_str());
out_tensor_helper->trt_tensor_ = transpose_layer_in->getOutput(0);
out_tensor_helper->format_ = Format::NHWC;
} else {
// transpose: NHWC->NCHW
nvinfer1::IShuffleLayer *transpose_layer_in = NHWC2NCHW(network, *input_tensor_helper.trt_tensor_);
if (transpose_layer_in == nullptr) {
MS_LOG(ERROR) << "op action convert failed";
return nullptr;
return RET_ERROR;
}
transpose_layer_in->setName((std::string(output->getName()) + "_input_transpose2NCHW").c_str());
output = transpose_layer_in->getOutput(0);
transpose_layer_in->setName(
(std::string(input_tensor_helper.trt_tensor_->getName()) + "_input_transpose2NCHW").c_str());
out_tensor_helper->trt_tensor_ = transpose_layer_in->getOutput(0);
out_tensor_helper->format_ = Format::NCHW;
}
}
return output;
return RET_OK;
}
int GetDimsVolume(const nvinfer1::Dims &dims) {
if (dims.nbDims == 0) {
return 0;
}
return std::accumulate(dims.d, dims.d + dims.nbDims, 1, std::multiplies<int64_t>());
}
int GetDimsVolume(const std::vector<int64_t> &shape) {
if (shape.size() == 0) {
return 0;
}
return std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<int64_t>());
}
void SerializeValue(void **buffer, const void *value, size_t cpy_size) {
std::memcpy(*buffer, value, cpy_size);
*buffer = static_cast<char *>(*buffer) + cpy_size;

6
mindspore/lite/src/delegate/tensorrt/tensorrt_utils.h
View File

@ -103,11 +103,13 @@ std::string GetTensorFormat(nvinfer1::ITensor *trt_tensors);
nvinfer1::ReduceOperation ConvertTRTReduceMode(schema::ReduceMode mode);
nvinfer1::ITensor *PreprocessInputs2SameDim(nvinfer1::INetworkDefinition *network,
const ITensorHelper &input_tensor_helper);
int PreprocessInputs2SameDim(nvinfer1::INetworkDefinition *network, const ITensorHelper &input_tensor_helper,
ITensorHelper *out_tensor_helper);
int GetDimsVolume(const nvinfer1::Dims &dims);
int GetDimsVolume(const std::vector<int64_t> &shape);
void SerializeValue(void **buffer, const void *value, size_t cpy_size);
void DeserializeValue(void const **buffer, size_t *buffer_size, void *value, size_t cpy_size);

4
mindspore/lite/test/config/models_tensorrt.cfg
View File

@ -15,7 +15,7 @@ ml_video_edit_imitate_filter.onnx
ml_video_edit_hair_dyeing_segmodel_20211119
ml_video_edit_detect_20211111
detect_curve2.pb
detect_straight.pb;1:input;1,19200,960,3;;offline_resize
# detect_straight.pb;1:input;1,19200,960,3;;offline_resize
direction.pb;1:input;1,2048,2048,1;;offline_resize
languageClassify.pb;1:input_0;2,32,512,1;;offline_resize
languageClassify_latin.pb;1:data;2,48,1,50;;offline_resize
@ -23,7 +23,7 @@ recognize_chineseEnglish.pb;1:input_0;1,2048,2048,1;;offline_resize
recognize_chineseEnglish_vertical.pb;1:input_0;1,2048,2048,1;;offline_resize
recognize_JapaneseKorean.pb;1:input_0;1,2048,2048,1;;offline_resize
recognize_latin.pb;1:input_0;1,2048,2048,1;;offline_resize
textremoval_v5_nofill.pb;3:input_images,input_masks,ones_image;1,1024,1024,3:1,1024,1024,1:1,1024,1024,1;;offline_resize
# textremoval_v5_nofill.pb;3:input_images,input_masks,ones_image;1,1024,1024,3:1,1024,1024,1:1,1024,1024,1;;offline_resize
# Run in distribution server
wide_and_deep_.mindir CONVERTER