[MSLITE] Support to convert 310 quantized caffe model.

mindspore/lite/CMakeLists.txt

@@ -34,6 +34,7 @@ option(MSLITE_ENABLE_DELEGATE "enable delegate use" on)
 option(MSLITE_ENABLE_FP16 "Whether to compile Fp16 operator" off)
 option(MSLITE_ENABLE_INT8 "Whether to compile Int8 operator" on)
 option(MSLITE_ENABLE_ACL "enable ACL" off)
+option(MSLITE_ENABLE_ACL_QUANT_PARAM "enable ACL_QUANT_PARAM" off)
 option(MSLITE_ENABLE_MODEL_ENCRYPTION "enable model encryption" off)
 option(MSLITE_ENABLE_SPARSE_COMPUTE "enable sparse kernel" off)
 option(MSLITE_ENABLE_RUNTIME_CONVERT "enable runtime convert" off)
@@ -158,6 +159,9 @@ endif()
 if(DEFINED ENV{MSLITE_ENABLE_ACL})
     set(MSLITE_ENABLE_ACL $ENV{MSLITE_ENABLE_ACL})
 endif()
+if(DEFINED ENV{MSLITE_ENABLE_ACL_QUANT_PARAM})
+    set(MSLITE_ENABLE_ACL_QUANT_PARAM $ENV{MSLITE_ENABLE_ACL_QUANT_PARAM})
+endif()
 if(DEFINED ENV{MSLITE_ENABLE_MODEL_ENCRYPTION})
     if((${CMAKE_SYSTEM_NAME} MATCHES "Linux" AND PLATFORM_X86_64)
             OR((PLATFORM_ARM64 OR PLATFORM_ARM32) AND ANDROID_NDK_TOOLCHAIN_INCLUDED))
@@ -458,6 +462,9 @@ if(MSLITE_ENABLE_ACL)
     add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=0)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--allow-shlib-undefined")
 
+    if(MSLITE_ENABLE_ACL_QUANT_PARAM)
+        add_definitions(-D ENABLE_ACL_QUANT_PARAM)
+    endif()
     if(DEFINED ENV{ASCEND_CUSTOM_PATH})
         set(ASCEND_PATH $ENV{ASCEND_CUSTOM_PATH})
     else()

mindspore/lite/tools/converter/CMakeLists.txt

@@ -270,9 +270,21 @@ if(MSLITE_GPU_BACKEND STREQUAL opencl)
 endif()
 
 file(GLOB PROTO_FILE ""
-        ${TOP_DIR}/third_party/proto/caffe/caffe.proto
         ${TOP_DIR}/third_party/proto/tensorflow/*.proto
         ${TOP_DIR}/third_party/proto/onnx/onnx.proto)
+file(GLOB CAFFE_PROTO_FILE "" ${TOP_DIR}/third_party/proto/caffe/caffe.proto)
+if(MSLITE_ENABLE_ACL AND MSLITE_ENABLE_ACL_QUANT_PARAM)
+    set(PROTO_PATCH_FILE ${TOP_DIR}/third_party/patch/caffe/caffe_proto_acl_quant_param.patch001)
+    message("patching caffe.proto < ${PROTO_PATCH_FILE}")
+    execute_process(COMMAND ${Patch_EXECUTABLE} -Np1 -i ${PROTO_PATCH_FILE} -d ${TOP_DIR}
+            -o ${TOP_DIR}/mindspore/lite/build/caffe.proto
+            RESULT_VARIABLE Result)
+    if(NOT Result EQUAL "0")
+        message(FATAL_ERROR "Failed patch: ${_LF_PATCH_FILE}")
+    endif()
+    file(GLOB CAFFE_PROTO_FILE "" ${TOP_DIR}/mindspore/lite/build/caffe.proto)
+endif()
+set(PROTO_FILE ${PROTO_FILE} ${CAFFE_PROTO_FILE})
 
 ms_protobuf_generate(PROTO_SRCS PROTO_HDRS ${PROTO_FILE})
 

mindspore/lite/tools/converter/adapter/acl/mapper/quant_dtype_cast_mapper.cc

@@ -0,0 +1,69 @@
+/**
+ * Copyright 2022 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "tools/converter/adapter/acl/mapper/quant_dtype_cast_mapper.h"
+#include <memory>
+#include "tools/converter/adapter/acl/mapper/primitive_mapper_register.h"
+#include "tools/converter/adapter/acl/mapper/tbe_op_def.h"
+#include "tools/converter/quant_param_holder.h"
+#include "src/common/log_util.h"
+#include "nnacl/op_base.h"
+
+namespace mindspore {
+namespace lite {
+namespace {
+constexpr auto kQuantInputNum = 2;
+constexpr auto kDequantInputNum = 3;
+}  // namespace
+
+STATUS QuantDTypeCastMapper::Mapper(const CNodePtr &cnode) {
+  ValueNodePtr value_node = nullptr;
+  PrimitivePtr src_prim = nullptr;
+  if (GetValueNodeAndPrimFromCnode(cnode, &value_node, &src_prim) != lite::RET_OK) {
+    MS_LOG(ERROR) << "Get primitive from cnode failed.";
+    return lite::RET_ERROR;
+  }
+
+  PrimitivePtr dst_prim = nullptr;
+  if (cnode->inputs().size() == kQuantInputNum) {
+    // map to Quant.
+    auto quant_params_holder_attr = src_prim->GetAttr("quant_params");
+    CHECK_NULL_RETURN(quant_params_holder_attr);
+    auto quant_params_holder = quant_params_holder_attr->cast<QuantParamHolderPtr>();
+    CHECK_NULL_RETURN(quant_params_holder);
+    MS_CHECK_TRUE_RET(!quant_params_holder->get_output_quant_params().empty(), RET_ERROR);
+    auto quant_param = quant_params_holder->get_output_quant_params().front();
+    MS_CHECK_TRUE_RET(!quant_param.empty(), RET_ERROR);
+    dst_prim = std::make_shared<acl::Quant>();
+    CHECK_NULL_RETURN(dst_prim);
+    dst_prim->AddAttr("scale", MakeValue(static_cast<float>(quant_param.front().scale)));
+    dst_prim->AddAttr("offset", MakeValue(static_cast<float>(quant_param.front().zeroPoint)));
+  } else if (cnode->inputs().size() == kDequantInputNum) {
+    // map to Dequant.
+    dst_prim = std::make_shared<acl::Dequant>();
+    CHECK_NULL_RETURN(dst_prim);
+  } else {
+    MS_LOG(ERROR) << "Invalid input size: " << cnode->inputs().size();
+    return lite::RET_ERROR;
+  }
+
+  value_node->set_value(dst_prim);
+  return RET_OK;
+}
+
+REGISTER_PRIMITIVE_MAPPER(kNameQuantDTypeCast, QuantDTypeCastMapper)
+}  // namespace lite
+}  // namespace mindspore

mindspore/lite/tools/converter/adapter/acl/mapper/quant_dtype_cast_mapper.h

@@ -0,0 +1,37 @@
+/**
+ * Copyright 2022 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ACL_MAPPER_PRIMITIVE_QUANT_DTYPE_CAST_MAPPER_H
+#define ACL_MAPPER_PRIMITIVE_QUANT_DTYPE_CAST_MAPPER_H
+
+#include "tools/converter/adapter/acl/mapper/primitive_mapper.h"
+#include "ops/quant_dtype_cast.h"
+
+using mindspore::ops::kNameQuantDTypeCast;
+
+namespace mindspore {
+namespace lite {
+class QuantDTypeCastMapper : public PrimitiveMapper {
+ public:
+  QuantDTypeCastMapper() : PrimitiveMapper(kNameQuantDTypeCast) {}
+
+  ~QuantDTypeCastMapper() override = default;
+
+  STATUS Mapper(const CNodePtr &cnode) override;
+};
+}  // namespace lite
+}  // namespace mindspore
+#endif  // ACL_MAPPER_PRIMITIVE_QUANT_DTYPE_CAST_MAPPER_H

mindspore/lite/tools/converter/adapter/acl/mapper/tbe_op_def.h

@@ -50,6 +50,8 @@ ADD_CONVERTER_TBE_OP(ResizeNearestNeighborV2)
 ADD_CONVERTER_TBE_OP(Conv2DBackpropInputV2)
 ADD_CONVERTER_TBE_OP(ConcatV2D)
 ADD_CONVERTER_TBE_OP(FillV1)
+ADD_CONVERTER_TBE_OP(Quant)
+ADD_CONVERTER_TBE_OP(Dequant)
 }  // namespace acl
 }  // namespace lite
 }  // namespace mindspore

mindspore/lite/tools/converter/parser/caffe/caffe_model_parser.cc

@@ -501,6 +501,19 @@ STATUS CaffeModelParser::ConvertLayerQuantParams(const caffe::LayerParameter &la
   auto quant_params_holder =
     std::make_shared<QuantParamHolder>(layer.bottom_size() + weight.blobs_size(), layer.top_size());
   MSLITE_CHECK_PTR(quant_params_holder);
+#ifdef ENABLE_ACL_QUANT_PARAM
+  // set quant parameter to output tensor of quant.
+  if (layer.type() == "Quant") {
+    QuantParamT quant_param;
+    const caffe::QuantParameter &layer_quant_param = layer.quant_param();
+    MS_CHECK_TRUE_RET(layer_quant_param.has_scale(), RET_ERROR);
+    quant_param.scale = layer_quant_param.scale();
+    MS_CHECK_TRUE_RET(layer_quant_param.has_offset(), RET_ERROR);
+    quant_param.zeroPoint = *(reinterpret_cast<int8_t *>(const_cast<char *>(layer_quant_param.offset().c_str())));
+    quant_param.inited = true;
+    quant_params_holder->set_output_quant_param(0, {quant_param});
+  }
+#endif
   primitive_c->AddAttr("quant_params", quant_params_holder);
   return RET_OK;
 }
@@ -556,6 +569,23 @@ STATUS CaffeModelParser::ConvertBlobs(const caffe::LayerParameter &layer, std::v
         buf[j] = layer.blobs(i).double_data(j);
       }
       tensor_info = CreateTensorInfo(buf.get(), count * sizeof(float), shape_vector, TypeId::kNumberTypeFloat32);
+#ifdef ENABLE_ACL_QUANT_PARAM
+    } else if (layer.blobs(i).has_int8_data()) {
+      const int8_t *data_ptr = reinterpret_cast<int8_t *>(const_cast<char *>(layer.blobs(i).int8_data().c_str()));
+      MSLITE_CHECK_PTR(data_ptr);
+      count = std::accumulate(shape_vector.begin(), shape_vector.end(), 1, std::multiplies<int>());
+      tensor_info = CreateTensorInfo(data_ptr, count * sizeof(int8_t), shape_vector, TypeId::kNumberTypeInt8);
+    } else if (layer.blobs(i).int32_data_size() > 0) {
+      count = layer.blobs(i).int32_data_size();
+      const int *data_ptr = layer.blobs(i).int32_data().data();
+      MSLITE_CHECK_PTR(data_ptr);
+      tensor_info = CreateTensorInfo(data_ptr, count * sizeof(int), shape_vector, TypeId::kNumberTypeInt32);
+    } else if (layer.blobs(i).uint64_data_size() > 0) {
+      count = layer.blobs(i).uint64_data_size();
+      const size_t *data_ptr = layer.blobs(i).uint64_data().data();
+      MSLITE_CHECK_PTR(data_ptr);
+      tensor_info = CreateTensorInfo(data_ptr, count * sizeof(size_t), shape_vector, TypeId::kNumberTypeUInt64);
+#endif
     } else {
       count = layer.blobs(i).data_size();
       const float *data_ptr = layer.blobs(i).data().data();

mindspore/lite/tools/converter/parser/caffe/caffe_quantize_parser.cc

@@ -0,0 +1,46 @@
+/**
+ * Copyright 2022 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifdef ENABLE_ACL_QUANT_PARAM
+#include "tools/converter/parser/caffe/caffe_quantize_parser.h"
+#include <memory>
+#include "ops/quant_dtype_cast.h"
+#include "nnacl/op_base.h"
+
+namespace mindspore {
+namespace lite {
+PrimitiveCPtr CaffeQuantizeParser::Parse(const caffe::LayerParameter &proto, const caffe::LayerParameter &weight) {
+  auto prim = std::make_unique<ops::QuantDTypeCast>();
+  MS_CHECK_TRUE_RET(prim != nullptr, nullptr);
+  if (proto.type() == "Quant") {
+    prim->set_src_t(kNumberTypeFloat32);
+    prim->set_dst_t(kNumberTypeInt8);
+  } else if (proto.type() == "DeQuant") {
+    prim->set_src_t(kNumberTypeInt32);
+    prim->set_dst_t(kNumberTypeFloat32);
+  } else {
+    MS_LOG(ERROR) << "Unsupported nodeType: " << proto.type();
+    return nullptr;
+  }
+
+  return prim->GetPrim();
+}
+
+CaffeNodeRegistrar g_caffeQuantizeParser("Quant", new CaffeQuantizeParser());
+CaffeNodeRegistrar g_caffeDeQuantizeParser("DeQuant", new CaffeQuantizeParser());
+}  // namespace lite
+}  // namespace mindspore
+#endif  // ENABLE_ACL_QUANT_PARAM

mindspore/lite/tools/converter/parser/caffe/caffe_quantize_parser.h

@@ -0,0 +1,36 @@
+/**
+ * Copyright 2022 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_TOOLS_CONVERTER_PARSER_CAFFE_CAFFE_QUANTIZE_PARSER_H_
+#define MINDSPORE_LITE_TOOLS_CONVERTER_PARSER_CAFFE_CAFFE_QUANTIZE_PARSER_H_
+
+#ifdef ENABLE_ACL_QUANT_PARAM
+#include "tools/converter/parser/caffe/caffe_node_parser.h"
+#include "tools/converter/parser/caffe/caffe_node_parser_registry.h"
+
+namespace mindspore {
+namespace lite {
+class CaffeQuantizeParser : public CaffeNodeParser {
+ public:
+  CaffeQuantizeParser() : CaffeNodeParser("quantize") {}
+  ~CaffeQuantizeParser() override = default;
+
+  PrimitiveCPtr Parse(const caffe::LayerParameter &proto, const caffe::LayerParameter &weight) override;
+};
+}  // namespace lite
+}  // namespace mindspore
+#endif  // ENABLE_ACL_QUANT_PARAM
+#endif  // MINDSPORE_LITE_TOOLS_CONVERTER_PARSER_CAFFE_CAFFE_QUANTIZE_PARSER_H_

third_party/patch/caffe/caffe_proto_acl_quant_param.patch001

@@ -0,0 +1,42 @@
+---
+ third_party/proto/caffe/caffe.proto | 10 ++++++++++
+ 1 file changed, 10 insertions(+)
+
+diff --git a/third_party/proto/caffe/caffe.proto b/third_party/proto/caffe/caffe.proto
+index 163734ba6c..1a2287b8a2 100755
+--- a/third_party/proto/caffe/caffe.proto
++++ b/third_party/proto/caffe/caffe.proto
+@@ -13,6 +13,9 @@ message BlobProto {
+   repeated float diff = 6 [packed = true];
+   repeated double double_data = 8 [packed = true];
+   repeated double double_diff = 9 [packed = true];
++  optional bytes int8_data = 10;
++  repeated int32 int32_data = 11 [packed = true];
++  repeated uint64 uint64_data = 12 [packed = true];
+ 
+   // 4D dimensions -- deprecated.  Use "shape" instead.
+   optional int32 num = 1 [default = 0];
+@@ -306,6 +309,12 @@ message ParamSpec {
+   optional float decay_mult = 4 [default = 1.0];
+ }
+ 
++// Quant message
++message QuantParameter {
++  optional float scale = 2;
++  optional bytes offset = 3;
++}
++
+ // NOTE
+ // Update the next available ID when you add a new LayerParameter field.
+ //
+@@ -424,6 +433,7 @@ message LayerParameter {
+   optional InterpParameter interp_param = 158;
+   optional ShuffleChannelParameter shuffle_channel_param = 159;
+   optional UpsampleParameter upsample_param = 160;
++  optional QuantParameter quant_param = 208;
+ }
+ 
+ // Message that stores parameters used to apply transformation
+-- 
+2.25.1
+