From c532a40d5507593cff64d5700d137cd2da19c247 Mon Sep 17 00:00:00 2001 From: yeyunpeng2020 Date: Thu, 19 Aug 2021 20:10:11 +0800 Subject: [PATCH] converter support image preprocess --- cmake/external_libs/opencv.cmake | 10 +-- cmake/package_lite.cmake | 18 ++++ mindspore/lite/tools/converter/CMakeLists.txt | 5 +- .../tools/converter/preprocess/CMakeLists.txt | 11 +++ .../converter/preprocess/image_preprocess.cc | 82 +++++++++++++++++++ .../converter/preprocess/image_preprocess.h | 37 +++++++++ .../preprocess/opencv_convert_utils.cc | 48 +++++++++++ .../preprocess/opencv_convert_utils.h | 29 +++++++ 8 files changed, 234 insertions(+), 6 deletions(-) create mode 100644 mindspore/lite/tools/converter/preprocess/CMakeLists.txt create mode 100644 mindspore/lite/tools/converter/preprocess/image_preprocess.cc create mode 100644 mindspore/lite/tools/converter/preprocess/image_preprocess.h create mode 100644 mindspore/lite/tools/converter/preprocess/opencv_convert_utils.cc create mode 100644 mindspore/lite/tools/converter/preprocess/opencv_convert_utils.h diff --git a/cmake/external_libs/opencv.cmake b/cmake/external_libs/opencv.cmake index 75edcc6a5a5..2d2c8dcc466 100644 --- a/cmake/external_libs/opencv.cmake +++ b/cmake/external_libs/opencv.cmake @@ -18,7 +18,7 @@ else() endif() if(ENABLE_GITEE) - if(PYTHON_VERSION MATCHES "3.9") + if("${BUILD_LITE}" STREQUAL "on" OR PYTHON_VERSION MATCHES "3.9") set(REQ_URL "https://gitee.com/mirrors/opencv/repository/archive/4.5.1.tar.gz") set(MD5 "e74309207f2fa88fb6cc417d8ea9ff09") elseif((PYTHON_VERSION MATCHES "3.7") OR (PYTHON_VERSION MATCHES "3.8")) @@ -29,7 +29,7 @@ if(ENABLE_GITEE) return() endif() else() - if(PYTHON_VERSION MATCHES "3.9") + if("${BUILD_LITE}" STREQUAL "on" OR PYTHON_VERSION MATCHES "3.9") set(REQ_URL "https://github.com/opencv/opencv/archive/4.5.1.tar.gz") set(MD5 "2205d3169238ec1f184438a96de68513") elseif((PYTHON_VERSION MATCHES "3.7") OR (PYTHON_VERSION MATCHES "3.8")) @@ -42,7 +42,7 @@ else() endif() if(WIN32) - if(PYTHON_VERSION MATCHES "3.9") + if("${BUILD_LITE}" STREQUAL "on" OR PYTHON_VERSION MATCHES "3.9") mindspore_add_pkg(opencv VER 4.5.1 LIBS libopencv_core451.dll.a libopencv_imgcodecs451.dll.a libopencv_imgproc451.dll.a @@ -97,7 +97,7 @@ if(WIN32) -DTIFF_LIBRARY=${tiff_LIB}) endif() else() - if(PYTHON_VERSION MATCHES "3.9") + if("${BUILD_LITE}" STREQUAL "on" OR PYTHON_VERSION MATCHES "3.9") mindspore_add_pkg(opencv VER 4.5.1 LIBS opencv_core opencv_imgcodecs opencv_imgproc @@ -150,7 +150,7 @@ else() endif() if(WIN32) - if(PYTHON_VERSION MATCHES "3.9") + if("${BUILD_LITE}" STREQUAL "on" OR PYTHON_VERSION MATCHES "3.9") include_directories(${opencv_INC}) add_library(mindspore::opencv_core ALIAS opencv::libopencv_core451.dll.a) add_library(mindspore::opencv_imgcodecs ALIAS opencv::libopencv_imgcodecs451.dll.a) diff --git a/cmake/package_lite.cmake b/cmake/package_lite.cmake index fff35b85b26..af35299686c 100644 --- a/cmake/package_lite.cmake +++ b/cmake/package_lite.cmake @@ -353,6 +353,13 @@ elseif(WIN32) COMPONENT ${RUNTIME_COMPONENT_NAME}) install(FILES ${protobuf_LIBPATH}/libprotobuf.a DESTINATION ${CONVERTER_ROOT_DIR}/lib COMPONENT ${RUNTIME_COMPONENT_NAME}) + file(GLOB_RECURSE OPENCV_LIB_LIST + ${opencv_LIBPATH}/../bin/libopencv_core* + ${opencv_LIBPATH}/../bin/libopencv_imgcodecs* + ${opencv_LIBPATH}/../bin/libopencv_imgproc* + ) + install(FILES ${OPENCV_LIB_LIST} DESTINATION ${CONVERTER_ROOT_DIR}/lib COMPONENT ${RUNTIME_COMPONENT_NAME}) + __install_micro_wrapper() __install_micro_codegen() endif() @@ -493,6 +500,17 @@ else() DESTINATION ${CONVERTER_ROOT_DIR}/lib COMPONENT ${RUNTIME_COMPONENT_NAME}) install(FILES ${glog_LIBPATH}/libglog.so.0.4.0 DESTINATION ${CONVERTER_ROOT_DIR}/lib RENAME libglog.so.0 COMPONENT ${RUNTIME_COMPONENT_NAME}) + + install(FILES ${opencv_LIBPATH}/libopencv_core.so.4.5.1 + DESTINATION ${CONVERTER_ROOT_DIR}/lib RENAME libopencv_core.so.4.5 + COMPONENT ${RUNTIME_COMPONENT_NAME}) + install(FILES ${opencv_LIBPATH}/libopencv_imgcodecs.so.4.5.1 + DESTINATION ${CONVERTER_ROOT_DIR}/lib RENAME libopencv_imgcodecs.so.4.5 + COMPONENT ${RUNTIME_COMPONENT_NAME}) + install(FILES ${opencv_LIBPATH}/libopencv_imgproc.so.4.5.1 + DESTINATION ${CONVERTER_ROOT_DIR}/lib RENAME libopencv_imgproc.so.4.5 + COMPONENT ${RUNTIME_COMPONENT_NAME}) + if(MSLITE_ENABLE_NNIE) install(FILES ${glog_LIBPATH}/libglog.so.0.4.0 DESTINATION ${CONVERTER_ROOT_DIR}/lib RENAME libglog.so COMPONENT ${RUNTIME_COMPONENT_NAME}) diff --git a/mindspore/lite/tools/converter/CMakeLists.txt b/mindspore/lite/tools/converter/CMakeLists.txt index 23b7aab4013..4c8102baba1 100644 --- a/mindspore/lite/tools/converter/CMakeLists.txt +++ b/mindspore/lite/tools/converter/CMakeLists.txt @@ -6,7 +6,8 @@ set(CCSRC_SRC ${CCSRC_DIR}/backend/optimizer/common/visit.cc ${CCSRC_DIR}/backend/optimizer/common/optimizer.cc ) - +set(ENABLE_GLIBCXX ON) +include(${TOP_DIR}/cmake/external_libs/opencv.cmake) include(${TOP_DIR}/cmake/external_libs/glog.cmake) include_directories(${TOP_DIR}/mindspore/ccsrc/backend/kernel_compiler/cpu) @@ -127,6 +128,7 @@ add_subdirectory(parser/tf) add_subdirectory(legacy_optimizer) add_subdirectory(quantizer) add_subdirectory(registry) +add_subdirectory(preprocess) add_subdirectory(${CORE_DIR} mindspore_core) set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../src) @@ -212,6 +214,7 @@ target_link_libraries(converter_lite PRIVATE mindspore::glog mindspore::protobuf mindspore::flatbuffers + preprocess_mid ) if(NOT MSVC) diff --git a/mindspore/lite/tools/converter/preprocess/CMakeLists.txt b/mindspore/lite/tools/converter/preprocess/CMakeLists.txt new file mode 100644 index 00000000000..e713ab73ce6 --- /dev/null +++ b/mindspore/lite/tools/converter/preprocess/CMakeLists.txt @@ -0,0 +1,11 @@ +file(GLOB_RECURSE PREPROCESS_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + *.cc + ) +set_property(SOURCE ${PREPROCESS_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_LITE) +add_library(preprocess_mid OBJECT + ${PREPROCESS_SRC_LIST} + ) +target_link_libraries(preprocess_mid + mindspore::opencv_core + mindspore::opencv_imgcodecs + mindspore::opencv_imgproc) \ No newline at end of file diff --git a/mindspore/lite/tools/converter/preprocess/image_preprocess.cc b/mindspore/lite/tools/converter/preprocess/image_preprocess.cc new file mode 100644 index 00000000000..061389bc046 --- /dev/null +++ b/mindspore/lite/tools/converter/preprocess/image_preprocess.cc @@ -0,0 +1,82 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tools/converter/preprocess/image_preprocess.h" +#include "src/common/log_adapter.h" +#include "include/errorcode.h" +namespace mindspore { +namespace lite { +namespace preprocess { +int ReadImage(const std::string &image_path, cv::Mat *image) { + *image = cv::imread(image_path); + if (image->empty() || image->data == nullptr) { + MS_LOG(ERROR) << "missing file, improper permissions, unsupported or invalid format."; + return RET_ERROR; + } + return RET_OK; +} + +int ConvertImageFormat(cv::Mat *image, cv::ColorConversionCodes to_format) { + if (to_format == cv::COLOR_COLORCVT_MAX) { + MS_LOG(ERROR) << "to_format is cv::COLOR_COLORCVT_MAX"; + return RET_ERROR; + } + cv::cvtColor(*image, *image, to_format); + return RET_OK; +} + +int Normalize(cv::Mat *image, const std::vector &mean, const std::vector &std) { + if (static_cast(mean.size()) != image->channels() && static_cast(std.size()) != image->channels()) { + MS_LOG(ERROR) << "mean size:" << mean.size() << " != image->dims:" << image->dims << " or scale size:" << std.size() + << " !=image->dims:" << image->dims; + return RET_ERROR; + } + std::vector channels(std.size()); + cv::split(*image, channels); + for (size_t i = 0; i < channels.size(); i++) { + channels[i].convertTo(channels[i], CV_32FC1, 1.0 / std[i], (0.0 - mean[i]) / std[i]); + } + cv::merge(channels, *image); + return RET_OK; +} + +int Resize(cv::Mat *image, int width, int height, cv::InterpolationFlags resize_method) { + if (resize_method == cv::INTER_MAX) { + MS_LOG(ERROR) << "resize method is cv::INTER_MAX"; + return RET_ERROR; + } + cv::resize(*image, *image, cv::Size(width, height), 0, 0, resize_method); + return RET_OK; +} + +int CenterCrop(cv::Mat *image, int width, int height) { + if (width > image->cols || height > image->rows) { + MS_LOG(ERROR) << "width:" << width << " > " + << "image->cols:" << image->cols << " or" + << "height:" << height << " > " + << "image->rows:" << image->rows; + return RET_ERROR; + } + const int offsetW = (image->cols - width) / 2; + const int offsetH = (image->rows - height) / 2; + const cv::Rect roi(offsetW, offsetH, width, height); + cv::Mat image_object = *(image); + *(image) = image_object(roi).clone(); + return RET_OK; +} +} // namespace preprocess +} // namespace lite +} // namespace mindspore diff --git a/mindspore/lite/tools/converter/preprocess/image_preprocess.h b/mindspore/lite/tools/converter/preprocess/image_preprocess.h new file mode 100644 index 00000000000..dac61bb491e --- /dev/null +++ b/mindspore/lite/tools/converter/preprocess/image_preprocess.h @@ -0,0 +1,37 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_LITE_TOOLS_CONVERTER_PREPROCESS_IMAGE_PREPROCESS_H +#define MINDSPORE_LITE_TOOLS_CONVERTER_PREPROCESS_IMAGE_PREPROCESS_H + +#include +#include +#include +namespace mindspore { +namespace lite { +namespace preprocess { +int ReadImage(const std::string &image_path, cv::Mat *image); + +int ConvertImageFormat(cv::Mat *image, cv::ColorConversionCodes to_format); + +int Normalize(cv::Mat *image, const std::vector &mean, const std::vector &std); + +int Resize(cv::Mat *image, int width, int height, cv::InterpolationFlags resize_method); + +int CenterCrop(cv::Mat *image, int width, int height); +} // namespace preprocess +} // namespace lite +} // namespace mindspore +#endif // MINDSPORE_LITE_TOOLS_CONVERTER_PREPROCESS_IMAGE_PREPROCESS_H diff --git a/mindspore/lite/tools/converter/preprocess/opencv_convert_utils.cc b/mindspore/lite/tools/converter/preprocess/opencv_convert_utils.cc new file mode 100644 index 00000000000..1d7a4ae22a7 --- /dev/null +++ b/mindspore/lite/tools/converter/preprocess/opencv_convert_utils.cc @@ -0,0 +1,48 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tools/converter/preprocess/opencv_convert_utils.h" +#include "src/common/log_adapter.h" +#include "include/errorcode.h" +namespace mindspore { +namespace lite { +namespace preprocess { +cv::ColorConversionCodes ConvertColorConversionCodes(const std::string &format) { + if (format == "RGB") { + return cv::COLOR_BGR2RGB; + } else if (format == "GRAY") { + return cv::COLOR_BGR2GRAY; + } else { + MS_LOG(ERROR) << "Unsupported format:" << format; + return cv::COLOR_COLORCVT_MAX; + } +} + +cv::InterpolationFlags ConvertResizeFlag(const std::string &flag) { + if (flag == "NEAREST") { + return cv::INTER_NEAREST; + } else if (flag == "LINEAR") { + return cv::INTER_LINEAR; + } else if (flag == "CUBIC") { + return cv::INTER_CUBIC; + } else { + MS_LOG(ERROR) << "Unsupported resize method:" << flag; + return cv::INTER_MAX; + } +} +} // namespace preprocess +} // namespace lite +} // namespace mindspore diff --git a/mindspore/lite/tools/converter/preprocess/opencv_convert_utils.h b/mindspore/lite/tools/converter/preprocess/opencv_convert_utils.h new file mode 100644 index 00000000000..7830d3e6699 --- /dev/null +++ b/mindspore/lite/tools/converter/preprocess/opencv_convert_utils.h @@ -0,0 +1,29 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_LITE_TOOLS_CONVERTER_PREPROCESS_OPENCV_CONVERT_UTILS_H +#define MINDSPORE_LITE_TOOLS_CONVERTER_PREPROCESS_OPENCV_CONVERT_UTILS_H +#include +#include +namespace mindspore { +namespace lite { +namespace preprocess { +cv::ColorConversionCodes ConvertColorConversionCodes(const std::string &format); + +cv::InterpolationFlags ConvertResizeFlag(const std::string &flag); +} // namespace preprocess +} // namespace lite +} // namespace mindspore +#endif // MINDSPORE_LITE_TOOLS_CONVERTER_PREPROCESS_OPENCV_CONVERT_UTILS_H