From b7ac26bfc6c00e11a5ce5159e2555ca09ad089b0 Mon Sep 17 00:00:00 2001 From: albert-yan Date: Thu, 2 Feb 2023 12:22:02 +0800 Subject: [PATCH] support cloud inference jni --- .gitignore | 4 ++ mindspore/lite/build_lite.sh | 32 +++++++++- .../com/mindspore/config/NativeLibrary.java | 62 ++++++++++++++++--- .../lite/test/st/scripts/run_benchmark_x86.sh | 2 +- .../tools/converter/registry/CMakeLists.txt | 6 ++ 5 files changed, 92 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index f0ed8ce0e31..e08ecf8333e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,10 @@ somas_meta/ trace_code_graph_* # mindspore lite java +mindspore/lite/java/.gradle/ +mindspore/lite/java/gradle +mindspore/lite/java/gradlew +mindspore/lite/java/gradlew.bat mindspore/lite/java/java/.gradle mindspore/lite/java/java/build mindspore/lite/java/java/gradle diff --git a/mindspore/lite/build_lite.sh b/mindspore/lite/build_lite.sh index 433e9a53dda..15669f267e0 100755 --- a/mindspore/lite/build_lite.sh +++ b/mindspore/lite/build_lite.sh @@ -119,9 +119,22 @@ build_lite_jni_and_jar() { cp ./libmindspore-lite-jni.so ${LITE_JAVA_PATH}/java/${NATIVE_PATH_ARCH}/libs/ cp ./libmindspore-lite-jni.so ${LITE_JAVA_PATH}/native/libs/${NATIVE_PATH_ARCH}/ cp ./libmindspore-lite-jni.so ${INSTALL_PREFIX}/${PKG_NAME}/runtime/lib/ - cp ${BASEPATH}/output/tmp/${PKG_NAME}/runtime/lib/*.so ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/ - if [ -f "`echo ${BASEPATH}/output/tmp/${PKG_NAME}/runtime/third_party/glog/${LIB_GLOG}`" ]; then - cp ${BASEPATH}/output/tmp/${PKG_NAME}/runtime/third_party/glog/libmindspore_glog.so* ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/libmindspore_glog.so + + RUNTIME_LIB_DIR="${BASEPATH}/output/tmp/${PKG_NAME}/runtime/lib" + if [ -d ${RUNTIME_LIB_DIR} ]; then + if [ "$(ls -A ${RUNTIME_LIB_DIR})" ]; then + cp ${RUNTIME_LIB_DIR}/*.so ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/ + fi + fi + CONVERTER_LIB_DIR="${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib" + if [ -d ${CONVERTER_LIB_DIR} ]; then + if [ "$(ls -A ${CONVERTER_LIB_DIR})" ]; then + cp ${CONVERTER_LIB_DIR}/*.so ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/ + fi + fi + + if [ -f "`echo ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/${LIB_GLOG}`" ]; then + cp ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/libmindspore_glog.so* ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/libmindspore_glog.so fi LIB_JPEG="libjpeg.so*" if [ -f "`echo ${BASEPATH}/output/tmp/${PKG_NAME}/runtime/third_party/libjpeg-turbo/lib/${LIB_JPEG}`" ]; then @@ -131,6 +144,19 @@ build_lite_jni_and_jar() { if [ -f "`echo ${BASEPATH}/output/tmp/${PKG_NAME}/runtime/third_party/libjpeg-turbo/lib/${LIB_TURBOJPEG}`" ]; then cp ${BASEPATH}/output/tmp/${PKG_NAME}/runtime/third_party/libjpeg-turbo/lib/${LIB_TURBOJPEG} ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/libturbojpeg.so fi + LIB_OPENCV_IMGPROC="libopencv_imgproc.so*" + if [ -f "`echo ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/${LIB_OPENCV_IMGPROC}`" ]; then + cp ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/${LIB_OPENCV_IMGPROC} ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/libopencv_imgproc.so + fi + LIB_OPENCV_CORE="libopencv_core.so*" + if [ -f "`echo ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/${LIB_OPENCV_CORE}`" ]; then + cp ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/${LIB_OPENCV_CORE} ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/libopencv_core.so + fi + LIB_OPENCV_IMGCODECS="libopencv_imgcodecs.so*" + if [ -f "`echo ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/${LIB_OPENCV_IMGCODECS}`" ]; then + cp ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/${LIB_OPENCV_IMGCODECS} ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/libopencv_imgcodecs.so + fi + if [[ "X$is_train" = "Xon" ]]; then cp ./libmindspore-lite-train-jni.so ${LITE_JAVA_PATH}/java/${NATIVE_PATH_ARCH}/libs/ cp ./libmindspore-lite-train-jni.so ${LITE_JAVA_PATH}/native/libs/${NATIVE_PATH_ARCH}/ diff --git a/mindspore/lite/java/src/main/java/com/mindspore/config/NativeLibrary.java b/mindspore/lite/java/src/main/java/com/mindspore/config/NativeLibrary.java index 7b92dddc860..16b2ed5412a 100644 --- a/mindspore/lite/java/src/main/java/com/mindspore/config/NativeLibrary.java +++ b/mindspore/lite/java/src/main/java/com/mindspore/config/NativeLibrary.java @@ -10,10 +10,15 @@ public class NativeLibrary { private static final Logger LOGGER = MindsporeLite.GetLogger(); private static final String GLOG_LIBNAME = "mindspore_glog"; - private static final String JPEG_LIBNAME = "jpeg"; - private static final String TURBOJPEG_LIBNAME = "turbojpeg"; - private static final String MINDDATA_LITE_LIBNAME = "minddata-lite"; + private static final String MINDSPORE_CORE_LIBNAME = "mindspore_core"; + private static final String OPENCV_CORE_LIBNAME = "opencv_core"; + private static final String OPENCV_IMGCODECS_LIBNAME = "opencv_imgcodecs"; + private static final String OPENCV_IMGPROC_LIBNAME = "opencv_imgproc"; + private static final String MSLITE_CONVERTER_PLUGIN_LIBNAME = "mslite_converter_plugin"; + private static final String MINDSPORE_CONVERTER_LIBNAME = "mindspore_converter"; private static final String MINDSPORE_LITE_LIBNAME = "mindspore-lite"; + private static final String MSPLUGIN_GE_LITERT_LIBNAME = "msplugin-ge-litert"; + private static final String RUNTIME_CONVERT_PLUGIN_LIBNAME = "runtime_convert_plugin"; private static final String MINDSPORE_LITE_JNI_LIBNAME = "mindspore-lite-jni"; private static final String MINDSPORE_LITE_TRAIN_LIBNAME = "mindspore-lite-train"; private static final String MINDSPORE_LITE_TRAIN_JNI_LIBNAME = "mindspore-lite-train-jni"; @@ -31,16 +36,32 @@ public class NativeLibrary { /** * Load native libs function. + * + * dynamic library as follows: + * libmindspore_glog.so + * libopencv_core.so + * libopencv_imgproc.so + * libopencv_imgcodecs.so + * libmslite_converter_plugin.so + * libmindspore_core.so + * libmindspore_converter.so + * libmindspore-lite.so + * mindspore-lite-jni + * + * For cloud inference, dlopen library as follows: + * libmsplugin-ge-litert + * libruntime_convert_plugin */ public static void loadLibs() { loadLib(makeResourceName("lib" + GLOG_LIBNAME + ".so")); - loadLib(makeResourceName("lib" + JPEG_LIBNAME + ".so")); - loadLib(makeResourceName("lib" + TURBOJPEG_LIBNAME + ".so")); - loadLib(makeResourceName("lib" + MINDDATA_LITE_LIBNAME + ".so")); + loadLib(makeResourceName("lib" + OPENCV_CORE_LIBNAME + ".so")); + loadLib(makeResourceName("lib" + OPENCV_IMGPROC_LIBNAME + ".so")); + loadLib(makeResourceName("lib" + OPENCV_IMGCODECS_LIBNAME + ".so")); + loadLib(makeResourceName("lib" + MINDSPORE_CORE_LIBNAME + ".so")); + loadLib(makeResourceName("lib" + MSLITE_CONVERTER_PLUGIN_LIBNAME + ".so")); + loadLib(makeResourceName("lib" + MINDSPORE_CONVERTER_LIBNAME + ".so")); loadLib(makeResourceName("lib" + MINDSPORE_LITE_LIBNAME + ".so")); loadLib(makeResourceName("lib" + MINDSPORE_LITE_JNI_LIBNAME + ".so")); - loadLib(makeResourceName("lib" + MINDSPORE_LITE_TRAIN_LIBNAME + ".so")); - loadLib(makeResourceName("lib" + MINDSPORE_LITE_TRAIN_JNI_LIBNAME + ".so")); } private static boolean isLibLoaded() { @@ -64,14 +85,14 @@ public class NativeLibrary { loadSuccess = true; LOGGER.info("loadLibrary " + MINDSPORE_LITE_JNI_LIBNAME + ": success"); } catch (UnsatisfiedLinkError e) { - LOGGER.info("tryLoadLibrary " + MINDSPORE_LITE_JNI_LIBNAME + " failed."); + LOGGER.info(String.format("tryLoadLibrary " + MINDSPORE_LITE_JNI_LIBNAME + " failed: %s", e.toString())); } try { System.loadLibrary(MINDSPORE_LITE_TRAIN_JNI_LIBNAME); loadSuccess = true; LOGGER.info("loadLibrary " + MINDSPORE_LITE_TRAIN_JNI_LIBNAME + ": success."); } catch (UnsatisfiedLinkError e) { - LOGGER.info("tryLoadLibrary " + MINDSPORE_LITE_TRAIN_JNI_LIBNAME + " failed."); + LOGGER.info(String.format("tryLoadLibrary " + MINDSPORE_LITE_TRAIN_JNI_LIBNAME + " failed: %s", e.toString())); } return loadSuccess; } @@ -92,6 +113,11 @@ public class NativeLibrary { final File tmpFile = new File(tmpDir.getCanonicalPath(), libName); tmpFile.deleteOnExit(); LOGGER.info(String.format("extract %d bytes to %s", copyLib(libResource, tmpFile), tmpFile)); + LOGGER.info(String.format("libName %s", libName)); + if (libName.equals("lib" + MINDSPORE_LITE_LIBNAME + ".so")) { + extractLib(makeResourceName("lib" + MSPLUGIN_GE_LITERT_LIBNAME + ".so"), tmpDir); + extractLib(makeResourceName("lib" + RUNTIME_CONVERT_PLUGIN_LIBNAME + ".so"), tmpDir); + } System.load(tmpFile.toString()); } catch (IOException e) { throw new UnsatisfiedLinkError( @@ -143,4 +169,20 @@ public class NativeLibrary { final String arch = System.getProperty("os.arch").toLowerCase(); return (arch.equals("amd64")) ? "x86_64" : arch; } + + private static void extractLib(String libResourceName, File targetDir) { + try { + final InputStream dependLibRes = NativeLibrary.class.getClassLoader().getResourceAsStream(libResourceName); + if (dependLibRes == null) { + LOGGER.warning(String.format("lib file: %s not exist.", libResourceName)); + return; + } + String dependLibName = libResourceName.substring(libResourceName.lastIndexOf("/") + 1); + final File tmpDependFile = new File(targetDir.getCanonicalPath(), dependLibName); + tmpDependFile.deleteOnExit(); + LOGGER.info(String.format("extract %d bytes to %s", copyLib(dependLibRes, tmpDependFile), tmpDependFile)); + } catch (IOException e) { + LOGGER.warning(String.format("extract library into tmp file (%s) failed.", e.toString())); + } + } } diff --git a/mindspore/lite/test/st/scripts/run_benchmark_x86.sh b/mindspore/lite/test/st/scripts/run_benchmark_x86.sh index 01969924e49..ee15da669df 100644 --- a/mindspore/lite/test/st/scripts/run_benchmark_x86.sh +++ b/mindspore/lite/test/st/scripts/run_benchmark_x86.sh @@ -140,7 +140,7 @@ function Run_x86_java() { fi echo ${model_name} >> "${run_x86_java_log_file}" echo "java -classpath .:${x86_path}/java/mindspore-lite-${version}-linux-x64/runtime/lib/mindspore-lite-java.jar Benchmark ${ms_models_path}/${model_name}.ms '${models_path}'/input_output/input/${model_name}.ms.bin '${models_path}'/input_output/output/${model_name}.ms.out 1" >> "${run_x86_java_log_file}" - java -classpath .:${x86_path}/java/mindspore-lite-${version}-linux-x64/runtime/lib/mindspore-lite-java.jar Benchmark ${ms_models_path}/${model_name}.ms ${models_path}/input_output/input/${model_name}.ms.bin ${models_path}/input_output/output/${model_name}.ms.out 1 + java -classpath .:${x86_path}/java/mindspore-lite-${version}-linux-x64/runtime/lib/mindspore-lite-java.jar Benchmark ${ms_models_path}/${model_name}.ms ${models_path}/input_output/input/${model_name}.ms.bin ${models_path}/input_output/output/${model_name}.ms.out 1 2>&1 | tee ${run_x86_java_log_file} if [ $? = 0 ]; then run_result='x86_java: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file} else diff --git a/mindspore/lite/tools/converter/registry/CMakeLists.txt b/mindspore/lite/tools/converter/registry/CMakeLists.txt index 1de7e18d38b..1630077526d 100644 --- a/mindspore/lite/tools/converter/registry/CMakeLists.txt +++ b/mindspore/lite/tools/converter/registry/CMakeLists.txt @@ -16,6 +16,7 @@ set(REG_SRC ${CONVERT_REG_SRC} ${KERNEL_REG_DIR}/../extendrt/delegate/plugin/tensorrt_executor_plugin.cc ${CORE_DIR}/utils/status.cc ${CONVERTER_DIR}/converter_context.cc + ${TOP_DIR}/mindspore/lite/src/common/file_utils.cc ) if(NOT (MSLITE_ENABLE_CLOUD_FUSION_INFERENCE OR MSLITE_ENABLE_CLOUD_INFERENCE)) set(REG_SRC ${REG_SRC} @@ -27,6 +28,11 @@ target_link_libraries(mslite_converter_plugin mindspore::glog) add_dependencies(mslite_converter_plugin fbs_src) add_dependencies(mslite_converter_plugin fbs_inner_src) +if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE) + target_link_libraries(mslite_converter_plugin + mindspore_core) +endif() + if("${CMAKE_BUILD_TYPE}" STREQUAL "Release" AND (NOT WIN32)) add_custom_command(TARGET mslite_converter_plugin POST_BUILD COMMAND strip ${CMAKE_CURRENT_BINARY_DIR}/libmslite_converter_plugin.so)