support cloud inference jni

.gitignore

@@ -9,6 +9,10 @@ somas_meta/
 trace_code_graph_*
 
 # mindspore lite java
+mindspore/lite/java/.gradle/
+mindspore/lite/java/gradle
+mindspore/lite/java/gradlew
+mindspore/lite/java/gradlew.bat
 mindspore/lite/java/java/.gradle
 mindspore/lite/java/java/build
 mindspore/lite/java/java/gradle

mindspore/lite/build_lite.sh

@@ -119,9 +119,22 @@ build_lite_jni_and_jar() {
     cp ./libmindspore-lite-jni.so ${LITE_JAVA_PATH}/java/${NATIVE_PATH_ARCH}/libs/
     cp ./libmindspore-lite-jni.so ${LITE_JAVA_PATH}/native/libs/${NATIVE_PATH_ARCH}/
     cp ./libmindspore-lite-jni.so ${INSTALL_PREFIX}/${PKG_NAME}/runtime/lib/
-    cp ${BASEPATH}/output/tmp/${PKG_NAME}/runtime/lib/*.so ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/
+
-    if [ -f "`echo ${BASEPATH}/output/tmp/${PKG_NAME}/runtime/third_party/glog/${LIB_GLOG}`" ]; then
+    RUNTIME_LIB_DIR="${BASEPATH}/output/tmp/${PKG_NAME}/runtime/lib"
-      cp ${BASEPATH}/output/tmp/${PKG_NAME}/runtime/third_party/glog/libmindspore_glog.so* ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/libmindspore_glog.so
+    if [ -d ${RUNTIME_LIB_DIR} ]; then
+      if [ "$(ls -A ${RUNTIME_LIB_DIR})" ]; then
+        cp ${RUNTIME_LIB_DIR}/*.so ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/
+      fi
+    fi
+    CONVERTER_LIB_DIR="${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib"
+    if [ -d ${CONVERTER_LIB_DIR} ]; then
+      if [ "$(ls -A ${CONVERTER_LIB_DIR})" ]; then
+        cp ${CONVERTER_LIB_DIR}/*.so ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/
+      fi
+    fi
+
+    if [ -f "`echo ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/${LIB_GLOG}`" ]; then
+      cp ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/libmindspore_glog.so* ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/libmindspore_glog.so
     fi
     LIB_JPEG="libjpeg.so*"
     if [ -f "`echo ${BASEPATH}/output/tmp/${PKG_NAME}/runtime/third_party/libjpeg-turbo/lib/${LIB_JPEG}`" ]; then
@@ -131,6 +144,19 @@ build_lite_jni_and_jar() {
     if [ -f "`echo ${BASEPATH}/output/tmp/${PKG_NAME}/runtime/third_party/libjpeg-turbo/lib/${LIB_TURBOJPEG}`" ]; then
       cp ${BASEPATH}/output/tmp/${PKG_NAME}/runtime/third_party/libjpeg-turbo/lib/${LIB_TURBOJPEG} ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/libturbojpeg.so
     fi
+    LIB_OPENCV_IMGPROC="libopencv_imgproc.so*"
+    if [ -f "`echo ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/${LIB_OPENCV_IMGPROC}`" ]; then
+      cp ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/${LIB_OPENCV_IMGPROC} ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/libopencv_imgproc.so
+    fi
+    LIB_OPENCV_CORE="libopencv_core.so*"
+    if [ -f "`echo ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/${LIB_OPENCV_CORE}`" ]; then
+      cp ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/${LIB_OPENCV_CORE} ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/libopencv_core.so
+    fi
+    LIB_OPENCV_IMGCODECS="libopencv_imgcodecs.so*"
+    if [ -f "`echo ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/${LIB_OPENCV_IMGCODECS}`" ]; then
+      cp ${BASEPATH}/output/tmp/${PKG_NAME}/tools/converter/lib/${LIB_OPENCV_IMGCODECS} ${LITE_JAVA_PATH}/src/main/resources/com/mindspore/lite/${RESOURCE_PATH_ARCH}/libopencv_imgcodecs.so
+    fi
+
     if [[ "X$is_train" = "Xon" ]]; then
       cp ./libmindspore-lite-train-jni.so ${LITE_JAVA_PATH}/java/${NATIVE_PATH_ARCH}/libs/
       cp ./libmindspore-lite-train-jni.so ${LITE_JAVA_PATH}/native/libs/${NATIVE_PATH_ARCH}/

mindspore/lite/java/src/main/java/com/mindspore/config/NativeLibrary.java

@@ -10,10 +10,15 @@ public class NativeLibrary {
     private static final Logger LOGGER = MindsporeLite.GetLogger();
 
     private static final String GLOG_LIBNAME = "mindspore_glog";
-    private static final String JPEG_LIBNAME = "jpeg";
+    private static final String MINDSPORE_CORE_LIBNAME = "mindspore_core";
-    private static final String TURBOJPEG_LIBNAME = "turbojpeg";
+    private static final String OPENCV_CORE_LIBNAME = "opencv_core";
-    private static final String MINDDATA_LITE_LIBNAME = "minddata-lite";
+    private static final String OPENCV_IMGCODECS_LIBNAME = "opencv_imgcodecs";
+    private static final String OPENCV_IMGPROC_LIBNAME = "opencv_imgproc";
+    private static final String MSLITE_CONVERTER_PLUGIN_LIBNAME = "mslite_converter_plugin";
+    private static final String MINDSPORE_CONVERTER_LIBNAME = "mindspore_converter";
     private static final String MINDSPORE_LITE_LIBNAME = "mindspore-lite";
+    private static final String MSPLUGIN_GE_LITERT_LIBNAME = "msplugin-ge-litert";
+    private static final String RUNTIME_CONVERT_PLUGIN_LIBNAME = "runtime_convert_plugin";
     private static final String MINDSPORE_LITE_JNI_LIBNAME = "mindspore-lite-jni";
     private static final String MINDSPORE_LITE_TRAIN_LIBNAME = "mindspore-lite-train";
     private static final String MINDSPORE_LITE_TRAIN_JNI_LIBNAME = "mindspore-lite-train-jni";
@@ -31,16 +36,32 @@ public class NativeLibrary {
 
     /**
      * Load native libs function.
+     *
+     * dynamic library as follows:
+     * libmindspore_glog.so
+     * libopencv_core.so
+     * libopencv_imgproc.so
+     * libopencv_imgcodecs.so
+     * libmslite_converter_plugin.so
+     * libmindspore_core.so
+     * libmindspore_converter.so
+     * libmindspore-lite.so
+     * mindspore-lite-jni
+     *
+     * For cloud inference, dlopen library as follows:
+     * libmsplugin-ge-litert
+     * libruntime_convert_plugin
      */
     public static void loadLibs() {
         loadLib(makeResourceName("lib" + GLOG_LIBNAME + ".so"));
-        loadLib(makeResourceName("lib" + JPEG_LIBNAME + ".so"));
+        loadLib(makeResourceName("lib" + OPENCV_CORE_LIBNAME + ".so"));
-        loadLib(makeResourceName("lib" + TURBOJPEG_LIBNAME + ".so"));
+        loadLib(makeResourceName("lib" + OPENCV_IMGPROC_LIBNAME + ".so"));
-        loadLib(makeResourceName("lib" + MINDDATA_LITE_LIBNAME + ".so"));
+        loadLib(makeResourceName("lib" + OPENCV_IMGCODECS_LIBNAME + ".so"));
+        loadLib(makeResourceName("lib" + MINDSPORE_CORE_LIBNAME + ".so"));
+        loadLib(makeResourceName("lib" + MSLITE_CONVERTER_PLUGIN_LIBNAME + ".so"));
+        loadLib(makeResourceName("lib" + MINDSPORE_CONVERTER_LIBNAME + ".so"));
         loadLib(makeResourceName("lib" + MINDSPORE_LITE_LIBNAME + ".so"));
         loadLib(makeResourceName("lib" + MINDSPORE_LITE_JNI_LIBNAME + ".so"));
-        loadLib(makeResourceName("lib" + MINDSPORE_LITE_TRAIN_LIBNAME + ".so"));
-        loadLib(makeResourceName("lib" + MINDSPORE_LITE_TRAIN_JNI_LIBNAME + ".so"));
     }
 
     private static boolean isLibLoaded() {
@@ -64,14 +85,14 @@ public class NativeLibrary {
             loadSuccess = true;
             LOGGER.info("loadLibrary " + MINDSPORE_LITE_JNI_LIBNAME + ": success");
         } catch (UnsatisfiedLinkError e) {
-            LOGGER.info("tryLoadLibrary " + MINDSPORE_LITE_JNI_LIBNAME + " failed.");
+            LOGGER.info(String.format("tryLoadLibrary " + MINDSPORE_LITE_JNI_LIBNAME + " failed: %s", e.toString()));
         }
         try {
             System.loadLibrary(MINDSPORE_LITE_TRAIN_JNI_LIBNAME);
             loadSuccess = true;
             LOGGER.info("loadLibrary " + MINDSPORE_LITE_TRAIN_JNI_LIBNAME + ": success.");
         } catch (UnsatisfiedLinkError e) {
-            LOGGER.info("tryLoadLibrary " + MINDSPORE_LITE_TRAIN_JNI_LIBNAME + " failed.");
+            LOGGER.info(String.format("tryLoadLibrary " + MINDSPORE_LITE_TRAIN_JNI_LIBNAME + " failed: %s", e.toString()));
         }
         return loadSuccess;
     }
@@ -92,6 +113,11 @@ public class NativeLibrary {
             final File tmpFile = new File(tmpDir.getCanonicalPath(), libName);
             tmpFile.deleteOnExit();
             LOGGER.info(String.format("extract %d bytes to %s", copyLib(libResource, tmpFile), tmpFile));
+            LOGGER.info(String.format("libName %s", libName));
+            if (libName.equals("lib" + MINDSPORE_LITE_LIBNAME + ".so")) {
+                extractLib(makeResourceName("lib" + MSPLUGIN_GE_LITERT_LIBNAME + ".so"), tmpDir);
+                extractLib(makeResourceName("lib" + RUNTIME_CONVERT_PLUGIN_LIBNAME + ".so"), tmpDir);
+            }
             System.load(tmpFile.toString());
         } catch (IOException e) {
             throw new UnsatisfiedLinkError(
@@ -143,4 +169,20 @@ public class NativeLibrary {
         final String arch = System.getProperty("os.arch").toLowerCase();
         return (arch.equals("amd64")) ? "x86_64" : arch;
     }
+
+    private static void extractLib(String libResourceName, File targetDir) {
+        try {
+            final InputStream dependLibRes = NativeLibrary.class.getClassLoader().getResourceAsStream(libResourceName);
+            if (dependLibRes == null) {
+                LOGGER.warning(String.format("lib file: %s not exist.", libResourceName));
+                return;
+            }
+            String dependLibName = libResourceName.substring(libResourceName.lastIndexOf("/") + 1);
+            final File tmpDependFile = new File(targetDir.getCanonicalPath(), dependLibName);
+            tmpDependFile.deleteOnExit();
+            LOGGER.info(String.format("extract %d bytes to %s", copyLib(dependLibRes, tmpDependFile), tmpDependFile));
+        } catch (IOException e) {
+            LOGGER.warning(String.format("extract library into tmp file (%s) failed.", e.toString()));
+        }
+    }
 }

mindspore/lite/test/st/scripts/run_benchmark_x86.sh

@@ -140,7 +140,7 @@ function Run_x86_java() {
         fi
         echo ${model_name} >> "${run_x86_java_log_file}"
         echo "java -classpath .:${x86_path}/java/mindspore-lite-${version}-linux-x64/runtime/lib/mindspore-lite-java.jar Benchmark ${ms_models_path}/${model_name}.ms '${models_path}'/input_output/input/${model_name}.ms.bin '${models_path}'/input_output/output/${model_name}.ms.out 1" >> "${run_x86_java_log_file}"
-        java -classpath .:${x86_path}/java/mindspore-lite-${version}-linux-x64/runtime/lib/mindspore-lite-java.jar Benchmark ${ms_models_path}/${model_name}.ms ${models_path}/input_output/input/${model_name}.ms.bin ${models_path}/input_output/output/${model_name}.ms.out 1
+        java -classpath .:${x86_path}/java/mindspore-lite-${version}-linux-x64/runtime/lib/mindspore-lite-java.jar Benchmark ${ms_models_path}/${model_name}.ms ${models_path}/input_output/input/${model_name}.ms.bin ${models_path}/input_output/output/${model_name}.ms.out 1 2>&1 | tee ${run_x86_java_log_file}
         if [ $? = 0 ]; then
             run_result='x86_java: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}
         else

mindspore/lite/tools/converter/registry/CMakeLists.txt

@@ -16,6 +16,7 @@ set(REG_SRC ${CONVERT_REG_SRC}
         ${KERNEL_REG_DIR}/../extendrt/delegate/plugin/tensorrt_executor_plugin.cc
         ${CORE_DIR}/utils/status.cc
         ${CONVERTER_DIR}/converter_context.cc
+        ${TOP_DIR}/mindspore/lite/src/common/file_utils.cc
         )
 if(NOT (MSLITE_ENABLE_CLOUD_FUSION_INFERENCE OR MSLITE_ENABLE_CLOUD_INFERENCE))
     set(REG_SRC ${REG_SRC}
@@ -27,6 +28,11 @@ target_link_libraries(mslite_converter_plugin mindspore::glog)
 add_dependencies(mslite_converter_plugin fbs_src)
 add_dependencies(mslite_converter_plugin fbs_inner_src)
 
+if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE)
+    target_link_libraries(mslite_converter_plugin
+            mindspore_core)
+endif()
+
 if("${CMAKE_BUILD_TYPE}" STREQUAL "Release" AND (NOT WIN32))
     add_custom_command(TARGET mslite_converter_plugin POST_BUILD COMMAND strip
             ${CMAKE_CURRENT_BINARY_DIR}/libmslite_converter_plugin.so)