support rm minddata

mindspore/lite/CMakeLists.txt

@@ -251,7 +251,9 @@ endif()
 
 if(MSLITE_ENABLE_TRAIN)
     set(SUPPORT_TRAIN on)
-    set(MSLITE_MINDDATA_IMPLEMENT full)
+    if(NOT MSLITE_MINDDATA_IMPLEMENT STREQUAL "off" OR NOT PLATFORM_ARM)
+        set(MSLITE_MINDDATA_IMPLEMENT full)
+    endif()
 endif()
 
 if(MSLITE_ENABLE_NPU)
@@ -610,7 +612,7 @@ if(MSLITE_ENABLE_TOOLS)
     endif()
 endif()
 
-if(NOT WIN32 AND MSLITE_ENABLE_TESTCASES)
+if(NOT WIN32 AND MSLITE_ENABLE_TESTCASES AND NOT MSLITE_MINDDATA_IMPLEMENT STREQUAL "off")
     add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/test)
 endif()
 

mindspore/lite/build_lite.sh

@@ -379,13 +379,18 @@ build_lite_arm64_and_jni() {
         is_train=off
     fi
     if [[ "X$is_train" = "Xon" ]]; then
-        cp ./${pkg_name}/runtime/lib/*.so* ${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/
+        cp ./${pkg_name}/runtime/lib/libmindspore-lite*so* ${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/
-        cp ./${pkg_name}/runtime/lib/*.so* ${LITE_JAVA_PATH}/native/libs/arm64-v8a/
+        cp ./${pkg_name}/runtime/lib/libmindspore-lite*so* ${LITE_JAVA_PATH}/native/libs/arm64-v8a/
-        cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/
-        cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/native/libs/arm64-v8a/
     fi
-    local npu_so=$pkg_name/runtime/third_party/hiai_ddk/lib/libhiai.so
+    minddata_so=${pkg_name}/runtime/lib/libminddata-lite.so
-    if [ -e "$npu_so" ]; then
+    if [ -e "${minddata_so}" ]; then
+       cp ./${pkg_name}/runtime/lib/libminddata-lite.so  ${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/
+       cp ./${pkg_name}/runtime/lib/libminddata-lite.so  ${LITE_JAVA_PATH}/native/libs/arm64-v8a/
+       cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/
+       cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/native/libs/arm64-v8a/
+    fi
+    local npu_so=${pkg_name}/runtime/third_party/hiai_ddk/lib/libhiai.so
+    if [ -e "${npu_so}" ]; then
       cp ./${pkg_name}/runtime/third_party/hiai_ddk/lib/*.so* ${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/
       cp ./${pkg_name}/runtime/third_party/hiai_ddk/lib/*.so* ${LITE_JAVA_PATH}/native/libs/arm64-v8a/
     fi
@@ -429,10 +434,15 @@ build_lite_arm32_and_jni() {
         is_train=off
     fi
     if [[ "X$is_train" = "Xon" ]]; then
-        cp ./${pkg_name}/runtime/lib/*.so* ${LITE_JAVA_PATH}/java/app/libs/armeabi-v7a/
+        cp ./${pkg_name}/runtime/lib/libmindspore-lite*so* ${LITE_JAVA_PATH}/java/app/libs/armeabi-v7a/
-        cp ./${pkg_name}/runtime/lib/*.so* ${LITE_JAVA_PATH}/native/libs/armeabi-v7a/
+        cp ./${pkg_name}/runtime/lib/libmindspore-lite*so* ${LITE_JAVA_PATH}/native/libs/armeabi-v7a/
-        cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/java/app/libs/armeabi-v7a/
+    fi
-        cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/native/libs/armeabi-v7a/
+    local minddata_so=${pkg_name}/runtime/lib/libminddata-lite.so
+    if [ -e "${minddata_so}" ]; then
+       cp ./${pkg_name}/runtime/lib/libminddata-lite.so  ${LITE_JAVA_PATH}/java/app/libs/armeabi-v7a/
+       cp ./${pkg_name}/runtime/lib/libminddata-lite.so  ${LITE_JAVA_PATH}/native/libs/armeabi-v7a/
+       cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/java/app/libs/armeabi-v7a/
+       cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/native/libs/armeabi-v7a/
     fi
     local npu_so=$pkg_name/runtime/third_party/hiai_ddk/lib/libhiai.so
     if [ -e "$npu_so" ]; then
@@ -504,13 +514,15 @@ build_aar() {
     cp ${LITE_JAVA_PATH}/build/libs/mindspore-lite-java.jar ${LITE_JAVA_PATH}/java/app/libs
     ${gradle_command} clean -p ${LITE_JAVA_PATH}/java/app
     ${gradle_command} assembleRelease  -p ${LITE_JAVA_PATH}/java/app
-    ${gradle_command} publish -PLITE_VERSION=${VERSION_STR} -p ${LITE_JAVA_PATH}/java/app
 
     cd ${LITE_JAVA_PATH}/java/app/build
     [ -n "${BASEPATH}" ] && rm -rf ${BASEPATH}/output/*.tar.gz*
-    zip -r ${BASEPATH}/output/mindspore-lite-maven-${VERSION_STR}.zip mindspore
+    local minddata_so=${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/libminddata-lite.so
-    cd ${BASEPATH}/output
+    if [ -e "${minddata_so}" ]; then
-    sha256sum mindspore-lite-maven-${VERSION_STR}.zip > mindspore-lite-maven-${VERSION_STR}.zip.sha256
+      cp ${LITE_JAVA_PATH}/java/app/build/outputs/aar/mindspore-lite.aar ${BASEPATH}/output/mindspore-lite-full-${VERSION_STR}.aar
+    else
+      cp ${LITE_JAVA_PATH}/java/app/build/outputs/aar/mindspore-lite.aar ${BASEPATH}/output/mindspore-lite-${VERSION_STR}.aar
+    fi
 }
 
 update_submodule()

mindspore/lite/java/java/app/build.gradle

@@ -36,20 +36,3 @@ dependencies {
     implementation fileTree(dir: "libs", include: ["*.jar"])
 }
 
-publishing {
-    repositories {
-        maven {
-            url uri("build")
-        }
-    }
-    publications {
-        maven(MavenPublication) {
-            groupId = 'mindspore'
-            artifactId = 'mindspore-lite'
-            version = project.hasProperty("LITE_VERSION") ? LITE_VERSION : ""
-            String path = project.buildDir.path + File.separator + "outputs" + File.separator + "aar" + File.separator + "mindspore-lite.aar";
-            artifact(path)
-        }
-    }
-}
-

mindspore/lite/java/native/CMakeLists.txt

@@ -90,7 +90,7 @@ else()
 endif()
 
 if(SUPPORT_TRAIN)
-    set(LITE_TRAIN_SO_NAME mindspore-lite-train minddata-lite)
+    set(LITE_TRAIN_SO_NAME mindspore-lite-train)
     set(JNI_TRAIN_SRC
         ${CMAKE_CURRENT_SOURCE_DIR}/runtime/train_session.cpp
         ${NEW_NATIVE_DIR}/train_config.cpp

mindspore/lite/src/CMakeLists.txt

@@ -73,15 +73,16 @@ set(API_SRC ${CXX_API_SRCS} ${C_API_SRCS})
 if(NOT MSLITE_ENABLE_RUNTIME_CONVERT)
     set(API_SRC ${API_SRC} ${CORE_DIR}/utils/status.cc)
 endif()
-
+if(MSLITE_MINDDATA_IMPLEMENT STREQUAL "full")
-file(GLOB CXX_API_TRAIN_SRCS
+    file(GLOB CXX_API_TRAIN_SRCS
-        ${CMAKE_CURRENT_SOURCE_DIR}/cxx_api/train/*.cc
+            ${CMAKE_CURRENT_SOURCE_DIR}/cxx_api/train/*.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/cxx_api/metrics/*.cc
+            ${CMAKE_CURRENT_SOURCE_DIR}/cxx_api/metrics/*.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/cxx_api/callback/*.cc
+            ${CMAKE_CURRENT_SOURCE_DIR}/cxx_api/callback/*.cc
-        )
+            )
-set(API_TRAIN_SRC
+    set(API_TRAIN_SRC
-        ${CXX_API_TRAIN_SRCS}
+            ${CXX_API_TRAIN_SRCS}
-        )
+            )
+endif()
 
 if(SUPPORT_NPU)
     include_directories(${DDK_PATH})
@@ -240,14 +241,19 @@ if(MSLITE_GPU_BACKEND STREQUAL cuda)
             ${CUDA_RUNTIME_SRC}
             )
 endif()
+if(MSLITE_MINDDATA_IMPLEMENT STREQUAL "full")
+  set(TRAIN_SRC_WITH_MD
+          ${CMAKE_CURRENT_SOURCE_DIR}/train/train_loop.cc
+          )
+endif()
 set(TRAIN_SRC
         ${API_TRAIN_SRC}
+        ${TRAIN_SRC_WITH_MD}
         ${CMAKE_CURRENT_SOURCE_DIR}/common/quant_utils.cc
         ${CMAKE_CURRENT_SOURCE_DIR}/train/train_populate_parameter.cc
         ${CMAKE_CURRENT_SOURCE_DIR}/train/train_session.cc
         ${CMAKE_CURRENT_SOURCE_DIR}/train/graph_fusion.cc
         ${CMAKE_CURRENT_SOURCE_DIR}/train/transfer_session.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/train/train_loop.cc
         ${CMAKE_CURRENT_SOURCE_DIR}/train/train_utils.cc
         ${CMAKE_CURRENT_SOURCE_DIR}/train/loss_monitor.cc
         ${CMAKE_CURRENT_SOURCE_DIR}/train/lr_scheduler.cc
@@ -408,12 +414,20 @@ if(SUPPORT_TRAIN)
   add_library(mindspore-lite-train SHARED $<TARGET_OBJECTS:lite_train_src_mid>)
   set_target_properties(mindspore-lite-train PROPERTIES OUTPUT_NAME "mindspore-lite-train")
   set_target_properties(mindspore-lite-train PROPERTIES CLEAN_DIRECT_OUTPUT 1)
-  target_link_libraries(mindspore-lite-train minddata-lite mindspore-lite)
+  if(MSLITE_MINDDATA_IMPLEMENT STREQUAL "full")
+      target_link_libraries(mindspore-lite-train minddata-lite mindspore-lite)
+  else()
+      target_link_libraries(mindspore-lite-train mindspore-lite)
+  endif()
 
   add_library(mindspore-lite-train_static STATIC $<TARGET_OBJECTS:lite_train_src_mid>)
   set_target_properties(mindspore-lite-train_static PROPERTIES OUTPUT_NAME "mindspore-lite-train")
   set_target_properties(mindspore-lite-train_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
-  target_link_libraries(mindspore-lite-train_static minddata-lite mindspore-lite)
+  if(MSLITE_MINDDATA_IMPLEMENT STREQUAL "full")
+      target_link_libraries(mindspore-lite-train_static minddata-lite mindspore-lite)
+  else()
+      target_link_libraries(mindspore-lite-train_static mindspore-lite)
+  endif()
 endif()
 
 ########################## build optimize and float16 library #################################

mindspore/lite/tools/benchmark_train/CMakeLists.txt

@@ -17,15 +17,15 @@ endif()
 
 if(PLATFORM_ARM32 OR PLATFORM_ARM64)
     if(SUPPORT_NPU AND ANDROID_STL STREQUAL "c++_static")
-        target_link_libraries(benchmark_train mindspore-lite minddata-lite mindspore-lite-train c++_shared)
+        target_link_libraries(benchmark_train mindspore-lite mindspore-lite-train c++_shared)
     else()
-        target_link_libraries(benchmark_train mindspore-lite minddata-lite mindspore-lite-train)
+        target_link_libraries(benchmark_train mindspore-lite mindspore-lite-train)
     endif()
 else()
     if(WIN32)
         target_link_libraries(benchmark_train mindspore-lite_static mindspore-lite-train_static pthread cpu_kernel_mid
-                              nnacl_mid minddata-lite)
+                              nnacl_mid)
     else()
-        target_link_libraries(benchmark_train mindspore-lite mindspore-lite-train pthread minddata-lite)
+        target_link_libraries(benchmark_train mindspore-lite mindspore-lite-train pthread)
     endif()
 endif()

mindspore/lite/tools/common/meta_graph_serializer.cc

@@ -136,12 +136,7 @@ bool MetaGraphSerializer::Init(const schema::MetaGraphT &graph, const std::strin
     MS_LOG(ERROR) << "Malloc data for file head failed";
     return false;
   }
-  auto ret = memset_s(head_data, kExternalDataHeadSize, 0, kExternalDataHeadSize);
+  memset(head_data, 0, kExternalDataHeadSize);
-  if (ret != EOK) {
-    MS_LOG(ERROR) << "memset failed: " << ret;
-    free(head_data);
-    return false;
-  }
   // magic number of weight file: 0x12345678
   auto sum_data = reinterpret_cast<uint32_t *>(head_data);
   sum_data[0] = 0x12345678;

scripts/lite_release_package.sh

@@ -25,7 +25,7 @@ function android_release_package()
     rm -rf ${pkg_name}
     tar -xzf ${input_path}/android_${arch}/${device}/${pkg_name}.tar.gz
     # Copy java runtime to Android package
-    cp ${input_path}/aar/mindspore-lite-*maven*.zip ${pkg_name}
+    cp ${input_path}/aar/mindspore-lite-*.aar ${pkg_name}
 
     mkdir -p ${output_path}/release/android/${device}/
     tar -czf ${output_path}/release/android/${device}/${pkg_name}.tar.gz ${pkg_name}