mindspore-Ecosystem
/
mindspore
21
0
Fork 27
Code Issues Pull Requests 6 Projects Releases Wiki Activity

support rm minddata

This commit is contained in:
zhengjun10 2022-01-22 14:45:19 +08:00
parent ee8fba498a
commit c15307dab2
8 changed files with 63 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
mindspore/lite/CMakeLists.txt
View File

@ -251,7 +251,9 @@ endif()
if(MSLITE_ENABLE_TRAIN)
set(SUPPORT_TRAIN on)
set(MSLITE_MINDDATA_IMPLEMENT full)
if(NOT MSLITE_MINDDATA_IMPLEMENT STREQUAL "off" OR NOT PLATFORM_ARM)
set(MSLITE_MINDDATA_IMPLEMENT full)
endif()
endif()
if(MSLITE_ENABLE_NPU)
@ -610,7 +612,7 @@ if(MSLITE_ENABLE_TOOLS)
endif()
endif()
if(NOT WIN32 AND MSLITE_ENABLE_TESTCASES)
if(NOT WIN32 AND MSLITE_ENABLE_TESTCASES AND NOT MSLITE_MINDDATA_IMPLEMENT STREQUAL "off")
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/test)
endif()

40
mindspore/lite/build_lite.sh
View File

@ -379,13 +379,18 @@ build_lite_arm64_and_jni() {
is_train=off
fi
if [[ "X$is_train" = "Xon" ]]; then
cp ./${pkg_name}/runtime/lib/*.so* ${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/
cp ./${pkg_name}/runtime/lib/*.so* ${LITE_JAVA_PATH}/native/libs/arm64-v8a/
cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/
cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/native/libs/arm64-v8a/
cp ./${pkg_name}/runtime/lib/libmindspore-lite*so* ${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/
cp ./${pkg_name}/runtime/lib/libmindspore-lite*so* ${LITE_JAVA_PATH}/native/libs/arm64-v8a/
fi
local npu_so=$pkg_name/runtime/third_party/hiai_ddk/lib/libhiai.so
if [ -e "$npu_so" ]; then
minddata_so=${pkg_name}/runtime/lib/libminddata-lite.so
if [ -e "${minddata_so}" ]; then
cp ./${pkg_name}/runtime/lib/libminddata-lite.so ${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/
cp ./${pkg_name}/runtime/lib/libminddata-lite.so ${LITE_JAVA_PATH}/native/libs/arm64-v8a/
cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/
cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/native/libs/arm64-v8a/
fi
local npu_so=${pkg_name}/runtime/third_party/hiai_ddk/lib/libhiai.so
if [ -e "${npu_so}" ]; then
cp ./${pkg_name}/runtime/third_party/hiai_ddk/lib/*.so* ${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/
cp ./${pkg_name}/runtime/third_party/hiai_ddk/lib/*.so* ${LITE_JAVA_PATH}/native/libs/arm64-v8a/
fi
@ -429,10 +434,15 @@ build_lite_arm32_and_jni() {
is_train=off
fi
if [[ "X$is_train" = "Xon" ]]; then
cp ./${pkg_name}/runtime/lib/*.so* ${LITE_JAVA_PATH}/java/app/libs/armeabi-v7a/
cp ./${pkg_name}/runtime/lib/*.so* ${LITE_JAVA_PATH}/native/libs/armeabi-v7a/
cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/java/app/libs/armeabi-v7a/
cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/native/libs/armeabi-v7a/
cp ./${pkg_name}/runtime/lib/libmindspore-lite*so* ${LITE_JAVA_PATH}/java/app/libs/armeabi-v7a/
cp ./${pkg_name}/runtime/lib/libmindspore-lite*so* ${LITE_JAVA_PATH}/native/libs/armeabi-v7a/
fi
local minddata_so=${pkg_name}/runtime/lib/libminddata-lite.so
if [ -e "${minddata_so}" ]; then
cp ./${pkg_name}/runtime/lib/libminddata-lite.so ${LITE_JAVA_PATH}/java/app/libs/armeabi-v7a/
cp ./${pkg_name}/runtime/lib/libminddata-lite.so ${LITE_JAVA_PATH}/native/libs/armeabi-v7a/
cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/java/app/libs/armeabi-v7a/
cp ./${pkg_name}/runtime/third_party/libjpeg-turbo/lib/*.so* ${LITE_JAVA_PATH}/native/libs/armeabi-v7a/
fi
local npu_so=$pkg_name/runtime/third_party/hiai_ddk/lib/libhiai.so
if [ -e "$npu_so" ]; then
@ -504,13 +514,15 @@ build_aar() {
cp ${LITE_JAVA_PATH}/build/libs/mindspore-lite-java.jar ${LITE_JAVA_PATH}/java/app/libs
${gradle_command} clean -p ${LITE_JAVA_PATH}/java/app
${gradle_command} assembleRelease -p ${LITE_JAVA_PATH}/java/app
${gradle_command} publish -PLITE_VERSION=${VERSION_STR} -p ${LITE_JAVA_PATH}/java/app
cd ${LITE_JAVA_PATH}/java/app/build
[ -n "${BASEPATH}" ] && rm -rf ${BASEPATH}/output/*.tar.gz*
zip -r ${BASEPATH}/output/mindspore-lite-maven-${VERSION_STR}.zip mindspore
cd ${BASEPATH}/output
sha256sum mindspore-lite-maven-${VERSION_STR}.zip > mindspore-lite-maven-${VERSION_STR}.zip.sha256
local minddata_so=${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/libminddata-lite.so
if [ -e "${minddata_so}" ]; then
cp ${LITE_JAVA_PATH}/java/app/build/outputs/aar/mindspore-lite.aar ${BASEPATH}/output/mindspore-lite-full-${VERSION_STR}.aar
else
cp ${LITE_JAVA_PATH}/java/app/build/outputs/aar/mindspore-lite.aar ${BASEPATH}/output/mindspore-lite-${VERSION_STR}.aar
fi
}
update_submodule()

17
mindspore/lite/java/java/app/build.gradle
View File

@ -36,20 +36,3 @@ dependencies {
implementation fileTree(dir: "libs", include: ["*.jar"])
}
publishing {
repositories {
maven {
url uri("build")
}
}
publications {
maven(MavenPublication) {
groupId = 'mindspore'
artifactId = 'mindspore-lite'
version = project.hasProperty("LITE_VERSION") ? LITE_VERSION : ""
String path = project.buildDir.path + File.separator + "outputs" + File.separator + "aar" + File.separator + "mindspore-lite.aar";
artifact(path)
}
}
}

2
mindspore/lite/java/native/CMakeLists.txt
View File

@ -90,7 +90,7 @@ else()
endif()
if(SUPPORT_TRAIN)
set(LITE_TRAIN_SO_NAME mindspore-lite-train minddata-lite)
set(LITE_TRAIN_SO_NAME mindspore-lite-train)
set(JNI_TRAIN_SRC
${CMAKE_CURRENT_SOURCE_DIR}/runtime/train_session.cpp
${NEW_NATIVE_DIR}/train_config.cpp

38
mindspore/lite/src/CMakeLists.txt
View File

@ -73,15 +73,16 @@ set(API_SRC ${CXX_API_SRCS} ${C_API_SRCS})
if(NOT MSLITE_ENABLE_RUNTIME_CONVERT)
set(API_SRC ${API_SRC} ${CORE_DIR}/utils/status.cc)
endif()
file(GLOB CXX_API_TRAIN_SRCS
${CMAKE_CURRENT_SOURCE_DIR}/cxx_api/train/*.cc
${CMAKE_CURRENT_SOURCE_DIR}/cxx_api/metrics/*.cc
${CMAKE_CURRENT_SOURCE_DIR}/cxx_api/callback/*.cc
)
set(API_TRAIN_SRC
${CXX_API_TRAIN_SRCS}
)
if(MSLITE_MINDDATA_IMPLEMENT STREQUAL "full")
file(GLOB CXX_API_TRAIN_SRCS
${CMAKE_CURRENT_SOURCE_DIR}/cxx_api/train/*.cc
${CMAKE_CURRENT_SOURCE_DIR}/cxx_api/metrics/*.cc
${CMAKE_CURRENT_SOURCE_DIR}/cxx_api/callback/*.cc
)
set(API_TRAIN_SRC
${CXX_API_TRAIN_SRCS}
)
endif()
if(SUPPORT_NPU)
include_directories(${DDK_PATH})
@ -240,14 +241,19 @@ if(MSLITE_GPU_BACKEND STREQUAL cuda)
${CUDA_RUNTIME_SRC}
)
endif()
if(MSLITE_MINDDATA_IMPLEMENT STREQUAL "full")
set(TRAIN_SRC_WITH_MD
${CMAKE_CURRENT_SOURCE_DIR}/train/train_loop.cc
)
endif()
set(TRAIN_SRC
${API_TRAIN_SRC}
${TRAIN_SRC_WITH_MD}
${CMAKE_CURRENT_SOURCE_DIR}/common/quant_utils.cc
${CMAKE_CURRENT_SOURCE_DIR}/train/train_populate_parameter.cc
${CMAKE_CURRENT_SOURCE_DIR}/train/train_session.cc
${CMAKE_CURRENT_SOURCE_DIR}/train/graph_fusion.cc
${CMAKE_CURRENT_SOURCE_DIR}/train/transfer_session.cc
${CMAKE_CURRENT_SOURCE_DIR}/train/train_loop.cc
${CMAKE_CURRENT_SOURCE_DIR}/train/train_utils.cc
${CMAKE_CURRENT_SOURCE_DIR}/train/loss_monitor.cc
${CMAKE_CURRENT_SOURCE_DIR}/train/lr_scheduler.cc
@ -408,12 +414,20 @@ if(SUPPORT_TRAIN)
add_library(mindspore-lite-train SHARED $<TARGET_OBJECTS:lite_train_src_mid>)
set_target_properties(mindspore-lite-train PROPERTIES OUTPUT_NAME "mindspore-lite-train")
set_target_properties(mindspore-lite-train PROPERTIES CLEAN_DIRECT_OUTPUT 1)
target_link_libraries(mindspore-lite-train minddata-lite mindspore-lite)
if(MSLITE_MINDDATA_IMPLEMENT STREQUAL "full")
target_link_libraries(mindspore-lite-train minddata-lite mindspore-lite)
else()
target_link_libraries(mindspore-lite-train mindspore-lite)
endif()
add_library(mindspore-lite-train_static STATIC $<TARGET_OBJECTS:lite_train_src_mid>)
set_target_properties(mindspore-lite-train_static PROPERTIES OUTPUT_NAME "mindspore-lite-train")
set_target_properties(mindspore-lite-train_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
target_link_libraries(mindspore-lite-train_static minddata-lite mindspore-lite)
if(MSLITE_MINDDATA_IMPLEMENT STREQUAL "full")
target_link_libraries(mindspore-lite-train_static minddata-lite mindspore-lite)
else()
target_link_libraries(mindspore-lite-train_static mindspore-lite)
endif()
endif()
########################## build optimize and float16 library #################################

8
mindspore/lite/tools/benchmark_train/CMakeLists.txt
View File

@ -17,15 +17,15 @@ endif()
if(PLATFORM_ARM32 OR PLATFORM_ARM64)
if(SUPPORT_NPU AND ANDROID_STL STREQUAL "c++_static")
target_link_libraries(benchmark_train mindspore-lite minddata-lite mindspore-lite-train c++_shared)
target_link_libraries(benchmark_train mindspore-lite mindspore-lite-train c++_shared)
else()
target_link_libraries(benchmark_train mindspore-lite minddata-lite mindspore-lite-train)
target_link_libraries(benchmark_train mindspore-lite mindspore-lite-train)
endif()
else()
if(WIN32)
target_link_libraries(benchmark_train mindspore-lite_static mindspore-lite-train_static pthread cpu_kernel_mid
nnacl_mid minddata-lite)
nnacl_mid)
else()
target_link_libraries(benchmark_train mindspore-lite mindspore-lite-train pthread minddata-lite)
target_link_libraries(benchmark_train mindspore-lite mindspore-lite-train pthread)
endif()
endif()

7
mindspore/lite/tools/common/meta_graph_serializer.cc
View File

@ -136,12 +136,7 @@ bool MetaGraphSerializer::Init(const schema::MetaGraphT &graph, const std::strin
MS_LOG(ERROR) << "Malloc data for file head failed";
return false;
}
auto ret = memset_s(head_data, kExternalDataHeadSize, 0, kExternalDataHeadSize);
if (ret != EOK) {
MS_LOG(ERROR) << "memset failed: " << ret;
free(head_data);
return false;
}
memset(head_data, 0, kExternalDataHeadSize);
// magic number of weight file: 0x12345678
auto sum_data = reinterpret_cast<uint32_t *>(head_data);
sum_data[0] = 0x12345678;

2
scripts/lite_release_package.sh
View File

@ -25,7 +25,7 @@ function android_release_package()
rm -rf ${pkg_name}
tar -xzf ${input_path}/android_${arch}/${device}/${pkg_name}.tar.gz
# Copy java runtime to Android package
cp ${input_path}/aar/mindspore-lite-*maven*.zip ${pkg_name}
cp ${input_path}/aar/mindspore-lite-*.aar ${pkg_name}
mkdir -p ${output_path}/release/android/${device}/
tar -czf ${output_path}/release/android/${device}/${pkg_name}.tar.gz ${pkg_name}