mindspore-Ecosystem
/
mindspore
21
0
Fork 27
Code Issues Pull Requests 6 Projects Releases Wiki Activity

Update Model Obf

This commit is contained in:
Shukun Zhang 2022-09-21 11:05:24 +08:00
parent ca3dfc8d15
commit ea0beea588
11 changed files with 54 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmake/package_lite.cmake
View File

@ -430,7 +430,7 @@ if(PLATFORM_ARM64)
install(FILES ${TOP_DIR}/mindspore/lite/build/src/${MINDSPORE_LITE_LIB_NAME}.a DESTINATION ${RUNTIME_LIB_DIR}
COMPONENT ${RUNTIME_COMPONENT_NAME})
endif()
if(ENABLE_MODEL_OBF)
if(MSLITE_ENABLE_MODEL_OBF)
install(FILES ${TOP_DIR}/mindspore/lite/tools/obfuscator/lib/android-aarch64/libmsdeobfuscator-lite.so
DESTINATION ${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME})
endif()
@ -669,7 +669,7 @@ elseif(PLATFORM_ARM32)
install(FILES ${TOP_DIR}/mindspore/lite/build/src/${MINDSPORE_LITE_LIB_NAME}.a DESTINATION ${RUNTIME_LIB_DIR}
COMPONENT ${RUNTIME_COMPONENT_NAME})
endif()
if(ENABLE_MODEL_OBF)
if(MSLITE_ENABLE_MODEL_OBF)
install(FILES ${TOP_DIR}/mindspore/lite/tools/obfuscator/lib/android-aarch32/libmsdeobfuscator-lite.so
DESTINATION ${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME})
endif()
@ -861,7 +861,7 @@ else()
install(FILES ${TOP_DIR}/mindspore/lite/build/src/${MINDSPORE_LITE_LIB_NAME}.a DESTINATION ${RUNTIME_LIB_DIR}
COMPONENT ${RUNTIME_COMPONENT_NAME})
endif()
if(ENABLE_MODEL_OBF)
if(MSLITE_ENABLE_MODEL_OBF)
install(FILES ${TOP_DIR}/mindspore/lite/tools/obfuscator/bin/linux-x64/msobfuscator
DESTINATION ${OBFUSCATOR_ROOT_DIR} PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE COMPONENT ${RUNTIME_COMPONENT_NAME})

11
mindspore/lite/CMakeLists.txt
View File

@ -52,10 +52,10 @@ option(MSLITE_ENABLE_CONVERT_PYTORCH_MODEL "enable to convert pytorch model" off
option(MSLITE_ENABLE_KERNEL_EXECUTOR "enable kernel executor" off)
option(MSLITE_ENABLE_GITEE_MIRROR "enable download third_party from gitee mirror" off)
option(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE "enable cloud and device fusion inference architecture" off)
option(MSLITE_ENABLE_MODEL_OBF "enable model obfuscation" off)
#Option that can be configured through manually
option(ENABLE_VERBOSE "" off)
option(ENABLE_MODEL_OBF "if support model obfuscation" off)
set(VERSION_STR "1.8.0" CACHE STRING "get from version")
add_definitions(-DVERSION_STR=\"${VERSION_STR}\")
@ -228,6 +228,10 @@ if(DEFINED ENV{ENABLE_FAST_HASH_TABLE})
set(MSLITE_DEPS_ROBIN_HOOD_HASHING on)
endif()
if(DEFINED ENV{MSLITE_ENABLE_MODEL_OBF})
set(MSLITE_ENABLE_MODEL_OBF $ENV{MSLITE_ENABLE_MODEL_OBF})
endif()
if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE)
add_compile_definitions(ENABLE_CLOUD_FUSION_INFERENCE)
add_compile_definitions(SUPPORT_GPU)
@ -481,6 +485,7 @@ message(STATUS "\tMSLITE_ENABLE_GRAPH_KERNEL = \t${MSLITE_ENABLE
message(STATUS "\tMSLITE_ENABLE_KERNEL_EXECUTOR = \t${MSLITE_ENABLE_KERNEL_EXECUTOR}")
message(STATUS "\tMSLITE_ENABLE_CLOUD_FUSION_INFERENCE = \t${MSLITE_ENABLE_CLOUD_FUSION_INFERENCE}")
message(STATUS "\tMSLITE_ENABLE_CAPTURE_SIGNALS = \t${MSLITE_ENABLE_CAPTURE_SIGNALS}")
message(STATUS "\tMSLITE_ENABLE_MODEL_OBF = \t${MSLITE_ENABLE_MODEL_OBF}")
if((MSLITE_ENABLE_CONVERTER OR MSLITE_ENABLE_TESTCASES) AND (
NOT MSLITE_ENABLE_MINDRT
@ -707,12 +712,12 @@ include_directories(${CORE_DIR}/mindrt/include)
include_directories(${CORE_DIR}/mindrt/src)
if(NOT WIN32 AND NOT APPLE)
if(ENABLE_MODEL_OBF)
if(MSLITE_ENABLE_MODEL_OBF)
add_compile_definitions(ENABLE_MODEL_OBF)
endif()
endif()
if(ENABLE_MODEL_OBF)
if(MSLITE_ENABLE_MODEL_OBF)
if(PLATFORM_ARM32)
set(OBF_LIB_DIR ${TOP_DIR}/mindspore/lite/tools/obfuscator/lib/android-aarch32)
elseif(PLATFORM_ARM64)

7
mindspore/lite/cmake/lite_options.cmake
View File

@ -49,10 +49,10 @@ option(MSLITE_ENABLE_CONVERT_PYTORCH_MODEL "enable to convert pytorch model" off
option(MSLITE_ENABLE_KERNEL_EXECUTOR "enable kernel executor" off)
option(MSLITE_ENABLE_GITEE_MIRROR "enable download third_party from gitee mirror" off)
option(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE "enable cloud and device fusion inference architecture" off)
option(MSLITE_ENABLE_MODEL_OBF "enable model obfuscation" off)
#Option that can be configured through manually
option(ENABLE_VERBOSE "" off)
option(ENABLE_MODEL_OBF "if support model obfuscation" off)
set(VERSION_STR "1.7.0" CACHE STRING "get from version")
if(MACHINE_LINUX_ARM64)
@ -197,6 +197,10 @@ if(DEFINED ENV{MSLITE_ENABLE_CLOUD_FUSION_INFERENCE})
set(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE $ENV{MSLITE_ENABLE_CLOUD_FUSION_INFERENCE})
endif()
if(DEFINED ENV{MSLITE_ENABLE_MODEL_OBF})
set(MSLITE_ENABLE_MODEL_OBF $ENV{MSLITE_ENABLE_MODEL_OBF})
endif()
if(TOOLCHAIN_NAME STREQUAL "himix200")
set(TARGET_HIMIX on)
set(TARGET_HIMIX200 on)
@ -394,3 +398,4 @@ message(STATUS "\tMSLITE_ENABLE_EXPERIMENTAL_KERNEL = \t${MSLITE_ENABLE
message(STATUS "\tMSLITE_ENABLE_GRAPH_KERNEL = \t${MSLITE_ENABLE_GRAPH_KERNEL}")
message(STATUS "\tMSLITE_ENABLE_KERNEL_EXECUTOR = \t${MSLITE_ENABLE_KERNEL_EXECUTOR}")
message(STATUS "\tMSLITE_ENABLE_CLOUD_FUSION_INFERENCE = \t${MSLITE_ENABLE_CLOUD_FUSION_INFERENCE}")
message(STATUS "\tMSLITE_ENABLE_MODEL_OBF = \t${MSLITE_ENABLE_MODEL_OBF}")

2
mindspore/lite/src/CMakeLists.txt
View File

@ -643,7 +643,7 @@ if(NOT WIN32)
target_link_libraries(mindspore-lite dl)
endif()
if(ENABLE_MODEL_OBF)
if(MSLITE_ENABLE_MODEL_OBF)
target_link_libraries(mindspore-lite ${OBF_LIB_DIR}/libmsdeobfuscator-lite.so)
target_link_libraries(mindspore-lite_static ${OBF_LIB_DIR}/libmsdeobfuscator-lite.so)
endif()

4
mindspore/lite/src/litert/lite_model.cc
View File

@ -348,8 +348,8 @@ int LiteModel::GenerateModelByVersion() {
if (schema_version_ == SCHEMA_VERSION::SCHEMA_CUR) {
#ifdef ENABLE_MODEL_OBF
if (IsMetaGraphObfuscated<schema::MetaGraph>(*reinterpret_cast<const schema::MetaGraph *>(meta_graph))) {
model_deobf =
GetModelDeObfuscator<schema::MetaGraph>(*reinterpret_cast<const schema::MetaGraph *>(meta_graph), this);
model_deobf = GetModelDeObfuscator<schema::MetaGraph>(*reinterpret_cast<const schema::MetaGraph *>(meta_graph),
this, this->buf_size_);
this->graph_.model_obfuscated_ = true;
if (model_deobf == nullptr) {
return RET_ERROR;

2
mindspore/lite/test/CMakeLists.txt
View File

@ -243,7 +243,7 @@ if(MSLITE_ENABLE_CONVERTER)
onnx_parser_mid tf_parser_mid)
endif()
if(ENABLE_MODEL_OBF)
if(MSLITE_ENABLE_MODEL_OBF)
target_link_libraries(lite-test ${OBF_LIB_DIR}/libmsdeobfuscator-lite.so)
endif()

2
mindspore/lite/tools/converter/CMakeLists.txt
View File

@ -390,7 +390,7 @@ if(NOT WIN32)
target_link_libraries(mindspore_converter dl)
endif()
if(ENABLE_MODEL_OBF)
if(MSLITE_ENABLE_MODEL_OBF)
target_link_libraries(mindspore_converter
${OBF_LIB_DIR}/libmsdeobfuscator-lite.so)
endif()

BIN
mindspore/lite/tools/obfuscator/bin/linux-x64/msobfuscator
View File

Binary file not shown.

42
mindspore/lite/tools/obfuscator/include/deobfuscator.h
View File

@ -14,37 +14,50 @@
* limitations under the License.
*/
#ifndef MINDSPORE_LITE_TOOLS_OBFUSCATOR_INCLUDE_DEOBFUSCATOR_H_
#define MINDSPORE_LITE_TOOLS_OBFUSCATOR_INCLUDE_DEOBFUSCATOR_H_
#ifndef MINDSPORE_LITE_TOOLS_OBFUSCATOR_INCLUDE_DEOBFUSCATOR_H
#define MINDSPORE_LITE_TOOLS_OBFUSCATOR_INCLUDE_DEOBFUSCATOR_H
#include <vector>
#include "src/common/common.h"
#include "src/common/log_adapter.h"
#include "include/model.h"
#include "schema/inner/model_generated.h"
#include "include/api/types.h"
#define IV_SIZE 16
namespace mindspore::lite {
struct DeObfuscator {
struct MS_API DeObfuscator {
std::vector<uint32_t> junk_tensor_Indices_;
std::vector<uint32_t> junk_node_Indices_;
std::vector<uint32_t> masking_values_;
std::vector<schema::PrimitiveType> all_prims_type_;
using PrimTypeVector = std::vector<schema::PrimitiveType>;
PrimTypeVector all_prims_type_;
unsigned char *obf_meta_data_;
uint32_t all_tensor_size_;
uint32_t all_node_size_;
uint32_t buf_size_;
bool with_sub_graph_;
void Free();
~DeObfuscator() { Free(); }
};
int DeObfuscateModel(Model *model, DeObfuscator *model_deobf);
bool DeObfuscatePrimitive(const schema::Primitive *src, uint32_t src_node_stat, unsigned char **dst_prim,
MS_API int DeObfuscateModel(Model *model, DeObfuscator *model_deobf);
MS_API bool DeObfuscatePrimitive(const schema::Primitive *src, uint32_t src_node_stat, unsigned char **dst_prim,
schema::PrimitiveType dst_type);
bool InitModelDeObfuscator(Model *model, DeObfuscator *model_deobf, const flatbuffers::Vector<uint8_t> *meta_data,
const flatbuffers::Vector<uint8_t> *decrypt_table, size_t node_num);
MS_API bool InitModelDeObfuscator(Model *model, DeObfuscator *model_deobf,
const flatbuffers::Vector<uint8_t> *meta_data, size_t node_num);
MS_API bool DeObfuscateTensors(Model *model, DeObfuscator *model_deobf);
MS_API bool DeObfuscateNodes(Model *model, DeObfuscator *model_deobf);
int NodeVerify(const Model *model);
int SubGraphVerify(const Model *model);
MS_API bool ModelVerify(const Model *model);
MS_API int DeObfuscateSubGraph(LiteGraph::SubGraph *subGraph, Model *model, DeObfuscator *model_deobf);
int DeObfuscateNodeIndex(LiteGraph::Node *node, DeObfuscator *model_deobf, uint32_t all_tensors_num);
int DeObfuscateIndex(uint32_t *orig_index, uint32_t modulus, DeObfuscator *model_deobf);
template <typename T = schema::MetaGraph>
bool IsMetaGraphObfuscated(const T &meta_graph) {
MS_API bool IsMetaGraphObfuscated(const T &meta_graph) {
if (meta_graph.obfMetaData() == nullptr) {
MS_LOG(INFO) << "obfMetaData is null.";
return false;
@ -53,14 +66,19 @@ bool IsMetaGraphObfuscated(const T &meta_graph) {
}
template <typename T = schema::MetaGraph>
DeObfuscator *GetModelDeObfuscator(const T &meta_graph, Model *model) {
MS_API DeObfuscator *GetModelDeObfuscator(const T &meta_graph, Model *model, size_t buf_size) {
if (meta_graph.obfMetaData() == nullptr || meta_graph.nodes() == nullptr || meta_graph.allTensors() == nullptr) {
MS_LOG(ERROR) << "invalid meta_graph!";
return nullptr;
}
auto meta_data = meta_graph.obfMetaData();
auto *model_deobfuscator = new (std::nothrow) DeObfuscator();
if (model_deobfuscator == nullptr) {
MS_LOG(ERROR) << "new model deobfuscator fail!";
return nullptr;
}
if (!InitModelDeObfuscator(model, model_deobfuscator, meta_data, nullptr, meta_graph.nodes()->size())) {
model_deobfuscator->buf_size_ = buf_size;
if (!InitModelDeObfuscator(model, model_deobfuscator, meta_data, meta_graph.nodes()->size())) {
MS_LOG(ERROR) << "init model deobfuscator fail!";
delete model_deobfuscator;
return nullptr;
@ -76,4 +94,4 @@ DeObfuscator *GetModelDeObfuscator(const T &meta_graph, Model *model) {
}
} // namespace mindspore::lite
#endif // MINDSPORE_LITE_TOOLS_OBFUSCATOR_INCLUDE_DEOBFUSCATOR_H_
#endif // MINDSPORE_LITE_TOOLS_OBFUSCATOR_INCLUDE_DEOBFUSCATOR_H

BIN
mindspore/lite/tools/obfuscator/lib/android-aarch64/libmsdeobfuscator-lite.so
View File

Binary file not shown.

BIN
mindspore/lite/tools/obfuscator/lib/linux-x64/libmsdeobfuscator-lite.so
View File

Binary file not shown.