Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add security compilation options

This commit is contained in:
buxue 2021-07-20 16:20:00 +08:00
parent ed4c9682b5
commit 733f765063
9 changed files with 114 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
build.sh
View File

@ -46,6 +46,7 @@ usage()
echo " -i Enable increment building, default off"
echo " -j[n] Set the threads when building (Default: -j8)"
echo " -e Use cpu, gpu or ascend"
echo " -s Enable security, default off"
echo " -P Enable dump anf graph to file in ProtoBuffer format, default on"
echo " -D Enable dumping of function graph ir, default on"
echo " -z Compile dataset & mindrecord, default on"
@ -83,6 +84,7 @@ checkopts()
THREAD_NUM=8
DEBUG_MODE="off"
VERBOSE=""
ENABLE_SECURITY="off"
ENABLE_COVERAGE="off"
RUN_TESTCASES="off"
RUN_CPP_ST_TESTS="off"
@ -118,8 +120,10 @@ checkopts()
DEVICE=""
ENABLE_HIDDEN="on"
TENSORRT_HOME=""
USER_ENABLE_DUMP_IR=false
USER_ENABLE_DEBUGGER=false
# Process the options
while getopts 'drvj:c:t:hsb:a:g:p:ie:m:l:I:RP:D:zM:V:K:B:En:A:S:k:W:H:L:' opt
while getopts 'drvj:c:t:hb:s:a:g:p:ie:m:l:I:RP:D:zM:V:K:B:En:A:S:k:W:H:L:' opt
do
CASE_SENSIVE_ARG=${OPTARG}
OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')
@ -205,6 +209,25 @@ checkopts()
fi
TRAIN_MODE=$(echo "$OPTARG" | tr '[a-z]' '[A-Z]')
;;
s)
check_on_off $OPTARG s
if [[ "X$OPTARG" == "Xon" ]]; then
if [[ $USER_ENABLE_DUMP_IR == true ]]; then
echo "enable security, the dump ir is not available"
usage
exit 1
fi
if [[ $USER_ENABLE_DEBUGGER == true ]]; then
echo "enable security, the debugger is not available"
usage
exit 1
fi
ENABLE_DUMP_IR="off"
ENABLE_DEBUGGER="off"
fi
ENABLE_SECURITY="$OPTARG"
echo "enable security"
;;
R)
ENABLE_TIMELINE="on"
echo "enable time_line record"
@ -236,6 +259,14 @@ checkopts()
;;
D)
check_on_off $OPTARG D
if [[ "X$OPTARG" == "Xon" ]]; then
if [[ "X$ENABLE_SECURITY" == "Xon" ]]; then
echo "enable security, the dump ir is not available"
usage
exit 1
fi
USER_ENABLE_DUMP_IR=true
fi
ENABLE_DUMP_IR="$OPTARG"
echo "enable dump function graph ir"
;;
@ -271,6 +302,14 @@ checkopts()
;;
B)
check_on_off $OPTARG B
if [[ "X$OPTARG" == "Xon" ]]; then
if [[ "X$ENABLE_SECURITY" == "Xon" ]]; then
echo "enable security, the debugger is not available"
usage
exit 1
fi
USER_ENABLE_DEBUGGER=true
fi
ENABLE_DEBUGGER="$OPTARG"
;;
E)
@ -409,6 +448,9 @@ build_mindspore()
if [[ "X$ENABLE_PROFILE" = "Xon" ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_PROFILE=ON"
fi
if [[ "X$ENABLE_SECURITY" = "Xon" ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_SECURITY=ON"
fi
if [[ "X$ENABLE_TIMELINE" = "Xon" ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_TIMELINE=ON"
fi

5
cmake/options.cmake
View File

@ -11,6 +11,7 @@ option(ENABLE_ASAN "Enable Google Sanitizer to find memory bugs")
option(ENABLE_LOAD_ANF_IR "Enable load ANF-IR as input of 'infer' stage of pipeline" OFF)
option(ENABLE_COVERAGE "Enable code coverage report" OFF)
option(USE_GLOG "Use glog to output log" OFF)
option(ENABLE_SECURITY "Enable security, maintenance function will be disabled, default off" OFF)
option(ENABLE_PROFILE "Enable pipeline profile, default off" OFF)
option(ENABLE_TIMELINE "Enable time line record" OFF)
option(ENABLE_DUMP_PROTO "Enable dump anf graph to file in ProtoBuffer format, default on" ON)
@ -100,6 +101,10 @@ if(ENABLE_PROFILE)
add_compile_definitions(ENABLE_PROFILE)
endif()
if(ENABLE_SECURITY)
add_compile_definitions(ENABLE_SECURITY)
endif()
if(ENABLE_TIMELINE)
add_compile_definitions(ENABLE_TIMELINE)
endif()

2
mindspore/ccsrc/backend/session/cpu_session.cc
View File

@ -46,10 +46,12 @@
namespace mindspore {
namespace session {
void CPUSession::Init(uint32_t device_id) {
#ifndef ENABLE_SECURITY
// Dump json config file if dump is enabled
auto &json_parser = DumpJsonParser::GetInstance();
json_parser.Parse();
json_parser.CopyMSCfgJsonToDir(rank_id_);
#endif
InitExecutor(kCPUDevice, device_id);
}

21
mindspore/ccsrc/debug/CMakeLists.txt
View File

@ -9,10 +9,6 @@ set(_DEBUG_SRC_LIST
"${CMAKE_CURRENT_SOURCE_DIR}/trace.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/env_config_parser.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/dump_json_parser.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/cpu_e2e_dump.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/dump_utils.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/npy_header.cc"
)
set(_OFFLINE_SRC_LIST
@ -45,9 +41,20 @@ if(ENABLE_DEBUGGER)
"${CMAKE_CURRENT_SOURCE_DIR}/debugger/debugger_utils.cc"
)
endif()
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
list(APPEND _DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/common.cc")
list(APPEND _DEBUG_SRC_LIST "data_dump/e2e_dump.cc")
if(NOT ENABLE_SECURITY)
list(APPEND _DEBUG_SRC_LIST
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/cpu_e2e_dump.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/dump_json_parser.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/dump_utils.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/npy_header.cc"
)
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
list(APPEND _DEBUG_SRC_LIST
"${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/data_dump/e2e_dump.cc"
)
endif()
endif()
set_property(SOURCE ${_DEBUG_SRC_LIST} ${_RDR_SRC_LIST} PROPERTY COMPILE_DEFINITIONS

2
mindspore/ccsrc/runtime/device/cpu/cpu_device_address.cc
View File

@ -37,6 +37,7 @@ CPUDeviceAddress::~CPUDeviceAddress() {
bool CPUDeviceAddress::DumpMemToFile(const std::string &filepath, const std::string &, const ShapeVector &host_shape,
TypeId host_type, bool) const {
bool ret = false;
#ifndef ENABLE_SECURITY
if (filepath.empty()) {
MS_LOG(ERROR) << "Dump file path is null!";
return ret;
@ -44,6 +45,7 @@ bool CPUDeviceAddress::DumpMemToFile(const std::string &filepath, const std::str
std::string path = filepath + '.' + format_;
MS_LOG(DEBUG) << "E2E Dump path is " << path;
ret = DumpJsonParser::DumpToFile(path, ptr_, size_, host_shape, host_type);
#endif
return ret;
}

10
mindspore/ccsrc/runtime/device/cpu/cpu_kernel_runtime.cc
View File

@ -405,9 +405,11 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph, bool) {
auto profiler_inst = profiler::cpu::CPUProfiler::GetInstance();
MS_EXCEPTION_IF_NULL(profiler_inst);
#ifndef ENABLE_SECURITY
auto &dump_json_parser = DumpJsonParser::GetInstance();
bool iter_dump_flag = dump_json_parser.GetIterDumpFlag();
uint32_t graph_id = kernel_graph->graph_id();
#endif
for (const auto &kernel : kernels) {
#ifdef ENABLE_PROFILE
@ -448,9 +450,11 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph, bool) {
} catch (std::exception &e) {
MS_LOG(EXCEPTION) << e.what() << "\nTrace:" << trace::DumpSourceLines(kernel);
}
#ifndef ENABLE_SECURITY
if (iter_dump_flag) {
CPUE2eDump::DumpCNodeData(kernel, graph_id);
}
#endif
if (profiler_inst->GetEnableFlag()) {
profiler_inst->OpDataProducerEnd();
}
@ -466,10 +470,14 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph, bool) {
MS_LOG(INFO) << "cpu kernel: " << kernel->fullname_with_scope() << " costs " << cost_time * 1e6 << " us";
#endif
}
#ifndef ENABLE_SECURITY
if (iter_dump_flag) {
CPUE2eDump::DumpParametersAndConst(kernel_graph, graph_id);
}
dump_json_parser.UpdateDumpIter();
if (graph_id == 0) {
dump_json_parser.UpdateDumpIter();
}
#endif
return true;
}
} // namespace cpu

2
mindspore/ccsrc/runtime/hardware/cpu/cpu_device_context.cc
View File

@ -45,12 +45,14 @@ bool CPUDeviceContext::Initialize() {
mem_manager_ = std::make_shared<CPUMemoryManager>();
MS_EXCEPTION_IF_NULL(mem_manager_);
#ifndef ENABLE_SECURITY
// Dump json config file if dump is enabled.
auto rank_id = GetRankID();
auto &json_parser = DumpJsonParser::GetInstance();
json_parser.Parse();
json_parser.CopyJsonToDir(rank_id);
json_parser.CopyMSCfgJsonToDir(rank_id);
#endif
initialized_ = true;
return true;

35
mindspore/ccsrc/utils/security_py.cc Normal file
View File

@ -0,0 +1,35 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "utils/utils.h"
#include "pybind_api/api_register.h"
namespace mindspore {
// Get whether security is enable
bool EnableSecurity() {
#ifdef ENABLE_SECURITY
return true;
#else
return false;
#endif
}
// Define python wrapper to judge security enable.
REGISTER_PYBIND_DEFINE(security, ([](py::module *const m) {
auto m_sub = m->def_submodule("security", "submodule for security");
(void)m_sub.def("enable_security", &EnableSecurity, "enable security");
}));
} // namespace mindspore

2
mindspore/ccsrc/vm/backend.cc
View File

@ -676,6 +676,7 @@ void PrepareForDebuggr(const GraphCompilerInfo &graph_compiler_info) {
}
#endif
#ifndef ENABLE_SECURITY
if (DumpJsonParser::GetInstance().e2e_dump_enabled()) {
DumpJsonParser::GetInstance().ClearGraph();
for (size_t i = 0; i < graph_compiler_info.graphs_.size(); ++i) {
@ -684,6 +685,7 @@ void PrepareForDebuggr(const GraphCompilerInfo &graph_compiler_info) {
}
}
}
#endif
}
} // namespace