p19647583
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add submodule id for log interface

This commit is contained in:
fary86 2020-04-27 14:22:47 +08:00
parent 0edc6d254a
commit 3f323e48e2
35 changed files with 359 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mindspore/ccsrc/CMakeLists.txt
View File

@ -116,6 +116,7 @@ foreach (_comp ${SUB_COMP})
endif ()
endforeach ()
set_property(SOURCE ${SUB_OBJECTS_SRC} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_ME)
add_library(mindspore STATIC ${SUB_OBJECTS_SRC})
target_link_libraries(mindspore proto_input)
target_link_libraries(mindspore securec mindspore::flatbuffers)

1
mindspore/ccsrc/common/CMakeLists.txt
View File

@ -1,2 +1,3 @@
file(GLOB_RECURSE _COMMON_ALL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_COMMON_ALL_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_COMMON)
add_library(_mindspore_common_obj OBJECT ${_COMMON_ALL_SRC_FILES})

2
mindspore/ccsrc/dataset/api/CMakeLists.txt
View File

@ -1,3 +1,5 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(APItoPython OBJECT
de_pipeline.cc
python_bindings.cc

2
mindspore/ccsrc/dataset/core/CMakeLists.txt
View File

@ -1,3 +1,5 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(core OBJECT
client.cc
config_manager.cc

2
mindspore/ccsrc/dataset/engine/CMakeLists.txt
View File

@ -3,6 +3,8 @@ if (ENABLE_TDTQUE)
add_subdirectory(tdt)
endif ()
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(engine OBJECT
execution_tree.cc
data_buffer.cc

2
mindspore/ccsrc/dataset/engine/datasetops/CMakeLists.txt
View File

@ -1,5 +1,7 @@
add_subdirectory(source)
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(engine-datasetops OBJECT
dataset_op.cc
parallel_op.cc

2
mindspore/ccsrc/dataset/engine/datasetops/source/CMakeLists.txt
View File

@ -1,6 +1,8 @@
ms_protobuf_generate(EXAMPLE_SRCS EXAMPLE_HDRS example.proto)
ms_protobuf_generate(FEATURE_SRCS FEATURE_HDRS feature.proto)
add_subdirectory(sampler)
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(engine-datasetops-source OBJECT
generator_op.cc
io_block.cc

2
mindspore/ccsrc/dataset/engine/datasetops/source/sampler/CMakeLists.txt
View File

@ -1,3 +1,5 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(engine-datasetops-source-sampler OBJECT
distributed_sampler.cc
pk_sampler.cc

2
mindspore/ccsrc/dataset/engine/tdt/CMakeLists.txt
View File

@ -1,3 +1,5 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(engine-tdt OBJECT
tdt_plugin.cc
${EXAMPLE_SRCS}

2
mindspore/ccsrc/dataset/kernels/CMakeLists.txt
View File

@ -1,5 +1,7 @@
add_subdirectory(image)
add_subdirectory(data)
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(kernels OBJECT
py_func_op.cc
tensor_op.cc)

2
mindspore/ccsrc/dataset/kernels/data/CMakeLists.txt
View File

@ -1,3 +1,5 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(kernels-data OBJECT
data_utils.cc
one_hot_op.cc

2
mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt
View File

@ -1,3 +1,5 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(kernels-image OBJECT
center_crop_op.cc
cut_out_op.cc

2
mindspore/ccsrc/dataset/util/CMakeLists.txt
View File

@ -1,3 +1,5 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(utils OBJECT
arena.cc
circular_pool.cc

1
mindspore/ccsrc/debug/CMakeLists.txt
View File

@ -13,4 +13,5 @@ if (ENABLE_DUMP_E2E)
list(APPEND _DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/e2e_dump.cc")
endif (ENABLE_DUMP_E2E)
set_property(SOURCE ${_DEBUG_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_DEBUG)
add_library(_mindspore_debug_obj OBJECT ${_DEBUG_SRC_LIST})

1
mindspore/ccsrc/device/CMakeLists.txt
View File

@ -47,4 +47,5 @@ if (ENABLE_GPU)
# add_library(_mindspore_device_cuda_obj OBJECT ${CUDA_SRC_LIST})
endif ()
set_property(SOURCE ${DEVICE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_DEVICE)
add_library(_mindspore_device_obj OBJECT ${DEVICE_SRC_LIST} ${D_SRC_LIST} ${CPU_SRC_LIST})

1
mindspore/ccsrc/gvar/CMakeLists.txt
View File

@ -1,4 +1,5 @@
file(GLOB_RECURSE MS_GVAR_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cc)
set_property(SOURCE ${MS_GVAR_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_COMMON)
add_library(mindspore_gvar SHARED ${MS_GVAR_SRC_LIST})
if (APPLE)
set_target_properties(mindspore_gvar PROPERTIES MACOSX_RPATH ON)

1
mindspore/ccsrc/ir/CMakeLists.txt
View File

@ -1,2 +1,3 @@
file(GLOB_RECURSE _IR_SRC_LIST ./*.cc dtype/*.cc)
set_property(SOURCE ${_IR_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_IR)
add_library(_mindspore_ir_obj OBJECT ${_IR_SRC_LIST})

1
mindspore/ccsrc/kernel/CMakeLists.txt
View File

@ -43,4 +43,5 @@ if (ENABLE_GPU)
# add_library(_mindspore_kernel_cuda_obj OBJECT ${CUDA_SRC_LIST})
endif()
set_property(SOURCE ${KERNEL_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_KERNEL)
add_library(_mindspore_kernel_obj OBJECT ${KERNEL_SRC_LIST} ${CPU_SRC_LIST} ${GPU_SRC_LIST} ${D_SRC_LIST})

1
mindspore/ccsrc/mindrecord/CMakeLists.txt
View File

@ -18,6 +18,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default")
# add shared link library
set_property(SOURCE ${DIR_LIB_SRCS} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(_c_mindrecord SHARED ${DIR_LIB_SRCS})
set_target_properties(_c_mindrecord PROPERTIES

1
mindspore/ccsrc/onnx/CMakeLists.txt
View File

@ -1,2 +1,3 @@
file(GLOB_RECURSE _ONNX_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_ONNX_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_ONNX)
add_library(_mindspore_onnx_obj OBJECT ${_ONNX_SRC_FILES})

1
mindspore/ccsrc/operator/CMakeLists.txt
View File

@ -1,2 +1,3 @@
file(GLOB_RECURSE _OPERATOR_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_OPERATOR_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_ANALYZER)
add_library(_mindspore_operator_obj OBJECT ${_OPERATOR_SRC_FILES})

1
mindspore/ccsrc/optimizer/CMakeLists.txt
View File

@ -1,2 +1,3 @@
file(GLOB_RECURSE _OPTIMIZER_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_OPTIMIZER_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_OPTIMIZER)
add_library(_mindspore_optimizer_obj OBJECT ${_OPTIMIZER_SRC_FILES})

1
mindspore/ccsrc/parallel/CMakeLists.txt
View File

@ -3,4 +3,5 @@ if (ENABLE_DUMP_PROTO)
list(REMOVE_ITEM _PARALLEL_SRC_FILES "parallel/strategy_checkpoint/parallel_strategy_checkpoint.cc")
endif ()
set_property(SOURCE ${_PARALLEL_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PARALLEL)
add_library(_mindspore_parallel_obj OBJECT ${_PARALLEL_SRC_FILES})

10
mindspore/ccsrc/pipeline/CMakeLists.txt
View File

@ -9,6 +9,16 @@ file(GLOB_RECURSE _PIPELINE_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"static_analysis/*.cc"
)
file(GLOB PIPELINE_SRC_FILES "*.cc")
set_property(SOURCE ${PIPELINE_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PIPELINE)
file(GLOB_RECURSE PARSER_SRC_FILES "parse/*.cc")
set_property(SOURCE ${PARSER_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PARSER)
file(GLOB_RECURSE ANALYZER_SRC_FILES "static_analysis/*.cc")
set_property(SOURCE ${ANALYZER_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_ANALYZER)
if (ENABLE_GE OR ENABLE_D)
file(GLOB_RECURSE _PIPELINE_GE_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "pipeline_ge.cc")
list(APPEND _PIPELINE_SRC_FILES ${_PIPELINE_GE_SRC_FILES})

3
mindspore/ccsrc/pre_activate/CMakeLists.txt
View File

@ -10,4 +10,5 @@ if (ENABLE_D)
list(APPEND _PREACTIVATE_SRC_LIST ${_D_SRC_LIST})
endif ()
add_library(_mindspore_pre_activate_obj OBJECT ${_PREACTIVATE_SRC_LIST})
set_property(SOURCE ${_PREACTIVATE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PRE_ACT)
add_library(_mindspore_pre_activate_obj OBJECT ${_PREACTIVATE_SRC_LIST})

3
mindspore/ccsrc/pybind_api/CMakeLists.txt
View File

@ -1,2 +1,3 @@
file(GLOB_RECURSE _PYBIND_API_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
add_library(_mindspore_pybind_api_obj OBJECT ${_PYBIND_API_SRC_LIST})
set_property(SOURCE ${_PYBIND_API_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_COMMON)
add_library(_mindspore_pybind_api_obj OBJECT ${_PYBIND_API_SRC_LIST})

1
mindspore/ccsrc/pynative/CMakeLists.txt
View File

@ -5,4 +5,5 @@ if (ENABLE_GE)
list(APPEND _PYNATIVE_SRC_LIST ${_GE_SRC_LIST})
endif ()
set_property(SOURCE ${_PYNATIVE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PYNATIVE)
add_library(_mindspore_pynative_obj OBJECT ${_PYNATIVE_SRC_LIST})

3
mindspore/ccsrc/session/CMakeLists.txt
View File

@ -26,4 +26,5 @@ if (ENABLE_D)
list(APPEND _SESSION_SRC_LIST ${_D_SRC_LIST})
endif ()
add_library(_mindspore_session_obj OBJECT ${_SESSION_SRC_LIST})
set_property(SOURCE ${_SESSION_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_SESSION)
add_library(_mindspore_session_obj OBJECT ${_SESSION_SRC_LIST})

1
mindspore/ccsrc/transform/CMakeLists.txt
View File

@ -1,5 +1,6 @@
if (ENABLE_GE OR ENABLE_D)
file(GLOB_RECURSE _TRANSFORM_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_TRANSFORM_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_GE_ADPT)
add_library(_mindspore_transform_obj OBJECT ${_TRANSFORM_SRC_LIST})
if (NOT ENABLE_GE)

1
mindspore/ccsrc/utils/CMakeLists.txt
View File

@ -5,4 +5,5 @@ if (NOT ENABLE_GE)
list(REMOVE_ITEM _UTILS_SRC_LIST ${_UTILS_GE_SRC_FILES})
endif ()
set_property(SOURCE ${_UTILS_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_UTILS)
add_library(_mindspore_utils_obj OBJECT ${_UTILS_SRC_LIST})

2
mindspore/ccsrc/utils/contract.h
View File

@ -56,7 +56,7 @@ class Ensures : public EnsuresAccess<T, R> {
if (!R::Check(value_)) {
LogStream contract_stream;
contract_stream << "contract error: " << signatory.extra_info << R::Desc();
LogWriter(signatory.location_info, EXCEPTION, ArgumentError) ^ contract_stream;
LogWriter(signatory.location_info, EXCEPTION, SUBMODULE_ID, ArgumentError) ^ contract_stream;
}
}
template <class O, typename = std::enable_if_t<std::is_convertible_v<O, T>>>

264
mindspore/ccsrc/utils/log_adapter.cc
View File

@ -17,8 +17,8 @@
#include "utils/log_adapter.h"
#include <unistd.h>
#include <map>
#include "pybind11/pybind11.h"
#include "debug/trace.h"
// namespace to support utils module definition
@ -155,14 +155,44 @@ static std::string ExceptionTypeToString(ExceptionType type) {
return std::string(type_names[type]);
}
static const char *GetSubModuleName(SubModuleId module_id) {
static const char *sub_module_names[NUM_SUBMODUES] = {
"UNKNOWN", // SM_UNKNOWN
"ANALYZER", // SM_ANALYZER
"COMMON", // SM_COMMON
"DEBUG", // SM_DEBUG
"DEVICE", // SM_DEVICE
"GE_ADPT", // SM_GE_ADPT
"IR", // SM_IR
"KERNEL", // SM_KERNEL
"MD", // SM_MD
"ME", // SM_ME
"ONNX", // SM_ONNX
"OPTIMIZER", // SM_OPTIMIZER
"PARALLEL", // SM_PARALLEL
"PARSER", // SM_PARSER
"PIPELINE", // SM_PIPELINE
"PRE_ACT", // SM_PRE_ACT
"PYNATIVE", // SM_PYNATIVE
"SESSION", // SM_SESSION
"UTILS", // SM_UTILS
"VM" // SM_VM
};
return sub_module_names[module_id % NUM_SUBMODUES];
}
void LogWriter::OutputLog(const std::ostringstream &msg) const {
#ifdef USE_GLOG
auto submodule_name = GetSubModuleName(submodule_);
google::LogMessage("", 0, GetGlogLevel(log_level_)).stream()
<< "[" << GetLogLevel(log_level_) << "] ME(" << getpid() << "," << GetProcName() << "):" << GetTime() << " "
<< "[" << GetLogLevel(log_level_) << "] " << submodule_name << "(" << getpid() << "," << GetProcName()
<< "):" << GetTime() << " "
<< "[" << location_.file_ << ":" << location_.line_ << "] " << location_.func_ << "] " << msg.str() << std::endl;
#else
auto str_msg = msg.str();
Dlog(static_cast<int>(ME), GetSlogLevel(log_level_), "[%s:%d] %s] %s", location_.file_, location_.line_,
auto slog_module_id = (submodule_ == SM_MD ? MD : ME);
Dlog(static_cast<int>(slog_module_id), GetSlogLevel(log_level_), "[%s:%d] %s] %s", location_.file_, location_.line_,
location_.func_, str_msg.c_str());
#endif
}
@ -230,6 +260,231 @@ static void InitMsLogLevel() {
#endif
// set default log level to WARNING for all sub modules
int g_ms_submodule_log_levels[NUM_SUBMODUES] = {WARNING};
enum LogConfigToken {
INVALID, // indicate invalid token
LEFT_BRACE, // '{'
RIGHT_BRACE, // '}'
VARIABLE, // '[A-Za-z][A-Za-z0-9_]*'
NUMBER, // [0-9]+
COMMA, // ','
COLON, // ';'
EOS, // End Of String, '\0'
};
static inline bool IsAlpha(char ch) { return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z'); }
static inline bool IsDigit(char ch) { return ch >= '0' && ch <= '9'; }
class LogConfigLexer {
public:
explicit LogConfigLexer(const std::string &text) : buffer_(text) {
cur_idx_ = 0;
cur_token_ = LogConfigToken::INVALID;
}
~LogConfigLexer() = default;
// skip white space, and return the first char after white space
char SkipWhiteSpace() {
while (cur_idx_ < buffer_.size()) {
char ch = buffer_[cur_idx_];
if (ch == ' ' || ch == '\t') {
++cur_idx_;
continue;
}
return ch;
}
return '\0';
}
LogConfigToken GetNext(std::string *ptr) {
#ifdef DEBUG
std::string text;
auto tok = GetNextInner(&text);
MS_LOG(DEBUG) << "Got token " << tok << " with value [" << text << "]";
if (ptr != nullptr) {
*ptr = text;
}
return tok;
}
LogConfigToken GetNextInner(std::string *ptr) {
#endif
char ch = SkipWhiteSpace();
// clang-format off
static const std::map<char, LogConfigToken> single_char_map = {
{'{', LogConfigToken::LEFT_BRACE},
{'}', LogConfigToken::RIGHT_BRACE},
{',', LogConfigToken::COMMA},
{':', LogConfigToken::COLON},
{'\0', LogConfigToken::EOS},
};
// clang-format on
auto iter = single_char_map.find(ch);
if (iter != single_char_map.end()) {
if (ptr != nullptr) {
*ptr = std::string() + ch;
}
++cur_idx_;
return iter->second;
} else if (IsAlpha(ch)) {
std::ostringstream oss;
do {
oss << ch;
ch = buffer_[++cur_idx_];
} while (cur_idx_ < buffer_.size() && (IsAlpha(ch) || IsDigit(ch) || ch == '_'));
if (ptr != nullptr) {
*ptr = std::string(oss.str());
}
return LogConfigToken::VARIABLE;
} else if (IsDigit(ch)) {
std::ostringstream oss;
do {
oss << ch;
ch = buffer_[++cur_idx_];
} while (cur_idx_ < buffer_.size() && IsDigit(ch));
if (ptr != nullptr) {
*ptr = std::string(oss.str());
}
return LogConfigToken::NUMBER;
}
return LogConfigToken::INVALID;
}
private:
std::string buffer_;
size_t cur_idx_;
LogConfigToken cur_token_;
std::string cur_text_;
};
class LogConfigParser {
public:
explicit LogConfigParser(const std::string &cfg) : lexer(cfg) {}
bool Expect(LogConfigToken expected, LogConfigToken tok) {
if (expected != tok) {
MS_LOG(ERROR) << "Expect " << expected << ", but got " << tok;
return false;
}
return true;
}
// The text of config MS_SUBMODULE_LOG_v is in the form {submodule1:log_level1,submodule2:log_level2,...}.
// Valid values of log levels are: 0 - debug, 1 - info, 2 - warning, 3 - error
// e.g. MS_SUBMODULE_LOG_v={PARSER:0, ANALYZER:2, PIPELINE:1}
std::map<std::string, std::string> Parse() {
std::map<std::string, std::string> log_levels;
bool flag_error = false;
std::string text;
auto tok = lexer.GetNext(&text);
// empty string
if (tok == LogConfigToken::EOS) {
return log_levels;
}
if (!Expect(LogConfigToken::LEFT_BRACE, tok)) {
return log_levels;
}
do {
std::string key, val;
tok = lexer.GetNext(&key);
if (!Expect(LogConfigToken::VARIABLE, tok)) {
flag_error = true;
break;
}
tok = lexer.GetNext(&text);
if (!Expect(LogConfigToken::COLON, tok)) {
flag_error = true;
break;
}
tok = lexer.GetNext(&val);
if (!Expect(LogConfigToken::NUMBER, tok)) {
flag_error = true;
break;
}
log_levels[key] = val;
tok = lexer.GetNext(&text);
} while (tok == LogConfigToken::COMMA);
if (!flag_error && !Expect(LogConfigToken::RIGHT_BRACE, tok)) {
flag_error = true;
}
if (flag_error) {
log_levels.clear();
}
return log_levels;
}
private:
LogConfigLexer lexer;
};
bool ParseLogLevel(const std::string &str_level, MsLogLevel *ptr_level) {
if (str_level.size() == 1) {
int ch = str_level.c_str()[0];
ch = ch - '0'; // substract ASCII code of '0', which is 48
if (ch >= DEBUG && ch <= ERROR) {
if (ptr_level != nullptr) {
*ptr_level = static_cast<MsLogLevel>(ch);
}
return true;
}
}
return false;
}
static MsLogLevel GetGlobalLogLevel() {
#ifdef USE_GLOG
return static_cast<MsLogLevel>(FLAGS_v);
#else
return static_cast<MsLogLevel>(g_mslog_level);
#endif
}
void InitSubModulesLogLevel() {
// initialize submodule's log level using global
auto global_log_level = GetGlobalLogLevel();
for (int i = 0; i < NUM_SUBMODUES; ++i) {
g_ms_submodule_log_levels[i] = global_log_level;
}
// set submodule's log level
auto submodule = GetEnv("MS_SUBMODULE_LOG_v");
MS_LOG(INFO) << "MS_SUBMODULE_LOG_v=`" << submodule << "`";
LogConfigParser parser(submodule);
auto configs = parser.Parse();
for (const auto &cfg : configs) {
int mod_idx = -1;
for (int i = 0; i < NUM_SUBMODUES; ++i) {
if (cfg.first == GetSubModuleName(static_cast<SubModuleId>(i))) {
mod_idx = i;
break;
}
}
if (mod_idx < 0) {
MS_LOG(WARNING) << "Undefined module name " << cfg.first << ", ignore it";
continue;
}
MsLogLevel submodule_log_level;
if (!ParseLogLevel(cfg.second, &submodule_log_level)) {
MS_LOG(WARNING) << "Illegal log level value " << cfg.second << " for " << cfg.first << ", ignore it.";
continue;
}
g_ms_submodule_log_levels[mod_idx] = submodule_log_level;
}
}
} // namespace mindspore
extern "C" {
@ -253,6 +508,7 @@ void mindspore_log_init(void) {
if (mindspore::GetEnv("GLOG_v").empty()) {
FLAGS_v = mindspore::WARNING;
}
// set default log file mode to 0640
if (mindspore::GetEnv("GLOG_logfile_mode").empty()) {
FLAGS_logfile_mode = 0640;
@ -265,6 +521,8 @@ void mindspore_log_init(void) {
FLAGS_logtostderr = true;
MS_LOG(WARNING) << "`GLOG_log_dir` is not set, output log to screen.";
}
mindspore::InitSubModulesLogLevel();
#else
mindspore::InitMsLogLevel();
#endif

48
mindspore/ccsrc/utils/log_adapter.h
View File

@ -97,14 +97,44 @@ constexpr std::ostream &operator<<(std::ostream &stream, const T &value) {
enum MsLogLevel : int { DEBUG = 0, INFO, WARNING, ERROR, EXCEPTION };
enum SubModuleId : int {
SM_UNKNOWN = 0, // unknown submodule
SM_ANALYZER, // static analyzer
SM_COMMON, // common
SM_DEBUG, // debug
SM_DEVICE, // device
SM_GE_ADPT, // ge adapter
SM_IR, // IR
SM_KERNEL, // kernel
SM_MD, // MindData
SM_ME, // MindExpression
SM_ONNX, // ONNX
SM_OPTIMIZER, // optimzer
SM_PARALLEL, // parallel
SM_PARSER, // parser
SM_PIPELINE, // ME pipeline
SM_PRE_ACT, // pre-activate
SM_PYNATIVE, // PyNative
SM_SESSION, // session
SM_UTILS, // utils
SM_VM, // VM
NUM_SUBMODUES // number of submodules
};
#ifndef SUBMODULE_ID
#define SUBMODULE_ID mindspore::SubModuleId::SM_ME
#endif
#ifndef USE_GLOG
extern int g_mslog_level;
#endif
extern int g_ms_submodule_log_levels[] __attribute__((visibility("default")));
class LogWriter {
public:
LogWriter(const LocationInfo &location, MsLogLevel log_level, ExceptionType excp_type = NoExceptionType)
: location_(location), log_level_(log_level), exception_type_(excp_type) {}
LogWriter(const LocationInfo &location, MsLogLevel log_level, SubModuleId submodule,
ExceptionType excp_type = NoExceptionType)
: location_(location), log_level_(log_level), submodule_(submodule), exception_type_(excp_type) {}
~LogWriter() = default;
void operator<(const LogStream &stream) const noexcept __attribute__((visibility("default")));
@ -115,6 +145,7 @@ class LogWriter {
LocationInfo location_;
MsLogLevel log_level_;
SubModuleId submodule_;
ExceptionType exception_type_;
};
@ -122,16 +153,13 @@ class LogWriter {
static_cast<void>(0), !(condition) \
? void(0) \
: mindspore::LogWriter(mindspore::LocationInfo(FILE_NAME, __LINE__, __FUNCTION__), level, \
excp_type) < mindspore::LogStream()
#define MSLOG_THROW(excp_type) \
mindspore::LogWriter(mindspore::LocationInfo(FILE_NAME, __LINE__, __FUNCTION__), mindspore::EXCEPTION, excp_type) ^ \
SUBMODULE_ID, excp_type) < mindspore::LogStream()
#define MSLOG_THROW(excp_type) \
mindspore::LogWriter(mindspore::LocationInfo(FILE_NAME, __LINE__, __FUNCTION__), mindspore::EXCEPTION, SUBMODULE_ID, \
excp_type) ^ \
mindspore::LogStream()
#ifdef USE_GLOG
#define IS_OUTPUT_ON(level) (level) >= FLAGS_v
#else
#define IS_OUTPUT_ON(level) (level) >= mindspore::g_mslog_level
#endif
#define IS_OUTPUT_ON(level) (level) >= mindspore::g_ms_submodule_log_levels[SUBMODULE_ID]
#define MS_LOG(level) MS_LOG_##level

1
mindspore/ccsrc/vm/CMakeLists.txt
View File

@ -1,2 +1,3 @@
file(GLOB_RECURSE _VM_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_VM_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_VM)
add_library(_mindspore_vm_obj OBJECT ${_VM_SRC_LIST})

5
tests/ut/cpp/common/test_main.cc
View File

@ -19,7 +19,12 @@
#include "pipeline/pipeline.h"
#include "pipeline/resource.h"
namespace mindspore {
extern void InitSubModulesLogLevel();
}
GTEST_API_ int main(int argc, char** argv) {
mindspore::InitSubModulesLogLevel();
testing::InitGoogleTest(&argc, argv);
int ret = RUN_ALL_TESTS();
mindspore::pipeline::ClearResAtexit();