p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!953 add submodule id for log interface 

Merge pull request !953 from fary86/add_submodule_id_for_log_interface
This commit is contained in:
mindspore-ci-bot 2020-05-11 21:13:05 +08:00 committed by Gitee
parent 7068e708de 3f323e48e2
commit c72589aad6
35 changed files with 359 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mindspore/ccsrc/CMakeLists.txt
View File

@ -116,6 +116,7 @@ foreach (_comp ${SUB_COMP})
endif ()
endforeach ()
set_property(SOURCE ${SUB_OBJECTS_SRC} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_ME)
add_library(mindspore STATIC ${SUB_OBJECTS_SRC})
target_link_libraries(mindspore proto_input)
target_link_libraries(mindspore securec mindspore::flatbuffers)

1
mindspore/ccsrc/common/CMakeLists.txt
View File

@ -1,2 +1,3 @@
file(GLOB_RECURSE _COMMON_ALL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_COMMON_ALL_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_COMMON)
add_library(_mindspore_common_obj OBJECT ${_COMMON_ALL_SRC_FILES})

2
mindspore/ccsrc/dataset/api/CMakeLists.txt
View File

@ -1,3 +1,5 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(APItoPython OBJECT
de_pipeline.cc
python_bindings.cc

2
mindspore/ccsrc/dataset/core/CMakeLists.txt
View File

@ -1,3 +1,5 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(core OBJECT
client.cc
config_manager.cc

2
mindspore/ccsrc/dataset/engine/CMakeLists.txt
View File

@ -3,6 +3,8 @@ if (ENABLE_TDTQUE)
add_subdirectory(tdt)
endif ()
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(engine OBJECT
execution_tree.cc
data_buffer.cc

2
mindspore/ccsrc/dataset/engine/datasetops/CMakeLists.txt
View File

@ -1,5 +1,7 @@
add_subdirectory(source)
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(engine-datasetops OBJECT
dataset_op.cc
parallel_op.cc

2
mindspore/ccsrc/dataset/engine/datasetops/source/CMakeLists.txt
View File

@ -1,6 +1,8 @@
ms_protobuf_generate(EXAMPLE_SRCS EXAMPLE_HDRS example.proto)
ms_protobuf_generate(FEATURE_SRCS FEATURE_HDRS feature.proto)
add_subdirectory(sampler)
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(engine-datasetops-source OBJECT
generator_op.cc
io_block.cc

2
mindspore/ccsrc/dataset/engine/datasetops/source/sampler/CMakeLists.txt
View File

@ -1,3 +1,5 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(engine-datasetops-source-sampler OBJECT
distributed_sampler.cc
pk_sampler.cc

2
mindspore/ccsrc/dataset/engine/tdt/CMakeLists.txt
View File

@ -1,3 +1,5 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(engine-tdt OBJECT
tdt_plugin.cc
${EXAMPLE_SRCS}

2
mindspore/ccsrc/dataset/kernels/CMakeLists.txt
View File

@ -1,5 +1,7 @@
add_subdirectory(image)
add_subdirectory(data)
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(kernels OBJECT
py_func_op.cc
tensor_op.cc)

2
mindspore/ccsrc/dataset/kernels/data/CMakeLists.txt
View File

@ -1,3 +1,5 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(kernels-data OBJECT
data_utils.cc
one_hot_op.cc

2
mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt
View File

@ -1,3 +1,5 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(kernels-image OBJECT
center_crop_op.cc
cut_out_op.cc

2
mindspore/ccsrc/dataset/util/CMakeLists.txt
View File

@ -1,3 +1,5 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(utils OBJECT
arena.cc
circular_pool.cc

1
mindspore/ccsrc/debug/CMakeLists.txt
View File

@ -13,4 +13,5 @@ if (ENABLE_DUMP_E2E)
list(APPEND _DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/e2e_dump.cc")
endif (ENABLE_DUMP_E2E)
set_property(SOURCE ${_DEBUG_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_DEBUG)
add_library(_mindspore_debug_obj OBJECT ${_DEBUG_SRC_LIST})

1
mindspore/ccsrc/device/CMakeLists.txt
View File

@ -47,4 +47,5 @@ if (ENABLE_GPU)
# add_library(_mindspore_device_cuda_obj OBJECT ${CUDA_SRC_LIST})
endif ()
set_property(SOURCE ${DEVICE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_DEVICE)
add_library(_mindspore_device_obj OBJECT ${DEVICE_SRC_LIST} ${D_SRC_LIST} ${CPU_SRC_LIST})

1
mindspore/ccsrc/gvar/CMakeLists.txt
View File

@ -1,4 +1,5 @@
file(GLOB_RECURSE MS_GVAR_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cc)
set_property(SOURCE ${MS_GVAR_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_COMMON)
add_library(mindspore_gvar SHARED ${MS_GVAR_SRC_LIST})
if (APPLE)
set_target_properties(mindspore_gvar PROPERTIES MACOSX_RPATH ON)

1
mindspore/ccsrc/ir/CMakeLists.txt
View File

@ -1,2 +1,3 @@
file(GLOB_RECURSE _IR_SRC_LIST ./*.cc dtype/*.cc)
set_property(SOURCE ${_IR_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_IR)
add_library(_mindspore_ir_obj OBJECT ${_IR_SRC_LIST})

1
mindspore/ccsrc/kernel/CMakeLists.txt
View File

@ -43,4 +43,5 @@ if (ENABLE_GPU)
# add_library(_mindspore_kernel_cuda_obj OBJECT ${CUDA_SRC_LIST})
endif()
set_property(SOURCE ${KERNEL_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_KERNEL)
add_library(_mindspore_kernel_obj OBJECT ${KERNEL_SRC_LIST} ${CPU_SRC_LIST} ${GPU_SRC_LIST} ${D_SRC_LIST})

1
mindspore/ccsrc/mindrecord/CMakeLists.txt
View File

@ -18,6 +18,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default")
# add shared link library
set_property(SOURCE ${DIR_LIB_SRCS} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
add_library(_c_mindrecord SHARED ${DIR_LIB_SRCS})
set_target_properties(_c_mindrecord PROPERTIES

1
mindspore/ccsrc/onnx/CMakeLists.txt
View File

@ -1,2 +1,3 @@
file(GLOB_RECURSE _ONNX_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_ONNX_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_ONNX)
add_library(_mindspore_onnx_obj OBJECT ${_ONNX_SRC_FILES})

1
mindspore/ccsrc/operator/CMakeLists.txt
View File

@ -1,2 +1,3 @@
file(GLOB_RECURSE _OPERATOR_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_OPERATOR_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_ANALYZER)
add_library(_mindspore_operator_obj OBJECT ${_OPERATOR_SRC_FILES})

1
mindspore/ccsrc/optimizer/CMakeLists.txt
View File

@ -1,2 +1,3 @@
file(GLOB_RECURSE _OPTIMIZER_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_OPTIMIZER_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_OPTIMIZER)
add_library(_mindspore_optimizer_obj OBJECT ${_OPTIMIZER_SRC_FILES})

1
mindspore/ccsrc/parallel/CMakeLists.txt
View File

@ -3,4 +3,5 @@ if (ENABLE_DUMP_PROTO)
list(REMOVE_ITEM _PARALLEL_SRC_FILES "parallel/strategy_checkpoint/parallel_strategy_checkpoint.cc")
endif ()
set_property(SOURCE ${_PARALLEL_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PARALLEL)
add_library(_mindspore_parallel_obj OBJECT ${_PARALLEL_SRC_FILES})

10
mindspore/ccsrc/pipeline/CMakeLists.txt
View File

@ -9,6 +9,16 @@ file(GLOB_RECURSE _PIPELINE_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"static_analysis/*.cc"
)
file(GLOB PIPELINE_SRC_FILES "*.cc")
set_property(SOURCE ${PIPELINE_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PIPELINE)
file(GLOB_RECURSE PARSER_SRC_FILES "parse/*.cc")
set_property(SOURCE ${PARSER_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PARSER)
file(GLOB_RECURSE ANALYZER_SRC_FILES "static_analysis/*.cc")
set_property(SOURCE ${ANALYZER_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_ANALYZER)
if (ENABLE_GE OR ENABLE_D)
file(GLOB_RECURSE _PIPELINE_GE_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "pipeline_ge.cc")
list(APPEND _PIPELINE_SRC_FILES ${_PIPELINE_GE_SRC_FILES})

3
mindspore/ccsrc/pre_activate/CMakeLists.txt
View File

@ -10,4 +10,5 @@ if (ENABLE_D)
list(APPEND _PREACTIVATE_SRC_LIST ${_D_SRC_LIST})
endif ()
add_library(_mindspore_pre_activate_obj OBJECT ${_PREACTIVATE_SRC_LIST})
set_property(SOURCE ${_PREACTIVATE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PRE_ACT)
add_library(_mindspore_pre_activate_obj OBJECT ${_PREACTIVATE_SRC_LIST})

3
mindspore/ccsrc/pybind_api/CMakeLists.txt
View File

@ -1,2 +1,3 @@
file(GLOB_RECURSE _PYBIND_API_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
add_library(_mindspore_pybind_api_obj OBJECT ${_PYBIND_API_SRC_LIST})
set_property(SOURCE ${_PYBIND_API_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_COMMON)
add_library(_mindspore_pybind_api_obj OBJECT ${_PYBIND_API_SRC_LIST})

1
mindspore/ccsrc/pynative/CMakeLists.txt
View File

@ -5,4 +5,5 @@ if (ENABLE_GE)
list(APPEND _PYNATIVE_SRC_LIST ${_GE_SRC_LIST})
endif ()
set_property(SOURCE ${_PYNATIVE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PYNATIVE)
add_library(_mindspore_pynative_obj OBJECT ${_PYNATIVE_SRC_LIST})

3
mindspore/ccsrc/session/CMakeLists.txt
View File

@ -26,4 +26,5 @@ if (ENABLE_D)
list(APPEND _SESSION_SRC_LIST ${_D_SRC_LIST})
endif ()
add_library(_mindspore_session_obj OBJECT ${_SESSION_SRC_LIST})
set_property(SOURCE ${_SESSION_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_SESSION)
add_library(_mindspore_session_obj OBJECT ${_SESSION_SRC_LIST})

1
mindspore/ccsrc/transform/CMakeLists.txt
View File

@ -1,5 +1,6 @@
if (ENABLE_GE OR ENABLE_D)
file(GLOB_RECURSE _TRANSFORM_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_TRANSFORM_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_GE_ADPT)
add_library(_mindspore_transform_obj OBJECT ${_TRANSFORM_SRC_LIST})
if (NOT ENABLE_GE)

1
mindspore/ccsrc/utils/CMakeLists.txt
View File

@ -5,4 +5,5 @@ if (NOT ENABLE_GE)
list(REMOVE_ITEM _UTILS_SRC_LIST ${_UTILS_GE_SRC_FILES})
endif ()
set_property(SOURCE ${_UTILS_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_UTILS)
add_library(_mindspore_utils_obj OBJECT ${_UTILS_SRC_LIST})

2
mindspore/ccsrc/utils/contract.h
View File

@ -56,7 +56,7 @@ class Ensures : public EnsuresAccess<T, R> {
if (!R::Check(value_)) {
LogStream contract_stream;
contract_stream << "contract error: " << signatory.extra_info << R::Desc();
LogWriter(signatory.location_info, EXCEPTION, ArgumentError) ^ contract_stream;
LogWriter(signatory.location_info, EXCEPTION, SUBMODULE_ID, ArgumentError) ^ contract_stream;
}
}
template <class O, typename = std::enable_if_t<std::is_convertible_v<O, T>>>

264
mindspore/ccsrc/utils/log_adapter.cc
View File

@ -17,8 +17,8 @@
#include "utils/log_adapter.h"
#include <unistd.h>
#include <map>
#include "pybind11/pybind11.h"
#include "debug/trace.h"
// namespace to support utils module definition
@ -155,14 +155,44 @@ static std::string ExceptionTypeToString(ExceptionType type) {
return std::string(type_names[type]);
}
static const char *GetSubModuleName(SubModuleId module_id) {
static const char *sub_module_names[NUM_SUBMODUES] = {
"UNKNOWN", // SM_UNKNOWN
"ANALYZER", // SM_ANALYZER
"COMMON", // SM_COMMON
"DEBUG", // SM_DEBUG
"DEVICE", // SM_DEVICE
"GE_ADPT", // SM_GE_ADPT
"IR", // SM_IR
"KERNEL", // SM_KERNEL
"MD", // SM_MD
"ME", // SM_ME
"ONNX", // SM_ONNX
"OPTIMIZER", // SM_OPTIMIZER
"PARALLEL", // SM_PARALLEL
"PARSER", // SM_PARSER
"PIPELINE", // SM_PIPELINE
"PRE_ACT", // SM_PRE_ACT
"PYNATIVE", // SM_PYNATIVE
"SESSION", // SM_SESSION
"UTILS", // SM_UTILS
"VM" // SM_VM
};
return sub_module_names[module_id % NUM_SUBMODUES];
}
void LogWriter::OutputLog(const std::ostringstream &msg) const {
#ifdef USE_GLOG
auto submodule_name = GetSubModuleName(submodule_);
google::LogMessage("", 0, GetGlogLevel(log_level_)).stream()
<< "[" << GetLogLevel(log_level_) << "] ME(" << getpid() << "," << GetProcName() << "):" << GetTime() << " "
<< "[" << GetLogLevel(log_level_) << "] " << submodule_name << "(" << getpid() << "," << GetProcName()
<< "):" << GetTime() << " "
<< "[" << location_.file_ << ":" << location_.line_ << "] " << location_.func_ << "] " << msg.str() << std::endl;
#else
auto str_msg = msg.str();
Dlog(static_cast<int>(ME), GetSlogLevel(log_level_), "[%s:%d] %s] %s", location_.file_, location_.line_,
auto slog_module_id = (submodule_ == SM_MD ? MD : ME);
Dlog(static_cast<int>(slog_module_id), GetSlogLevel(log_level_), "[%s:%d] %s] %s", location_.file_, location_.line_,
location_.func_, str_msg.c_str());
#endif
}
@ -230,6 +260,231 @@ static void InitMsLogLevel() {
#endif
// set default log level to WARNING for all sub modules
int g_ms_submodule_log_levels[NUM_SUBMODUES] = {WARNING};
enum LogConfigToken {
INVALID, // indicate invalid token
LEFT_BRACE, // '{'
RIGHT_BRACE, // '}'
VARIABLE, // '[A-Za-z][A-Za-z0-9_]*'
NUMBER, // [0-9]+
COMMA, // ','
COLON, // ';'
EOS, // End Of String, '\0'
};
static inline bool IsAlpha(char ch) { return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z'); }
static inline bool IsDigit(char ch) { return ch >= '0' && ch <= '9'; }
class LogConfigLexer {
public:
explicit LogConfigLexer(const std::string &text) : buffer_(text) {
cur_idx_ = 0;
cur_token_ = LogConfigToken::INVALID;
}
~LogConfigLexer() = default;
// skip white space, and return the first char after white space
char SkipWhiteSpace() {
while (cur_idx_ < buffer_.size()) {
char ch = buffer_[cur_idx_];
if (ch == ' ' || ch == '\t') {
++cur_idx_;
continue;
}
return ch;
}
return '\0';
}
LogConfigToken GetNext(std::string *ptr) {
#ifdef DEBUG
std::string text;
auto tok = GetNextInner(&text);
MS_LOG(DEBUG) << "Got token " << tok << " with value [" << text << "]";
if (ptr != nullptr) {
*ptr = text;
}
return tok;
}
LogConfigToken GetNextInner(std::string *ptr) {
#endif
char ch = SkipWhiteSpace();
// clang-format off
static const std::map<char, LogConfigToken> single_char_map = {
{'{', LogConfigToken::LEFT_BRACE},
{'}', LogConfigToken::RIGHT_BRACE},
{',', LogConfigToken::COMMA},
{':', LogConfigToken::COLON},
{'\0', LogConfigToken::EOS},
};
// clang-format on
auto iter = single_char_map.find(ch);
if (iter != single_char_map.end()) {
if (ptr != nullptr) {
*ptr = std::string() + ch;
}
++cur_idx_;
return iter->second;
} else if (IsAlpha(ch)) {
std::ostringstream oss;
do {
oss << ch;
ch = buffer_[++cur_idx_];
} while (cur_idx_ < buffer_.size() && (IsAlpha(ch) || IsDigit(ch) || ch == '_'));
if (ptr != nullptr) {
*ptr = std::string(oss.str());
}
return LogConfigToken::VARIABLE;
} else if (IsDigit(ch)) {
std::ostringstream oss;
do {
oss << ch;
ch = buffer_[++cur_idx_];
} while (cur_idx_ < buffer_.size() && IsDigit(ch));
if (ptr != nullptr) {
*ptr = std::string(oss.str());
}
return LogConfigToken::NUMBER;
}
return LogConfigToken::INVALID;
}
private:
std::string buffer_;
size_t cur_idx_;
LogConfigToken cur_token_;
std::string cur_text_;
};
class LogConfigParser {
public:
explicit LogConfigParser(const std::string &cfg) : lexer(cfg) {}
bool Expect(LogConfigToken expected, LogConfigToken tok) {
if (expected != tok) {
MS_LOG(ERROR) << "Expect " << expected << ", but got " << tok;
return false;
}
return true;
}
// The text of config MS_SUBMODULE_LOG_v is in the form {submodule1:log_level1,submodule2:log_level2,...}.
// Valid values of log levels are: 0 - debug, 1 - info, 2 - warning, 3 - error
// e.g. MS_SUBMODULE_LOG_v={PARSER:0, ANALYZER:2, PIPELINE:1}
std::map<std::string, std::string> Parse() {
std::map<std::string, std::string> log_levels;
bool flag_error = false;
std::string text;
auto tok = lexer.GetNext(&text);
// empty string
if (tok == LogConfigToken::EOS) {
return log_levels;
}
if (!Expect(LogConfigToken::LEFT_BRACE, tok)) {
return log_levels;
}
do {
std::string key, val;
tok = lexer.GetNext(&key);
if (!Expect(LogConfigToken::VARIABLE, tok)) {
flag_error = true;
break;
}
tok = lexer.GetNext(&text);
if (!Expect(LogConfigToken::COLON, tok)) {
flag_error = true;
break;
}
tok = lexer.GetNext(&val);
if (!Expect(LogConfigToken::NUMBER, tok)) {
flag_error = true;
break;
}
log_levels[key] = val;
tok = lexer.GetNext(&text);
} while (tok == LogConfigToken::COMMA);
if (!flag_error && !Expect(LogConfigToken::RIGHT_BRACE, tok)) {
flag_error = true;
}
if (flag_error) {
log_levels.clear();
}
return log_levels;
}
private:
LogConfigLexer lexer;
};
bool ParseLogLevel(const std::string &str_level, MsLogLevel *ptr_level) {
if (str_level.size() == 1) {
int ch = str_level.c_str()[0];
ch = ch - '0'; // substract ASCII code of '0', which is 48
if (ch >= DEBUG && ch <= ERROR) {
if (ptr_level != nullptr) {
*ptr_level = static_cast<MsLogLevel>(ch);
}
return true;
}
}
return false;
}
static MsLogLevel GetGlobalLogLevel() {
#ifdef USE_GLOG
return static_cast<MsLogLevel>(FLAGS_v);
#else
return static_cast<MsLogLevel>(g_mslog_level);
#endif
}
void InitSubModulesLogLevel() {
// initialize submodule's log level using global
auto global_log_level = GetGlobalLogLevel();
for (int i = 0; i < NUM_SUBMODUES; ++i) {
g_ms_submodule_log_levels[i] = global_log_level;
}
// set submodule's log level
auto submodule = GetEnv("MS_SUBMODULE_LOG_v");
MS_LOG(INFO) << "MS_SUBMODULE_LOG_v=`" << submodule << "`";
LogConfigParser parser(submodule);
auto configs = parser.Parse();
for (const auto &cfg : configs) {
int mod_idx = -1;
for (int i = 0; i < NUM_SUBMODUES; ++i) {
if (cfg.first == GetSubModuleName(static_cast<SubModuleId>(i))) {
mod_idx = i;
break;
}
}
if (mod_idx < 0) {
MS_LOG(WARNING) << "Undefined module name " << cfg.first << ", ignore it";
continue;
}
MsLogLevel submodule_log_level;
if (!ParseLogLevel(cfg.second, &submodule_log_level)) {
MS_LOG(WARNING) << "Illegal log level value " << cfg.second << " for " << cfg.first << ", ignore it.";
continue;
}
g_ms_submodule_log_levels[mod_idx] = submodule_log_level;
}
}
} // namespace mindspore
extern "C" {
@ -253,6 +508,7 @@ void mindspore_log_init(void) {
if (mindspore::GetEnv("GLOG_v").empty()) {
FLAGS_v = mindspore::WARNING;
}
// set default log file mode to 0640
if (mindspore::GetEnv("GLOG_logfile_mode").empty()) {
FLAGS_logfile_mode = 0640;
@ -265,6 +521,8 @@ void mindspore_log_init(void) {
FLAGS_logtostderr = true;
MS_LOG(WARNING) << "`GLOG_log_dir` is not set, output log to screen.";
}
mindspore::InitSubModulesLogLevel();
#else
mindspore::InitMsLogLevel();
#endif

48
mindspore/ccsrc/utils/log_adapter.h
View File

@ -97,14 +97,44 @@ constexpr std::ostream &operator<<(std::ostream &stream, const T &value) {
enum MsLogLevel : int { DEBUG = 0, INFO, WARNING, ERROR, EXCEPTION };
enum SubModuleId : int {
SM_UNKNOWN = 0, // unknown submodule
SM_ANALYZER, // static analyzer
SM_COMMON, // common
SM_DEBUG, // debug
SM_DEVICE, // device
SM_GE_ADPT, // ge adapter
SM_IR, // IR
SM_KERNEL, // kernel
SM_MD, // MindData
SM_ME, // MindExpression
SM_ONNX, // ONNX
SM_OPTIMIZER, // optimzer
SM_PARALLEL, // parallel
SM_PARSER, // parser
SM_PIPELINE, // ME pipeline
SM_PRE_ACT, // pre-activate
SM_PYNATIVE, // PyNative
SM_SESSION, // session
SM_UTILS, // utils
SM_VM, // VM
NUM_SUBMODUES // number of submodules
};
#ifndef SUBMODULE_ID
#define SUBMODULE_ID mindspore::SubModuleId::SM_ME
#endif
#ifndef USE_GLOG
extern int g_mslog_level;
#endif
extern int g_ms_submodule_log_levels[] __attribute__((visibility("default")));
class LogWriter {
public:
LogWriter(const LocationInfo &location, MsLogLevel log_level, ExceptionType excp_type = NoExceptionType)
: location_(location), log_level_(log_level), exception_type_(excp_type) {}
LogWriter(const LocationInfo &location, MsLogLevel log_level, SubModuleId submodule,
ExceptionType excp_type = NoExceptionType)
: location_(location), log_level_(log_level), submodule_(submodule), exception_type_(excp_type) {}
~LogWriter() = default;
void operator<(const LogStream &stream) const noexcept __attribute__((visibility("default")));
@ -115,6 +145,7 @@ class LogWriter {
LocationInfo location_;
MsLogLevel log_level_;
SubModuleId submodule_;
ExceptionType exception_type_;
};
@ -122,16 +153,13 @@ class LogWriter {
static_cast<void>(0), !(condition) \
? void(0) \
: mindspore::LogWriter(mindspore::LocationInfo(FILE_NAME, __LINE__, __FUNCTION__), level, \
excp_type) < mindspore::LogStream()
#define MSLOG_THROW(excp_type) \
mindspore::LogWriter(mindspore::LocationInfo(FILE_NAME, __LINE__, __FUNCTION__), mindspore::EXCEPTION, excp_type) ^ \
SUBMODULE_ID, excp_type) < mindspore::LogStream()
#define MSLOG_THROW(excp_type) \
mindspore::LogWriter(mindspore::LocationInfo(FILE_NAME, __LINE__, __FUNCTION__), mindspore::EXCEPTION, SUBMODULE_ID, \
excp_type) ^ \
mindspore::LogStream()
#ifdef USE_GLOG
#define IS_OUTPUT_ON(level) (level) >= FLAGS_v
#else
#define IS_OUTPUT_ON(level) (level) >= mindspore::g_mslog_level
#endif
#define IS_OUTPUT_ON(level) (level) >= mindspore::g_ms_submodule_log_levels[SUBMODULE_ID]
#define MS_LOG(level) MS_LOG_##level

1
mindspore/ccsrc/vm/CMakeLists.txt
View File

@ -1,2 +1,3 @@
file(GLOB_RECURSE _VM_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_VM_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_VM)
add_library(_mindspore_vm_obj OBJECT ${_VM_SRC_LIST})

5
tests/ut/cpp/common/test_main.cc
View File

@ -19,7 +19,12 @@
#include "pipeline/pipeline.h"
#include "pipeline/resource.h"
namespace mindspore {
extern void InitSubModulesLogLevel();
}
GTEST_API_ int main(int argc, char** argv) {
mindspore::InitSubModulesLogLevel();
testing::InitGoogleTest(&argc, argv);
int ret = RUN_ALL_TESTS();
mindspore::pipeline::ClearResAtexit();