p12346870
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!12360 Adjust log level for cache server/admin/perf 

From: @lixiachen
Reviewed-by: @robingrosman
Signed-off-by: @robingrosman
This commit is contained in:
mindspore-ci-bot 2021-02-17 08:58:16 +08:00 committed by Gitee
parent c5df707b69 389dec70d1
commit f9f24ca94d
11 changed files with 38 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt vendored
View File

@ -27,7 +27,7 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
endif()
endif()
if(NUMA_FOUND)
if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
ADD_DEFINITIONS(-DCACHE_LOCAL_CLIENT)
endif()

6
mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin.cc vendored
View File

@ -15,12 +15,10 @@
*/
#include <unistd.h>
#include <iostream>
#ifdef USE_GLOG
#include <glog/logging.h>
#endif
#include "minddata/dataset/engine/cache/cache_admin_arg.h"
#include "minddata/dataset/engine/cache/cache_common.h"
#include "minddata/dataset/util/path.h"
#include "mindspore/core/utils/log_adapter.h"
namespace ms = mindspore;
namespace ds = mindspore::dataset;
@ -31,7 +29,7 @@ int main(int argc, char **argv) {
std::stringstream arg_stream;
#ifdef USE_GLOG
FLAGS_minloglevel = google::WARNING;
FLAGS_logtostderr = false;
FLAGS_log_dir = ds::DefaultLogDir();
// Create default log dir
ds::Path log_dir = ds::Path(FLAGS_log_dir);

13
mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc vendored
View File

@ -63,6 +63,15 @@ CacheAdminArgHandler::CacheAdminArgHandler()
port_ = 0; // cause the port range validation to generate an error during the validation checks
}
}
const char *env_log_level = std::getenv("GLOG_v");
if (env_log_level != nullptr) {
char *end = nullptr;
log_level_ = strtol(env_log_level, &end, 10);
if (*end != '\0') {
std::cerr << "Log level from env variable GLOG_v is invalid\n";
log_level_ = -1; // cause the log level range validation to generate an error during the validation checks
}
}
// Initialize the command mappings
arg_map_["-h"] = ArgValue::kArgHost;
arg_map_["--hostname"] = ArgValue::kArgHost;
@ -333,7 +342,7 @@ Status CacheAdminArgHandler::Validate() {
if (num_workers_ < 1 || num_workers_ > max_num_workers)
return Status(StatusCode::kMDSyntaxError,
"Number of workers must be in range of 1 and " + std::to_string(max_num_workers) + ".");
if (log_level_ < 0 || log_level_ > 3) return Status(StatusCode::kMDSyntaxError, "Log level must be in range (0..3).");
if (log_level_ < 0 || log_level_ > 4) return Status(StatusCode::kMDSyntaxError, "Log level must be in range (0..4).");
if (memory_cap_ratio_ <= 0 || memory_cap_ratio_ > 1)
return Status(StatusCode::kMDSyntaxError, "Memory cap ratio should be positive and no greater than 1");
if (port_ < 1025 || port_ > 65535) return Status(StatusCode::kMDSyntaxError, "Port must be in range (1025..65535).");
@ -608,7 +617,7 @@ void CacheAdminArgHandler::Help() {
std::cerr << " [[-p | --port] <port number>] Default is " << kCfgDefaultCachePort << ".\n";
std::cerr << " [[-w | --workers] <number of workers>] Default is " << kDefaultNumWorkers << ".\n";
std::cerr << " [[-s | --spilldir] <spilling directory>] Default is no spilling.\n";
std::cerr << " [[-l | --loglevel] <log level>] Default is 1 (warning level).\n";
std::cerr << " [[-l | --loglevel] <log level>] Default is 1 (INFO level).\n";
std::cerr << " [--destroy_session | -d] <session id>\n";
std::cerr << " [[-p | --port] <port number>]\n";
std::cerr << " [--generate_session | -g]\n";

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_hw.cc vendored
View File

@ -32,7 +32,7 @@ CacheServerHW::CacheServerHW() {
MS_LOG(DEBUG) << "Number of cpu(s) : " << num_cpus_;
#ifdef NUMA_ENABLED
if (numa_enabled()) {
MS_LOG(WARNING) << "Numa support enabled";
MS_LOG(INFO) << "Numa support enabled";
for (auto i = 0; i <= numa_max_node(); ++i) {
int64_t free_avail;
int64_t mem_avail = numa_node_size(i, &free_avail);

13
mindspore/ccsrc/minddata/dataset/engine/cache/cache_main.cc vendored
View File

@ -17,14 +17,12 @@
#include "minddata/dataset/engine/cache/cache_server.h"
#include <sys/types.h>
#include <unistd.h>
#ifdef USE_GLOG
#include <glog/logging.h>
#endif
#include <cstdlib>
#include <thread>
#include <chrono>
#include "minddata/dataset/engine/cache/cache_common.h"
#include "minddata/dataset/engine/cache/cache_ipc.h"
#include "mindspore/core/utils/log_adapter.h"
namespace ms = mindspore;
namespace ds = mindspore::dataset;
@ -61,8 +59,7 @@ ms::Status StartServer(int argc, char **argv) {
ds::SharedMessage msg;
if (daemonize) {
#ifdef USE_GLOG
// temporary setting log level to WARNING to avoid logging when creating dir
FLAGS_minloglevel = google::WARNING;
FLAGS_logtostderr = false;
FLAGS_log_dir = ds::DefaultLogDir();
// Create cache server default log dir
ds::Path log_dir = ds::Path(FLAGS_log_dir);
@ -70,7 +67,7 @@ ms::Status StartServer(int argc, char **argv) {
if (rc.IsError()) {
return rc;
}
FLAGS_minloglevel = strtol(argv[5], nullptr, 10);
ms::g_ms_submodule_log_levels[SUBMODULE_ID] = strtol(argv[5], nullptr, 10);
google::InitGoogleLogging(argv[0]);
#endif
rc = msg.Create();
@ -120,8 +117,8 @@ ms::Status StartServer(int argc, char **argv) {
}
// Dump the summary
MS_LOG(WARNING) << "Cache server has started successfully and is listening on port " << port << std::endl;
MS_LOG(WARNING) << builder << std::endl;
MS_LOG(INFO) << "Cache server has started successfully and is listening on port " << port << std::endl;
MS_LOG(INFO) << builder << std::endl;
// Create the instance with some sanity checks built in
rc = builder.Build();
if (rc.IsOk()) {

4
mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc vendored
View File

@ -669,7 +669,7 @@ Status CacheServer::GetCacheMissKeys(CacheRequest *rq, CacheReply *reply) {
inline Status GenerateClientSessionID(session_id_type session_id, CacheReply *reply) {
reply->set_result(std::to_string(session_id));
MS_LOG(WARNING) << "Server generated new session id " << session_id;
MS_LOG(INFO) << "Server generated new session id " << session_id;
return Status::OK();
}
@ -1081,7 +1081,7 @@ Status CacheServer::DestroySession(CacheRequest *rq) {
// Finally remove the session itself
auto n = active_sessions_.erase(drop_session_id);
if (n > 0) {
MS_LOG(WARNING) << "Session destroyed with id " << drop_session_id;
MS_LOG(INFO) << "Session destroyed with id " << drop_session_id;
return Status::OK();
} else {
if (found) {

2
mindspore/ccsrc/minddata/dataset/engine/cache/perf/CMakeLists.txt vendored
View File

@ -8,6 +8,7 @@ if(ENABLE_CACHE)
target_link_libraries(cache_perf
_c_dataengine
_c_mindrecord
mindspore
mindspore::protobuf
mindspore_gvar
${PYTHON_LIBRARIES}
@ -21,6 +22,7 @@ if(ENABLE_CACHE)
target_link_libraries(cache_pipeline
_c_dataengine
_c_mindrecord
mindspore
mindspore::protobuf
mindspore_gvar
${PYTHON_LIBRARIES}

10
mindspore/ccsrc/minddata/dataset/engine/cache/perf/cache_perf.cc vendored
View File

@ -14,22 +14,22 @@
* limitations under the License.
*/
#ifdef USE_GLOG
#include <glog/logging.h>
#endif
#include <iostream>
#include "minddata/dataset/engine/cache/perf/cache_perf_run.h"
#include <iostream>
#include "mindspore/core/utils/log_adapter.h"
namespace ms = mindspore;
namespace ds = mindspore::dataset;
int main(int argc, char **argv) {
#ifdef USE_GLOG
FLAGS_logtostderr = false;
FLAGS_log_dir = "/tmp";
google::InitGoogleLogging(argv[0]);
#endif
ds::CachePerfRun cachePerfRun;
if (cachePerfRun.ProcessArgs(argc, argv) == 0) {
std::cout << cachePerfRun << std::endl;
ds::Status rc = cachePerfRun.Run();
ms::Status rc = cachePerfRun.Run();
if (rc.IsError()) {
std::cerr << rc.ToString() << std::endl;
}

2
mindspore/ccsrc/minddata/dataset/engine/cache/perf/cache_perf_run.cc vendored
View File

@ -548,7 +548,7 @@ Status CachePerfRun::Run() {
RETURN_IF_NOT_OK(cache_builder_.Build(&cc_));
Status rc = cc_->CreateCache(crc_, false);
// Duplicate key is fine.
if (rc.IsError() && rc.get_code() != StatusCode::kDuplicateKey) {
if (rc.IsError() && rc != StatusCode::kMDDuplicateKey) {
return rc;
}

12
mindspore/ccsrc/minddata/dataset/engine/cache/perf/cache_pipeline.cc vendored
View File

@ -14,22 +14,22 @@
* limitations under the License.
*/
#ifdef USE_GLOG
#include <glog/logging.h>
#endif
#include <string.h>
#include "minddata/dataset/engine/cache/perf/cache_pipeline_run.h"
#include <string.h>
#include "mindspore/core/utils/log_adapter.h"
namespace ms = mindspore;
namespace ds = mindspore::dataset;
int main(int argc, char **argv) {
#ifdef USE_GLOG
FLAGS_logtostderr = false;
FLAGS_log_dir = "/tmp";
FLAGS_minloglevel = google::WARNING;
google::InitGoogleLogging(argv[0]);
#endif
ds::CachePipelineRun cachePipelineRun;
if (cachePipelineRun.ProcessArgs(argc, argv) == 0) {
ds::Status rc = cachePipelineRun.Run();
ms::Status rc = cachePipelineRun.Run();
// If we hit any error, send the rc back to the parent.
if (rc.IsError()) {
ds::ErrorMsg proto;

4
mindspore/ccsrc/minddata/dataset/engine/cache/perf/cache_pipeline_run.cc vendored
View File

@ -177,7 +177,7 @@ Status CachePipelineRun::Run() {
Status rc = cc_->CreateCache(crc_, false);
// Duplicate key is fine.
if (rc.IsError() && rc.get_code() != StatusCode::kDuplicateKey) {
if (rc.IsError() && rc != StatusCode::kMDDuplicateKey) {
return rc;
}
@ -313,7 +313,7 @@ Status CachePipelineRun::WriterWorkerEntry(int32_t worker_id) {
rc = cc_->AsyncWriteBuffer(std::move(buffer));
auto end_tick = std::chrono::steady_clock::now();
if (rc.IsError()) {
if (rc.IsOutofMemory() || rc.IsNoSpace()) {
if (rc == StatusCode::kMDOutOfMemory || rc == StatusCode::kMDNoSpace) {
MS_LOG(WARNING) << "Pipeline number " << my_pipeline_ + 1 << " worker id " << worker_id << ": "
<< rc.ToString();
resource_err = true;