remove profiling parameters in set_context function

This commit is contained in:
yanghaitao1 2021-09-02 22:31:14 -04:00
parent a4fb56ca48
commit 83302ad23c
22 changed files with 200 additions and 49 deletions

View File

@ -154,9 +154,9 @@ file(GLOB_RECURSE COMM_PROTO_IN RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "ps/core/pr
ms_protobuf_generate(COMM_PROTO_SRCS COMM_PROTO_HDRS ${COMM_PROTO_IN})
list(APPEND MINDSPORE_PROTO_LIST ${COMM_PROTO_SRCS})
include_directories("${CMAKE_BINARY_DIR}/profiler/device/common")
include_directories("${CMAKE_BINARY_DIR}/profiler/device/ascend")
file(GLOB_RECURSE PROFILER_PROTO_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"profiler/device/common/memory_profiling.proto")
"profiler/device/ascend/memory_profiling.proto")
ms_protobuf_generate_py(PROFILER_MEM_PROTO_PY PROFILER_MEM_PROTO_HDRS_PY PROFILER_MEM_PROTO_PYS ${PROFILER_PROTO_LIST})
list(APPEND MINDSPORE_PROTO_LIST ${PROFILER_MEM_PROTO_PY})

View File

@ -37,11 +37,11 @@
#include "debug/rdr/running_data_recorder.h"
#endif
#include "common/thread_pool.h"
#include "profiler/device/common/memory_profiling.h"
#include "profiler/device/ascend/memory_profiling.h"
using mindspore::profiler::MemoryProfiling;
using mindspore::profiler::NodeMemory;
using mindspore::profiler::TensorMemory;
using mindspore::profiler::ascend::MemoryProfiling;
using mindspore::profiler::ascend::NodeMemory;
using mindspore::profiler::ascend::TensorMemory;
namespace mindspore {
namespace somas {

View File

@ -81,12 +81,12 @@
#include "ps/ps_cache/ps_cache_manager.h"
#endif
#include "runtime/device/ascend/ascend_bucket.h"
#include "profiler/device/common/memory_profiling.h"
#include "profiler/device/ascend/memory_profiling.h"
#ifndef ENABLE_SECURITY
using mindspore::device::ascend::ProfilingManager;
using mindspore::profiler::ascend::MemoryProfiling;
#endif
using mindspore::profiler::MemoryProfiling;
namespace mindspore {
namespace session {

View File

@ -1363,11 +1363,14 @@ void ReleaseGeTsd() {
#ifndef ENABLE_SECURITY
void StartUpProfiling() {
auto ms_context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(ms_context);
if (!ms_context->get_param<bool>(MS_CTX_ENABLE_PROFILING)) {
#ifdef ENABLE_D
if (!ProfilingManager::GetInstance().IsProfiling()) {
return;
}
auto ms_context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(ms_context);
MS_LOG(INFO) << "Startup profiling";
// Start up profiling before OpenTsd
uint32_t device_id = ms_context->get_param<uint32_t>(MS_CTX_DEVICE_ID);
@ -1378,6 +1381,7 @@ void StartUpProfiling() {
MS_EXCEPTION_IF_NULL(runtime_instance);
runtime_instance->PreInit();
}
#endif
}
#endif

View File

@ -21,6 +21,7 @@
#include <string>
#include "utils/profile.h"
#include "utils/ms_context.h"
#include "profiler/device/profiling.h"
namespace mindspore {
void PynativeProfiler::SetEnableProfilingFlag() {
@ -28,9 +29,10 @@ void PynativeProfiler::SetEnableProfilingFlag() {
if (flag) {
return;
}
auto ms_context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(ms_context);
enable_profiler_flag_ = ms_context->get_param<bool>(MS_CTX_ENABLE_PROFILING);
auto profiler_manager = profiler::ProfilerManager::GetInstance();
MS_EXCEPTION_IF_NULL(profiler_manager);
enable_profiler_flag_ = profiler_manager->GetProfilingEnableFlag();
flag = true;
}

View File

@ -5,7 +5,7 @@ endif()
if(ENABLE_D)
file(GLOB_RECURSE PROFILER_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"device/common/*.cc" "device/cpu/*.cc")
"device/cpu/*.cc" "device/ascend/*.cc")
endif()
if(ENABLE_CPU AND NOT (ENABLE_D OR ENABLE_GPU))

View File

@ -0,0 +1,52 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "profiler/device/ascend/ascend_profiling.h"
#include <string>
#include "pybind_api/api_register.h"
#include "utils/log_adapter.h"
#include "utils/utils.h"
namespace mindspore {
namespace profiler {
namespace ascend {
std::shared_ptr<AscendProfiler> AscendProfiler::ascend_profiler_ = std::make_shared<AscendProfiler>();
std::shared_ptr<AscendProfiler> &AscendProfiler::GetInstance() { return ascend_profiler_; }
void AscendProfiler::StepProfilingEnable(const bool enable_flag) {
MS_LOG(INFO) << "Start profiling";
enable_flag_ = enable_flag;
}
void AscendProfiler::Start(const std::string &profiling_options) {
profiling_options_ = profiling_options;
StepProfilingEnable(true);
}
void AscendProfiler::Stop() {
MS_LOG(INFO) << "Stop profiling";
StepProfilingEnable(false);
}
REGISTER_PYBIND_DEFINE(AscendProfiler_, ([](const py::module *m) {
(void)py::class_<AscendProfiler, std::shared_ptr<AscendProfiler>>(*m, "AscendProfiler")
.def_static("get_instance", &AscendProfiler::GetInstance, "AscendProfiler get_instance.")
.def("start", &AscendProfiler::Start, py::arg("profiling_options"), "start")
.def("stop", &AscendProfiler::Stop, "stop");
}));
} // namespace ascend
} // namespace profiler
} // namespace mindspore

View File

@ -0,0 +1,49 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_PROFILER_DEVICE_ASCEND_PROFILING_H
#define MINDSPORE_CCSRC_PROFILER_DEVICE_ASCEND_PROFILING_H
#include <string>
#include <memory>
#include "profiler/device/profiling.h"
namespace mindspore {
namespace profiler {
namespace ascend {
class AscendProfiler : public Profiler {
public:
static std::shared_ptr<AscendProfiler> &GetInstance();
AscendProfiler() : profiling_options_("") {}
~AscendProfiler() = default;
AscendProfiler(const AscendProfiler &) = delete;
AscendProfiler &operator=(const AscendProfiler &) = delete;
void Init(const std::string &profileDataPath) { return; }
void Stop();
void StepProfilingEnable(const bool enable_flag) override;
void OpDataProducerEnd() { return; }
void Start(const std::string &profiling_options);
bool GetProfilingEnableFlag() const { return enable_flag_; }
std::string GetProfilingOptions() const { return profiling_options_; }
void SaveProfileData() { return; }
void ClearInst() { return; }
private:
static std::shared_ptr<AscendProfiler> ascend_profiler_;
std::string profiling_options_;
};
} // namespace ascend
} // namespace profiler
} // namespace mindspore
#endif

View File

@ -14,26 +14,28 @@
* limitations under the License.
*/
#include "profiler/device/common/memory_profiling.h"
#include "profiler/device/ascend/memory_profiling.h"
#include <fstream>
#include <memory>
#include "utils/log_adapter.h"
#include "utils/ms_context.h"
#include "utils/ms_utils.h"
#include "nlohmann/json.hpp"
#include "profiler/device/ascend/ascend_profiling.h"
namespace mindspore {
namespace profiler {
namespace ascend {
constexpr char kOutputPath[] = "output";
bool MemoryProfiling::IsMemoryProfilingEnable() const {
auto context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context);
if (!context->get_param<bool>(MS_CTX_ENABLE_PROFILING)) {
auto ascend_profiler = AscendProfiler::GetInstance();
MS_EXCEPTION_IF_NULL(ascend_profiler);
if (!ascend_profiler->GetProfilingEnableFlag()) {
return false;
}
const std::string prof_options_str = context->get_param<std::string>(MS_CTX_PROFILING_OPTIONS);
const std::string prof_options_str = ascend_profiler->GetProfilingOptions();
nlohmann::json options = nlohmann::json::parse(prof_options_str);
if (options["profile_memory"] == "off") {
return false;
@ -97,9 +99,9 @@ void MemoryProfiling::MemoryToPB() {
}
std::string MemoryProfiling::GetOutputPath() const {
auto context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context);
const std::string options_str = context->get_param<std::string>(MS_CTX_PROFILING_OPTIONS);
auto ascend_profiler = AscendProfiler::GetInstance();
MS_EXCEPTION_IF_NULL(ascend_profiler);
const std::string options_str = ascend_profiler->GetProfilingOptions();
nlohmann::json options_json;
try {
options_json = nlohmann::json::parse(options_str);
@ -152,5 +154,6 @@ void MemoryProfiling::SaveMemoryProfiling() {
MS_LOG(INFO) << "Start save memory profiling data to " << file << " end";
return;
}
} // namespace ascend
} // namespace profiler
} // namespace mindspore

View File

@ -26,6 +26,7 @@
namespace mindspore {
namespace profiler {
namespace ascend {
class NodeMemory {
public:
NodeMemory() : node_name_(""), node_id_(0) {}
@ -121,6 +122,7 @@ class MemoryProfiling {
std::map<uint32_t, std::shared_ptr<GraphMemory>> graph_memory_;
uint64_t device_mem_size_;
};
} // namespace ascend
} // namespace profiler
} // namespace mindspore
#endif

View File

@ -25,6 +25,9 @@
#if ENABLE_GPU
#include "profiler/device/gpu/gpu_profiling.h"
#endif
#if ENABLE_D
#include "profiler/device/ascend/ascend_profiling.h"
#endif
namespace mindspore {
namespace profiler {
@ -92,9 +95,14 @@ std::shared_ptr<ProfilerManager> &ProfilerManager::GetInstance() {
return profiler_manager_inst_;
}
bool ProfilerManager::GetEnableRecorderActorFlag() {
bool ProfilerManager::GetProfilingEnableFlag() {
#if ENABLE_GPU
return profiler::gpu::GPUProfiler::GetInstance()->GetEnableFlag();
#endif
#if ENABLE_D
auto ascend_instance = profiler::ascend::AscendProfiler::GetInstance();
MS_EXCEPTION_IF_NULL(ascend_instance);
return ascend_instance->GetProfilingEnableFlag();
#endif
return false;
}
@ -107,5 +115,14 @@ void ProfilerManager::RecordOneStepStartEndInfo() {
}
#endif
}
std::string ProfilerManager::GetProfilingOptions() const {
#if ENABLE_D
auto ascend_instance = profiler::ascend::AscendProfiler::GetInstance();
MS_EXCEPTION_IF_NULL(ascend_instance);
return ascend_instance->GetProfilingOptions();
#endif
return "";
}
} // namespace profiler
} // namespace mindspore

View File

@ -59,8 +59,9 @@ class ProfilerManager {
~ProfilerManager() = default;
ProfilerManager(const ProfilerManager &) = delete;
ProfilerManager &operator=(const ProfilerManager &) = delete;
bool GetEnableRecorderActorFlag();
bool GetProfilingEnableFlag();
void RecordOneStepStartEndInfo();
std::string GetProfilingOptions() const;
private:
static std::shared_ptr<ProfilerManager> profiler_manager_inst_;

View File

@ -66,6 +66,7 @@ using mindspore::dataset::TdtHandle;
#endif
#include "backend/session/pynative_task_manager.h"
#include "profiler/device/profiling.h"
#ifndef ENABLE_SECURITY
using mindspore::device::ascend::ProfilingManager;
@ -73,6 +74,7 @@ using mindspore::device::ascend::ProfilingUtils;
#endif
using mindspore::device::ascend::tasksink::TaskGenerator;
using mindspore::ge::model_runner::ModelRunner;
using HcclCollectiveGroup = mindspore::device::ascend::collective::HcclCollectiveGroup;
using mindspore::kernel::tbe::TbeUtils;
using std::vector;
@ -236,7 +238,7 @@ void AsyncDataDumpUninit() {
void AscendKernelRuntime::ReportProfilingData() {
auto context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context);
if (context->get_param<bool>(MS_CTX_ENABLE_PROFILING) &&
if (ProfilingManager::GetInstance().IsProfiling() &&
context->get_param<int>(MS_CTX_EXECUTION_MODE) == kPynativeMode) {
// Save Profiling Framework data
OpNameTaskStreamReporter reporter(device_id_, "nonsink", stream_id_task_id_op_name_map_);
@ -334,7 +336,11 @@ bool AscendKernelRuntime::Init() {
auto ms_context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(ms_context);
auto execution_mode = ms_context->get_param<int>(MS_CTX_EXECUTION_MODE);
auto profiling_flag = ms_context->get_param<bool>(MS_CTX_ENABLE_PROFILING);
auto profiler_manager = profiler::ProfilerManager::GetInstance();
MS_EXCEPTION_IF_NULL(profiler_manager);
auto profiling_flag = profiler_manager->GetProfilingEnableFlag();
if (execution_mode == kPynativeMode && profiling_flag) {
pynative_mode_profiling_flag_ = true;
}
@ -1164,11 +1170,12 @@ bool AscendKernelRuntime::CheckGraphIdValid(GraphId graph_id) const {
}
void AscendKernelRuntime::KernelLaunchProfiling(const std::string &kernel_name) {
auto context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context);
if (!context->get_param<bool>(MS_CTX_ENABLE_PROFILING)) {
auto profiler_manager = profiler::ProfilerManager::GetInstance();
MS_EXCEPTION_IF_NULL(profiler_manager);
if (!profiler_manager->GetProfilingEnableFlag()) {
return;
}
// save task info
uint32_t stream_id;
uint32_t task_id;

View File

@ -18,9 +18,11 @@
#include "runtime/device/ascend/ascend_memory_pool.h"
#include "utils/ms_context.h"
#include "runtime/mem.h"
#include "profiler/device/common/memory_profiling.h"
#include "runtime/device/ascend/profiling/profiling_manager.h"
#include "profiler/device/ascend/memory_profiling.h"
using mindspore::profiler::MemoryProfiling;
using mindspore::device::ascend::ProfilingManager;
using mindspore::profiler::ascend::MemoryProfiling;
namespace mindspore {
namespace device {

View File

@ -117,13 +117,13 @@ Status ProfilingManager::GetProfConf(const NotNull<MsprofGeOptions *> prof) {
return PROF_FAILED;
}
auto context = MsContext::GetInstance();
if (context == nullptr) {
MS_LOG(ERROR) << "Context is nullptr.";
auto profiler_manager = profiler::ProfilerManager::GetInstance();
if (profiler_manager == nullptr) {
MS_LOG(ERROR) << "Profiler manager instance is nullptr.";
return PROF_FAILED;
}
const string prof_options_str = profiler_manager->GetProfilingOptions();
const string prof_options_str = context->get_param<std::string>(MS_CTX_PROFILING_OPTIONS);
const nlohmann::json options_all = nlohmann::json::parse(prof_options_str);
nlohmann::json options_for_cann;
options_for_cann["output"] = options_all["output"];

View File

@ -28,9 +28,11 @@
#include "toolchain/slog.h"
#include "runtime/base.h"
#include "runtime/device/ascend/profiling/profiling_callback_register.h"
#include "profiler/device/profiling.h"
using std::map;
using std::string;
namespace mindspore {
namespace device {
namespace ascend {
@ -50,9 +52,9 @@ class ProfilingManager {
bool StopProfiling();
inline bool IsProfiling() const {
auto context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context);
return context->get_param<bool>(MS_CTX_ENABLE_PROFILING);
auto profiler_manager = profiler::ProfilerManager::GetInstance();
MS_EXCEPTION_IF_NULL(profiler_manager);
return profiler_manager->GetProfilingEnableFlag();
}
Status PluginInit() const;
void PluginUnInit() const;

View File

@ -27,6 +27,7 @@
#include "runtime/device/ascend/profiling/reporter/point_reporter.h"
#include "nlohmann/json.hpp"
#include "base/core_ops.h"
#include "profiler/device/profiling.h"
namespace mindspore {
namespace device {
@ -43,9 +44,9 @@ constexpr uint64_t kProfilingIterEndLogId = 4;
constexpr auto kDouble = 2;
nlohmann::json GetContextProfilingOption() {
auto context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context);
const string prof_options_str = context->get_param<std::string>(MS_CTX_PROFILING_OPTIONS);
auto profiler_manager = profiler::ProfilerManager::GetInstance();
MS_EXCEPTION_IF_NULL(profiler_manager);
const string prof_options_str = profiler_manager->GetProfilingOptions();
nlohmann::json j;
try {
j = nlohmann::json::parse(prof_options_str);

View File

@ -50,7 +50,7 @@ void RecorderActor::RecordOnStepEnd(OpContext<DeviceTensor> *const op_context) {
MS_EXCEPTION_IF_NULL(op_context);
// todo clear
// Record iter_start, fp_start and iter_end op name and timestamp at the step end. (GPU)
if (profiler::ProfilerManager::GetInstance()->GetEnableRecorderActorFlag()) {
if (profiler::ProfilerManager::GetInstance()->GetProfilingEnableFlag()) {
profiler::ProfilerManager::GetInstance()->RecordOneStepStartEndInfo();
}
}

View File

@ -31,6 +31,8 @@
#ifdef ENABLE_GE
#include "transform/graph_ir/df_graph_manager.h"
#endif
#include "profiler/device/profiling.h"
namespace py = pybind11;
namespace mindspore {
@ -162,9 +164,14 @@ void GetGeOptions(const std::shared_ptr<MsContext> &ms_context_ptr, std::map<std
(*ge_options)["ge.exec.dumpMode"] = "output";
MS_LOG(INFO) << "The enable dump state is " << std::to_string(ms_context_ptr->get_param<bool>(MS_CTX_ENABLE_DUMP))
<< " and save dump path is " << ms_context_ptr->get_param<std::string>(MS_CTX_SAVE_DUMP_PATH) << ".";
(*ge_options)["ge.exec.profilingMode"] = std::to_string(ms_context_ptr->get_param<bool>(MS_CTX_ENABLE_PROFILING));
if (ms_context_ptr->get_param<bool>(MS_CTX_ENABLE_PROFILING)) {
(*ge_options)["ge.exec.profilingOptions"] = ms_context_ptr->get_param<std::string>(MS_CTX_PROFILING_OPTIONS);
auto profiler_manager = profiler::ProfilerManager::GetInstance();
if (profiler_manager == nullptr) {
MS_LOG(EXCEPTION) << "Profiler manager is nullptr";
}
(*ge_options)["ge.exec.profilingMode"] = std::to_string(profiler_manager->GetProfilingEnableFlag());
if (profiler_manager->GetProfilingEnableFlag()) {
(*ge_options)["ge.exec.profilingOptions"] = profiler_manager->GetProfilingOptions();
}
(*ge_options)["rank_table_file"] = "";

View File

@ -156,7 +156,8 @@ class Profiler:
logger.error(msg)
raise ValueError(msg)
# use context interface to open profiling, for the new mindspore version(after 2020.5.21)
context.set_context(enable_profiling=True, profiling_options=profiling_options)
self._ascend_profiler = c_expression.AscendProfiler.get_instance()
self._ascend_profiler.start(profiling_options)
base_profiling_container_path = os.path.join(self._output_path, "container")
container_path = os.path.join(base_profiling_container_path, self._dev_id)
data_path = os.path.join(container_path, "data")
@ -334,7 +335,7 @@ class Profiler:
logger.warning(err.message)
os.environ['PROFILING_MODE'] = str("false")
context.set_context(enable_profiling=False)
self._ascend_profiler.stop()
def _gpu_analyse(self):
"""Collect and analyse gpu performance data"""

View File

@ -162,7 +162,8 @@ file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"../../../mindspore/ccsrc/transform/graph_ir/op_declare/*.cc"
"../../../mindspore/ccsrc/ps/*.cc"
"../../../mindspore/ccsrc/fl/*.cc"
"../../../mindspore/ccsrc/profiler/device/common/*.cc"
"../../../mindspore/ccsrc/profiler/device/ascend/*.cc"
"../../../mindspore/ccsrc/profiler/device/profiling.cc"
"../../../mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp32/adam_fp32.c"
)