Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!49678 no rdr warning with infinite send 

Merge pull request !49678 from luoyang/no_warning_with_infinite_send
This commit is contained in:
i-robot 2023-03-06 02:15:28 +00:00 committed by Gitee
parent 6d271df3f8 cf0f287fcf
commit ad98167a6c
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
3 changed files with 26 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc
View File

@ -134,7 +134,7 @@ Status BatchOp::operator()() {
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) && ENABLE_PYTHON
if ((num_workers_ > 1 || batch_map_func_) && GetMemoryUsage() > MAX_MEMORY_USAGE_THRESHOLD) {
MS_LOG(WARNING) << "Memory consumption is more than " << (GetMemoryUsage() * 100) << "%, "
<< "which may cause oom error. Please reduce num_parallel_workers size / "
<< "which may cause OOM. Please reduce num_parallel_workers size / "
<< "optimize 'per_batch_map' function / other python data preprocess function to "
<< "reduce memory usage.";
}

24
mindspore/ccsrc/minddata/dataset/engine/datasetops/data_queue_op.cc
View File

@ -23,6 +23,7 @@
#include "minddata/dataset/engine/gpu_item_connector.h"
#include "minddata/dataset/engine/dataset_iterator.h"
#include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h"
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/util/task_manager.h"
#ifdef WITH_BACKEND
@ -88,8 +89,29 @@ DataQueueOp::DataQueueOp(const std::string channel_name, DeviceType device_type,
DataQueueOp::~DataQueueOp() {
#ifdef ENABLE_DUMP_IR
// BFS iter execution tree to get send epoch from EpochControl Op
std::vector<std::shared_ptr<DatasetOp>> child_node = this->Children();
size_t node_index = 0;
int32_t num_epochs = 0;
while (child_node.size() != 0 && node_index < child_node.size()) {
auto node = child_node[node_index];
if (node->Name() == kEpochCtrlOp) {
EpochCtrlOp *op = dynamic_cast<EpochCtrlOp *>(node.get());
if (op != nullptr) {
num_epochs = op->NumEpochs();
break;
}
}
auto child_child_node = node->Children();
if (!child_child_node.empty()) {
std::copy(child_child_node.begin(), child_child_node.end(), std::back_inserter(child_node));
}
++node_index;
}
// won't print rdr if call stop_send manually or send infinite epoch
std::string rdr_msg = md_channel_info_->ToString();
if (!send_finished_ && !rdr_msg.empty()) {
if (!send_finished_ && !rdr_msg.empty() && num_epochs != -1) {
MS_LOG(WARNING) << rdr_msg;
}
#endif

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.h
View File

@ -55,6 +55,8 @@ class EpochCtrlOp : public RepeatOp {
/// \return Status The status code returned
Status GetNextRowPullMode(TensorRow *const row) override;
int32_t NumEpochs() { return num_repeats_; }
protected:
/// \brief Gets the implementation status for operator in pull mode
/// \return implementation status