forked from mindspore-Ecosystem/mindspore
!49678 no rdr warning with infinite send
Merge pull request !49678 from luoyang/no_warning_with_infinite_send
This commit is contained in:
commit
ad98167a6c
|
@ -134,7 +134,7 @@ Status BatchOp::operator()() {
|
||||||
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) && ENABLE_PYTHON
|
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) && ENABLE_PYTHON
|
||||||
if ((num_workers_ > 1 || batch_map_func_) && GetMemoryUsage() > MAX_MEMORY_USAGE_THRESHOLD) {
|
if ((num_workers_ > 1 || batch_map_func_) && GetMemoryUsage() > MAX_MEMORY_USAGE_THRESHOLD) {
|
||||||
MS_LOG(WARNING) << "Memory consumption is more than " << (GetMemoryUsage() * 100) << "%, "
|
MS_LOG(WARNING) << "Memory consumption is more than " << (GetMemoryUsage() * 100) << "%, "
|
||||||
<< "which may cause oom error. Please reduce num_parallel_workers size / "
|
<< "which may cause OOM. Please reduce num_parallel_workers size / "
|
||||||
<< "optimize 'per_batch_map' function / other python data preprocess function to "
|
<< "optimize 'per_batch_map' function / other python data preprocess function to "
|
||||||
<< "reduce memory usage.";
|
<< "reduce memory usage.";
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,6 +23,7 @@
|
||||||
|
|
||||||
#include "minddata/dataset/engine/gpu_item_connector.h"
|
#include "minddata/dataset/engine/gpu_item_connector.h"
|
||||||
#include "minddata/dataset/engine/dataset_iterator.h"
|
#include "minddata/dataset/engine/dataset_iterator.h"
|
||||||
|
#include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h"
|
||||||
#include "minddata/dataset/util/status.h"
|
#include "minddata/dataset/util/status.h"
|
||||||
#include "minddata/dataset/util/task_manager.h"
|
#include "minddata/dataset/util/task_manager.h"
|
||||||
#ifdef WITH_BACKEND
|
#ifdef WITH_BACKEND
|
||||||
|
@ -88,8 +89,29 @@ DataQueueOp::DataQueueOp(const std::string channel_name, DeviceType device_type,
|
||||||
|
|
||||||
DataQueueOp::~DataQueueOp() {
|
DataQueueOp::~DataQueueOp() {
|
||||||
#ifdef ENABLE_DUMP_IR
|
#ifdef ENABLE_DUMP_IR
|
||||||
|
// BFS iter execution tree to get send epoch from EpochControl Op
|
||||||
|
std::vector<std::shared_ptr<DatasetOp>> child_node = this->Children();
|
||||||
|
size_t node_index = 0;
|
||||||
|
int32_t num_epochs = 0;
|
||||||
|
while (child_node.size() != 0 && node_index < child_node.size()) {
|
||||||
|
auto node = child_node[node_index];
|
||||||
|
if (node->Name() == kEpochCtrlOp) {
|
||||||
|
EpochCtrlOp *op = dynamic_cast<EpochCtrlOp *>(node.get());
|
||||||
|
if (op != nullptr) {
|
||||||
|
num_epochs = op->NumEpochs();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
auto child_child_node = node->Children();
|
||||||
|
if (!child_child_node.empty()) {
|
||||||
|
std::copy(child_child_node.begin(), child_child_node.end(), std::back_inserter(child_node));
|
||||||
|
}
|
||||||
|
++node_index;
|
||||||
|
}
|
||||||
|
|
||||||
|
// won't print rdr if call stop_send manually or send infinite epoch
|
||||||
std::string rdr_msg = md_channel_info_->ToString();
|
std::string rdr_msg = md_channel_info_->ToString();
|
||||||
if (!send_finished_ && !rdr_msg.empty()) {
|
if (!send_finished_ && !rdr_msg.empty() && num_epochs != -1) {
|
||||||
MS_LOG(WARNING) << rdr_msg;
|
MS_LOG(WARNING) << rdr_msg;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -55,6 +55,8 @@ class EpochCtrlOp : public RepeatOp {
|
||||||
/// \return Status The status code returned
|
/// \return Status The status code returned
|
||||||
Status GetNextRowPullMode(TensorRow *const row) override;
|
Status GetNextRowPullMode(TensorRow *const row) override;
|
||||||
|
|
||||||
|
int32_t NumEpochs() { return num_repeats_; }
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
/// \brief Gets the implementation status for operator in pull mode
|
/// \brief Gets the implementation status for operator in pull mode
|
||||||
/// \return implementation status
|
/// \return implementation status
|
||||||
|
|
Loading…
Reference in New Issue