p83571920
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!11909 Remove unused ExecTree optimizer code 

From: @nsyca
Reviewed-by: 
Signed-off-by:
This commit is contained in:
mindspore-ci-bot 2021-02-04 20:20:05 +08:00 committed by Gitee
parent 3bf5681ebc 46483d89b1
commit 7f709d9528
79 changed files with 303 additions and 1449 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,7 +16,6 @@
#include "minddata/dataset/engine/datasetops/batch_op.h"
#include <utility>
#include <iomanip>
#include "utils/ms_utils.h"
#ifdef ENABLE_PYTHON
@ -24,7 +23,6 @@
#endif
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/kernels/data/data_utils.h"
#include "minddata/dataset/util/status.h"
@ -498,18 +496,6 @@ Status BatchOp::UnpackPadInfo(const PadInfo &pad_info,
return Status::OK();
}
// Visitor accept method for NodePass
Status BatchOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<BatchOp>(), modified);
}
// Visitor pre-accept method for NodePass
Status BatchOp::PreAccept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->PreRunOnNode(shared_from_base<BatchOp>(), modified);
}
Status BatchOp::ComputeColMap() {
CHECK_FAIL_RETURN_UNEXPECTED(child_.size() == 1,
"Batch has " + std::to_string(child_.size()) + " child/children, expects only 1 child.");

14
mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -193,18 +193,6 @@ class BatchOp : public ParallelOp {
// @return Status The status code returned
Status operator()() override;
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status PreAccept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return kBatchOp; }

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/bucket_batch_by_length_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -15,22 +15,15 @@
*/
#include "minddata/dataset/engine/datasetops/bucket_batch_by_length_op.h"
#include <map>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "pybind11/numpy.h"
#include "pybind11/pybind11.h"
#include "pybind11/stl.h"
#include "minddata/dataset/core/pybind_support.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/engine/dataset_iterator.h"
#include "minddata/dataset/engine/datasetops/parallel_op.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/util/status.h"
namespace py = pybind11;

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -18,7 +18,6 @@
#include <iomanip>
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/engine/opt/pass.h"
namespace mindspore {
namespace dataset {
@ -160,12 +159,6 @@ void BuildSentencePieceVocabOp::Next(std::string *sentence) {
ret_status_ = Status::OK();
}
// Pre-Visitor accept method for NodePass
Status BuildSentencePieceVocabOp::PreAccept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call the pre-visitation
return p->PreRunOnNode(shared_from_base<BuildSentencePieceVocabOp>(), modified);
}
Status BuildSentencePieceVocabOp::Builder::Build(std::shared_ptr<BuildSentencePieceVocabOp> *op) {
(*op) = std::make_shared<BuildSentencePieceVocabOp>(builder_vocab_, builder_col_names_, builder_vocab_size_,
builder_character_coverage_, builder_model_type_, builder_params_,

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -63,7 +63,7 @@ class BuildSentencePieceVocabOp : public PipelineOp {
}
// Setter method
// @param float charactor corverage - to determine the minimum symbols
// @param float character corverage - to determine the minimum symbols
// @return Builder & reference to builder class object
Builder &SetCharacterCoverage(float character_coverage) {
builder_character_coverage_ = character_coverage;
@ -166,11 +166,6 @@ class BuildSentencePieceVocabOp : public PipelineOp {
bool Done();
void Next(std::string *sentence);
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status PreAccept(NodePass *p, bool *const modified) override;
private:
bool read_done_;
Status ret_status_;

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -23,7 +23,6 @@
#include <unordered_map>
#include <utility>
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/engine/opt/pass.h"
namespace mindspore {
namespace dataset {
@ -230,11 +229,5 @@ void BuildVocabOp::Print(std::ostream &out, bool show_all) const {
<< "\n\n";
}
}
// Pre-Visitor accept method for NodePass
Status BuildVocabOp::PreAccept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call the pre-visitation
return p->PreRunOnNode(shared_from_base<BuildVocabOp>(), modified);
}
} // namespace dataset
} // namespace mindspore

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -168,12 +168,6 @@ class BuildVocabOp : public ParallelOp {
Status Reset() override { RETURN_STATUS_UNEXPECTED("Reset shouldn't be called in BuildVocabOp"); }
/// \brief Base-class override for NodePass pre-visit acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status PreAccept(NodePass *p, bool *const modified) override;
private:
const int32_t interval_;
bool special_first_;

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -14,7 +14,6 @@
* limitations under the License.
*/
#include "minddata/dataset/engine/datasetops/cache_lookup_op.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/global_context.h"
@ -133,11 +132,5 @@ Status CacheLookupOp::ComputeColMap() {
// So we will just return OK and let MergeOp (our parent) to handle it.
return Status::OK();
}
// Visitor accept method for NodePass
Status CacheLookupOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<CacheLookupOp>(), modified);
}
} // namespace dataset
} // namespace mindspore

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -103,12 +103,6 @@ class CacheLookupOp : public CacheBase, public SamplerRT {
bool AllowCacheMiss() override { return true; }
std::string Name() const override { return kCacheLookupOp; }
/// \brief Base-class override for NodePass visitor acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status Accept(NodePass *p, bool *const modified) override;
protected:
Status ComputeColMap() override;

21
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -23,7 +23,6 @@
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/util/system_pool.h"
#include "minddata/dataset/util/task_manager.h"
@ -205,10 +204,10 @@ Status CacheMergeOp::Cleaner() {
return Status::OK();
}
Status CacheMergeOp::PrepareNodePostAction() { // Run any common code from super class first before adding our own
// specific logic
Status CacheMergeOp::PrepareOperator() { // Run any common code from super class first before adding our own
// specific logic
CHECK_FAIL_RETURN_UNEXPECTED(child_.size() == 2, "Incorrect number of children");
RETURN_IF_NOT_OK(ParallelOp::PrepareNodePostAction());
RETURN_IF_NOT_OK(DatasetOp::PrepareOperator());
// Get the computed check sum from all ops in the cache miss class
uint32_t cache_crc = DatasetOp::GenerateCRC(child_[kCacheMissChildIdx]);
// This is a mappable cache op so the id's need to be generated.
@ -263,18 +262,6 @@ Status CacheMergeOp::Builder::Build(std::shared_ptr<CacheMergeOp> *ptr) {
return Status::OK();
}
// Pre-Visitor accept method for NodePass
Status CacheMergeOp::PreAccept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call the pre-visitation
return p->PreRunOnNode(shared_from_base<CacheMergeOp>(), modified);
}
// Visitor accept method for NodePass
Status CacheMergeOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<CacheMergeOp>(), modified);
}
Status CacheMergeOp::EoeReceived(int32_t worker_id) {
// Send the eoe up.
MS_LOG(DEBUG) << "Cache merge sending eoe";

25
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -144,7 +144,7 @@ class CacheMergeOp : public ParallelOp {
/// \param numWorkers Number of parallel workers as a derived class of ParallelOp
/// \param opConnector Size Connector size as a derived class of ParallelOp
/// \param numCleaners Number of cleaners to move cache miss rows into the cache server
/// \param cache_client CacheClient to commmunicate with the Cache server
/// \param cache_client CacheClient to communicate with the Cache server
/// \param sampler as a derived class of ParallelOp
CacheMergeOp(int32_t numWorkers, int32_t opConnectorSize, int32_t numCleaners,
std::shared_ptr<CacheClient> cache_client, const std::shared_ptr<SamplerRT> &sampler);
@ -157,31 +157,24 @@ class CacheMergeOp : public ParallelOp {
return out;
}
/// \brief Master thread responsible to spawn all the necessary worker threads for the two streams and
/// the threads for the cleaners.
/// the threads for the cleaners.
/// \return
Status operator()() override;
/// \brief Entry function for worker thread that fetch rows from CacheLookupOp
/// \param workerId
/// \return Status object
Status WorkerEntry(int32_t workerId) override;
Status PrepareNodePostAction() override;
/// \brief Perform specific post-operations on CacheOp
/// \return Status The status code returned
Status PrepareOperator() override;
/// \brief Entry function for worker thread that fetch rows from the cache miss stream
/// \param workerId
/// \return Status object
Status CacheMissWorkerEntry(int32_t workerId);
/// \brief Base-class override for NodePass pre-visit acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status PreAccept(NodePass *p, bool *const modified) override;
/// \brief Base-class override for NodePass visitor acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status Accept(NodePass *p, bool *const modified) override;
/// \brief Base-class override for eoe handling
/// \param worker_id
/// \return Status object

23
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,14 +17,12 @@
#include <memory>
#include <utility>
#include <vector>
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/engine/datasetops/repeat_op.h"
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/util/log_adapter.h"
#include "minddata/dataset/util/task_manager.h"
@ -207,9 +205,6 @@ Status CacheOp::RegisterResources() {
return Status::OK();
}
// Base-class override for setting specific CacheOp configurations. This code will be called
// during the execution tree prepare phase BEFORE traversing down to child operators.
uint32_t CacheOp::PrepareFlags() const { return ExecutionTree::kDePrepCache; }
// Base-class override for special eoe handler.
// CacheOp must override this because it shall not perform default handling of eoe. Instead
// the CacheOp manages actions related to the end of the epoch.
@ -225,21 +220,9 @@ Status CacheOp::EofReceived(int32_t worker_id) {
return Status::OK();
}
// Pre-Visitor accept method for NodePass
Status CacheOp::PreAccept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call the pre-visitation
return p->PreRunOnNode(shared_from_base<CacheOp>(), modified);
}
// Visitor accept method for NodePass
Status CacheOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<CacheOp>(), modified);
}
Status CacheOp::PrepareNodePostAction() {
Status CacheOp::PrepareOperator() {
// Run any common code from super class first before adding our own
RETURN_IF_NOT_OK(ParallelOp::PrepareNodePostAction());
RETURN_IF_NOT_OK(DatasetOp::PrepareOperator());
// Get the computed check sum from all ops in our cache path below us and ask the cache op to create it's cache
uint32_t cache_crc = DatasetOp::GenerateCRC(shared_from_this());
// This is a non-mappable cache op so the id's need to be generated.

38
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -113,35 +113,35 @@ class CacheOp : public CacheBase, public RandomAccessOp {
// Destructor
~CacheOp();
/// \brief Base-class override for setting specific CacheOp configurations. This code will be called
/// during the execution tree prepare phase BEFORE traversing down to child operators.
uint32_t PrepareFlags() const override;
/// \brief Base-class override for special eoe handler.
/// CacheOp must override this because it shall not perform default handling of eoe. Instead
/// the CacheOp manages actions related to the end of the epoch.
/// \notes CacheOp must override this because it shall not perform default handling of eoe. Instead
/// the CacheOp manages actions related to the end of the epoch.
/// \return Status The status code returned
Status EoeReceived(int32_t worker_id) override;
/// \brief Base-class override for NodePass pre-visit acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status PreAccept(NodePass *p, bool *const modified) override;
/// \brief Base-class override for NodePass visitor acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status Accept(NodePass *p, bool *const modified) override;
/// \brief Base-class override for handling cases when an eof is received.
/// \param worker_id - The worker id
/// \return Status The status code returned
Status EofReceived(int32_t worker_id) override;
// \brief Class functor operator ().
/// \return Status The status code returned
Status operator()() override;
/// \brief Entry function for worker thread that fetch rows from CacheLookupOp
/// \param workerId
/// \return Status The status code returned
Status WorkerEntry(int32_t worker_id) override;
/// \brief Base-class override for handling cases if we allow cache miss
/// \brief Base-class override for handling cases if we allow cache miss.
bool AllowCacheMiss() override { return false; }
/// \brief Base-class override for the name of this operator
/// \brief Base-class override for the name of this operator.
std::string Name() const override { return kCacheOp; }
Status PrepareNodePostAction() override;
/// \brief Perform specific post-operations on CacheOp
/// \return Status The status code returned
Status PrepareOperator() override;
private:
WaitPost rows_cache_done_;

37
mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -21,7 +21,6 @@
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "utils/ms_utils.h"
namespace mindspore {
@ -70,6 +69,13 @@ void ConcatOp::Print(std::ostream &out, bool show_all) const {
}
}
// This definition is added to pass the cyclomatic complexity rule of <= 20 units
// The NOLINT directive is to disable cpplint check.
// Clang format and cpplint give conflicting recommendations on this line below.
#define f(fv, sv, shard_index) \
(((fv) == -1 && (sv) == -1) || ((fv) < (sv) && (shard_index) >= (fv) && (shard_index) < (sv)) || \
((fv) > (sv) && ((shard_index) >= (fv) || (shard_index) < (sv)))) // NOLINT
// Main entry point for Concat
Status ConcatOp::operator()() {
children_num_ = static_cast<int32_t>(child_.size());
@ -78,6 +84,7 @@ Status ConcatOp::operator()() {
int eof_count = 0;
int sample_number = 0;
bool is_not_mappable = true;
bool is_not_mappable_or_second_ne_zero = true;
int num_shard = 1;
int shard_index = 0;
std::shared_ptr<DistributedSamplerRT> distribute_sampler = std::dynamic_pointer_cast<DistributedSamplerRT>(sampler_);
@ -99,26 +106,28 @@ Status ConcatOp::operator()() {
RETURN_IF_NOT_OK(Verify(i, buf));
}
// 3. Put the data into output_connector
if (!children_flag_and_nums_.empty()) is_not_mappable = children_flag_and_nums_[i].first;
if (!children_flag_and_nums_.empty()) {
is_not_mappable = children_flag_and_nums_[i].first;
is_not_mappable_or_second_ne_zero = is_not_mappable || (!children_flag_and_nums_[i].second);
}
while (!buf->eoe() && !buf->eof()) {
// if dataset is not mappable or generator dataset which source is yield, cannot get the number of samples in
// python layer), we use filtering to get data
if (sample_number % num_shard == shard_index && (is_not_mappable || !children_flag_and_nums_[i].second)) {
if (sample_number % num_shard == shard_index && is_not_mappable_or_second_ne_zero) {
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(buf)));
} else if (!is_not_mappable && children_flag_and_nums_[i].second) { // if dataset is mappable or generator
// dataset which source is not yield
} else if (!is_not_mappable_or_second_ne_zero) {
// if dataset is mappable or generator dataset which source is not yield,
// get the start and end subscripts of valid values
int fv = children_start_end_index_[i].first, sv = children_start_end_index_[i].second;
// determine whether the data allocated to the current shard id is false data
if ((fv == -1 && sv == -1) || (fv < sv && shard_index >= fv && shard_index < sv) ||
(fv > sv && (shard_index >= fv || shard_index < sv))) {
if (f(fv, sv, shard_index)) {
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(buf)));
}
}
// if dataSet is no mappable or generator dataset` which source is yeild, sample_number+=1
if (is_not_mappable || !children_flag_and_nums_[i].second) {
// if dataset is not mappable or generator dataset which source is yield, sample_number+=1
if (is_not_mappable_or_second_ne_zero) {
sample_number++;
}
@ -127,7 +136,7 @@ Status ConcatOp::operator()() {
// if dataset is mappable,We don't use filtering to pick data.
// so sample_number plus the length of the entire dataset
if (!is_not_mappable && children_flag_and_nums_[i].second) {
if (!is_not_mappable_or_second_ne_zero) {
sample_number += children_flag_and_nums_[i].second;
}
}
@ -190,12 +199,6 @@ Status ConcatOp::ComputeColMap() {
return Status::OK();
}
// Visitor pre-accept method for NodePass
Status ConcatOp::PreAccept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->PreRunOnNode(shared_from_base<ConcatOp>(), modified);
}
// Gets the number of classes
Status ConcatOp::GetNumClasses(int64_t *num_classes) {
int64_t max_num_classes = -1;

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -105,12 +105,6 @@ class ConcatOp : public PipelineOp {
// @return - Status
Status ComputeColMap() override;
/// \brief Base-class override for NodePass pre-visit acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status PreAccept(NodePass *p, bool *const modified) override;
/// \brief Gets the number of classes
/// \param[out] num_classes the number of classes
/// \return Status - The status code return

28
mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -26,10 +26,8 @@
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/datasetops/device_queue_op.h"
#include "minddata/dataset/engine/datasetops/source/sampler/sampler.h"
#include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h"
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/opt/pass.h"
#ifndef ENABLE_ANDROID
#include "utils/system/crc32c.h"
#include "utils/log_adapter.h"
@ -338,13 +336,8 @@ Status DatasetOp::EofReceived(int32_t worker_id) {
return (out_connector_->Add(static_cast<int>(worker_id), std::move(eof_buffer)));
}
// During tree prepare phase, operators may have specific pre-operations to perform depending on
// their role.
Status DatasetOp::PrepareNodePreAction() { return Status::OK(); }
// During tree prepare phase, operators may have specific post-operations to perform depending on
// their role.
Status DatasetOp::PrepareNodePostAction() {
// During tree prepare phase, operators may have specific post-operations to perform depending on their role.
Status DatasetOp::PrepareOperator() {
// Creating Connector object for each op.
// The consumer of the root node is assumed to be one thread.
// If multiple threads are consuming from the root node, they will get the ordered data in round robin fashion.
@ -364,9 +357,6 @@ Status DatasetOp::PrepareNodePostAction() {
return Status::OK();
}
// Getter function. Base class does not have any special flags setting.
uint32_t DatasetOp::PrepareFlags() const { return ExecutionTree::kDePrepNone; }
// Derived classes may implement the reset function if the operator is stateful and needs
// specific reset handling that is not contained in this common code version of the reset.
Status DatasetOp::Reset() {
@ -402,18 +392,6 @@ Status DatasetOp::ComputeColMap() {
return Status::OK();
}
Status DatasetOp::PreAccept(NodePass *p, bool *const modified) {
// DatasetOp is the base class of visitor target pre-visit.
// This method will only be called if its derived class does not implement one.
return p->PreRunOnNode(shared_from_this(), modified);
}
Status DatasetOp::Accept(NodePass *p, bool *const modified) {
// DatasetOp is the base class of visitor target.
// This method will only be called if its derived class does not implement one.
return p->RunOnNode(shared_from_this(), modified);
}
// Getter for the sampler, and it also removes the sampler from the op
Status DatasetOp::FetchRemoveSampler(std::shared_ptr<SamplerRT> *sampler) {
*sampler = sampler_; // It's okay if it sampler_ points to nullptr

68
mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h
View File

@ -78,19 +78,19 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
// Flags that control operator runtime behaviours
enum OpState { kDeOpRunning = 0, kDeOpIdle = 1, kDeOpTerminated };
/// Constructor
/// \brief Constructor
/// \param op_connector_size - The size for the output connector of this operator.
/// \param sampler - The sampler for the op
DatasetOp(int32_t op_connector_size, std::shared_ptr<SamplerRT> sampler);
/// Destructor
/// \brief Destructor
virtual ~DatasetOp() { tree_ = nullptr; }
/// Adds a operator to become our child.
/// \brief Adds a operator to become our child.
/// \param child - shared pointer to the child to add.
Status AddChild(std::shared_ptr<DatasetOp> child);
/// Remove a operator from our children.
/// \brief Remove a operator from our children.
/// \param child - shared pointer to the child to remove.
Status RemoveChild(std::shared_ptr<DatasetOp> child);
@ -114,9 +114,9 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
// Getter function to get all of our parents.
std::vector<DatasetOp *> parents() const;
// Inserts a operator as the parent current op.
// Inserted op will become the sole parent of the current op.
// The existing parent of the current op will be transferred to the inserted op.
// \brief Inserts a operator as the parent current op.
// \notes Inserted op will become the sole parent of the current op.
// The existing parent of the current op will be transferred to the inserted op.
Status InsertAsParent(std::shared_ptr<DatasetOp> to_add);
/// \brief Creates the connector within this operator
@ -140,9 +140,9 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
}
/// \brief Class functor operator ().
/// DatasetOps operate by launching a thread (see ExecutionTree).
/// This pure virtual version makes the requirement that derived classes must provide a functor
/// that will execute their main runtime loop code.
/// \notes DatasetOps operate by launching a thread (see ExecutionTree).
/// This pure virtual version makes the requirement that derived classes must provide a functor
/// that will execute their main runtime loop code.
/// \return Status The status code returned
virtual Status operator()() = 0;
@ -170,8 +170,8 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
virtual Status GetNextBuffer(std::unique_ptr<DataBuffer> *p_buffer, int32_t worker_id, bool retry_if_eoe);
/// \brief Gets the next buffer from the given child . This function also has built-in eoe and eof
/// message handling so that child classes don't have to manually code pass-through logic when
/// those messages are received.
/// message handling so that child classes don't have to manually code pass-through logic when
/// those messages are received.
/// \param p_buffer - The shared pointer for the fetched buffer to return (by reference)
/// \param worker_id - The worker id
/// \return Status The status code returned
@ -194,44 +194,34 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
virtual Status GetClassIndexing(std::vector<std::pair<std::string, std::vector<int32_t>>> *output_class_indexing);
/// \brief Performs handling for when an eoe message is received.
/// The base class implementation simply flows the eoe message to output. Derived classes
/// may override if they need to perform special eoe handling.
/// The base class implementation simply flows the eoe message to output. Derived classes
/// may override if they need to perform special eoe handling.
/// \param worker_id - The worker id
/// \return Status The status code returned
virtual Status EoeReceived(int32_t worker_id);
/// \brief Performs handling for when an eof message is received.
/// The base class implementation simply flows the eof message to output. Derived classes
/// may override if they need to perform special eof handling.
/// The base class implementation simply flows the eof message to output. Derived classes
/// may override if they need to perform special eof handling.
/// \param worker_id - The worker id
/// \return Status The status code returned
virtual Status EofReceived(int32_t worker_id);
/// \brief Derived classes may implement the reset function if the operator is stateful and needs
/// specific reset handling that is not contained in this common code version of the reset
/// specific reset handling that is not contained in this common code version of the reset
/// \return Status The status code returned
virtual Status Reset();
/// \brief During tree prepare phase, operators may have specific pre-operations to perform depending on
/// their role.
/// \notes Derived versions of this function should always call it's superclass version first
/// before providing their own implementations.
virtual Status PrepareNodePreAction();
/// \brief During tree prepare phase, operators may have specific post-operations to perform depending on
/// their role.
/// their role.
/// \notes Derived versions of this function should always call it's superclass version first
/// before providing their own implementations.
virtual Status PrepareNodePostAction();
/// before providing their own implementations.
virtual Status PrepareOperator();
/// \brief Getter function
/// \return The operator id
int32_t id() const { return operator_id_; }
/// \brief Getter function
/// \return The prepare flags
virtual uint32_t PrepareFlags() const;
/// \brief Getter function
/// \return The number of workers in this op
virtual int32_t num_workers() const = 0;
@ -315,22 +305,6 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
/// \return Vector of Children
std::vector<std::shared_ptr<DatasetOp>> Children() const { return child_; }
/// \brief Base method for NodePass pre-visit. A tree walk consists of walking down the tree and also walking back up
/// in a depth-first order. PreAccept is the node visit on the way down, whereas the regular Accept is the main
/// visit on the way back up the tree during a post-order traversal. Subclass needs to override this if it
/// requires special node visit access. Check "dataset/engine/opt/pass.h" for more details.
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
virtual Status PreAccept(NodePass *p, bool *const modified);
/// \brief Base method for NodePass visit. Subclass needs to override this if it requires special node visit access.
/// Check "dataset/engine/opt/pass.h" for more details.
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
virtual Status Accept(NodePass *p, bool *const modified);
/// Op name getter
/// \return Name of the current Op
virtual std::string Name() const = 0;
@ -401,7 +375,7 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
void AddParent(DatasetOp *parent);
/// Compute the current op's column map using its child's column map.
/// Get called during the tree post-prepare phase in PrepareNodePostAction.
/// Get called during the tree post-prepare phase in PrepareOperator.
/// This base implementation just inherits the map from child 0, and can only be used if the number of children is 1.
/// Operations changing the column map it inherits from the child must overwrite this function.
/// \return - Status

15
mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,17 +16,10 @@
#include "minddata/dataset/engine/datasetops/device_queue_op.h"
#include <iomanip>
#include <iostream>
#include <memory>
#include <utility>
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/dataset_iterator.h"
#include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/engine/perf/profiling.h"
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/util/task_manager.h"
@ -490,12 +483,6 @@ void DeviceQueueOp::Print(std::ostream &out, bool show_all) const {
}
}
// Visitor accept method for NodePass
Status DeviceQueueOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<DeviceQueueOp>(), modified);
}
void DeviceQueueOp::ProfilingRecorder(bool isProfilingEnable, std::shared_ptr<DeviceQueueTracing> profiling_node,
int64_t send_batch, int32_t tdt_cost, uint64_t *batch_start_time,
uint64_t *end_time, int32_t connector_capacity, int32_t connector_size) {

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -168,12 +168,6 @@ class DeviceQueueOp : public PipelineOp {
Status operator()() override;
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
// Record the pipeline profiling info
void ProfilingRecorder(bool isProfilingEnable, std::shared_ptr<DeviceQueueTracing> profiling_node, int64_t send_batch,
int32_t tdt_cost, uint64_t *batch_start_time, uint64_t *end_time, int32_t connector_capacity,

12
mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc
View File

@ -19,7 +19,6 @@
#include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h"
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/util/log_adapter.h"
namespace mindspore {
@ -102,17 +101,6 @@ Status EpochCtrlOp::EoeReceived(int32_t worker_id) {
return Status::OK();
}
// Pre-Visitor accept method for NodePass
Status EpochCtrlOp::PreAccept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call the pre-visitation
return p->PreRunOnNode(shared_from_base<EpochCtrlOp>(), modified);
}
// Visitor accept method for NodePass
Status EpochCtrlOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call the pre-visitation
return p->RunOnNode(shared_from_base<EpochCtrlOp>(), modified);
}
int64_t EpochCtrlOp::GetTreeRepeatCount() { return child_[0]->GetTreeRepeatCount(); }
} // namespace dataset
} // namespace mindspore

16
mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -42,7 +42,7 @@ class EpochCtrlOp : public RepeatOp {
Status Build(std::shared_ptr<EpochCtrlOp> *);
};
// Contructor
// Constructor
explicit EpochCtrlOp(int32_t num_epoch);
// Destructor
@ -65,18 +65,6 @@ class EpochCtrlOp : public RepeatOp {
// @param worker_id - The worker id
Status EoeReceived(int32_t worker_id) override;
/// \brief Base-class override for NodePass pre-visit acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status PreAccept(NodePass *p, bool *const modified) override;
/// \brief Base-class override for NodePass visitor acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status Accept(NodePass *p, bool *const modified) override;
int64_t GetTreeRepeatCount() override;
};
} // namespace dataset

18
mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,18 +16,14 @@
#include "minddata/dataset/engine/datasetops/filter_op.h"
#include <algorithm>
#include <cstring>
#include <iomanip>
#include <iostream>
#include <memory>
#include <vector>
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/kernels/tensor_op.h"
#include "minddata/dataset/util/log_adapter.h"
#include "minddata/dataset/util/task_manager.h"
@ -247,17 +243,5 @@ Status FilterOp::InvokePredicateFunc(const TensorRow &input, bool *out_predicate
return Status(StatusCode::kSuccess, "FilterOp predicate func call succeed");
}
// Visitor accept method for NodePass
Status FilterOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<FilterOp>(), modified);
}
// Visitor pre-accept method for NodePass
Status FilterOp::PreAccept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->PreRunOnNode(shared_from_base<FilterOp>(), modified);
}
} // namespace dataset
} // namespace mindspore

14
mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -121,18 +121,6 @@ class FilterOp : public ParallelOp {
// @param show_all A bool to control if you want to show all info or just a summary.
void Print(std::ostream &out, bool show_all) const override;
/// \brief Base-class override for NodePass pre-visit acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status PreAccept(NodePass *p, bool *const modified) override;
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return kFilterOp; }

16
mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -26,7 +26,7 @@
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/datasetops/map_op/cpu_map_job.h"
#include "minddata/dataset/engine/datasetops/map_op/gpu_map_job.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/kernels/tensor_op.h"
#include "minddata/dataset/util/log_adapter.h"
#include "minddata/dataset/util/task_manager.h"
@ -445,18 +445,6 @@ void MapOp::CreateFinalColMap(std::unordered_map<std::string, int32_t> *col_name
}
}
// Visitor accept method for NodePass
Status MapOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<MapOp>(), modified);
}
// Visitor pre-accept method for NodePass
Status MapOp::PreAccept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->PreRunOnNode(shared_from_base<MapOp>(), modified);
}
Status MapOp::WaitForWorkers() {
// reset num_paused workers to 0
num_workers_paused_ = 0;

14
mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -177,18 +177,6 @@ class MapOp : public ParallelOp {
// @return the number of threads consuming data from previous op's output Connector.
int32_t num_consumers() const override;
/// \brief Base-class override for NodePass pre-visit acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status PreAccept(NodePass *p, bool *const modified) override;
/// \brief Base-class override for NodePass visitor acceptor.
/// \param[in] p Pointer to the NodePass to be accepted.
/// \param[out] modified Whether this node visit modified the pipeline.
/// \return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return kMapOp; }

22
mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -68,26 +68,6 @@ class ParallelOp : public DatasetOp {
return out;
}
// During tree prepare phase, operators may have specific pre-operations to perform depending on
// their role.
// @notes Derived versions of this function should always call it's superclass version first
// before providing their own implementations.
// @return Status - The error return code
Status PrepareNodePreAction() override {
// Run common code from super class before adding ParallelOp specific logic
return (DatasetOp::PrepareNodePreAction());
}
// During tree prepare phase, operators may have specific post-operations to perform depending on
// their role.
// @notes Derived versions of this function should always call it's superclass version first
// before providing their own implementations.
// @return Status - The error return code
Status PrepareNodePostAction() override {
// Run common code from super class before adding ParallelOp specific logic
return (DatasetOp::PrepareNodePostAction());
}
// Override base class reset to provide reset actions specific to the ParallelOp class.
// @return Status The status code returned
Status Reset() override;

20
mindspore/ccsrc/minddata/dataset/engine/datasetops/pipeline_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -67,24 +67,6 @@ class PipelineOp : public DatasetOp {
// @return The number of threads that push data to the output connector
int32_t num_producers() const override { return 1; }
// During tree prepare phase, operators may have specific pre-operations to perform depending on
// their role.
// @notes Derived versions of this function should always call it's superclass version first
// before providing their own implementations.
Status PrepareNodePreAction() override {
// Run common code from super class before adding PipelineOp specific logic
return (DatasetOp::PrepareNodePreAction());
}
// During tree prepare phase, operators may have specific post-operations to perform depending on
// their role.
// @notes Derived versions of this function should always call it's superclass version first
// before providing their own implementations.
Status PrepareNodePostAction() override {
// Run common code from super class before adding PipelineOp specific logic
return (DatasetOp::PrepareNodePostAction());
}
protected:
// *******************************************************************************
// I'm predicting there will be common arguments or functionality for pipeline ops,

10
mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -23,9 +23,7 @@
#include <utility>
#include <vector>
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/util/log_adapter.h"
namespace mindspore {
@ -132,12 +130,6 @@ Status ProjectOp::EoeReceived(int32_t worker_id) {
Status ProjectOp::EofReceived(int32_t worker_id) { return Status::OK(); }
// Visitor accept method for NodePass
Status ProjectOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<ProjectOp>(), modified);
}
// Compute the column map and save it into our own column name map
// We cannot use the super class ComputeColMap here because we're making a modification of the
// map from the child map.

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -101,12 +101,6 @@ class ProjectOp : public PipelineOp {
// @return Status The status code returned
Status EofReceived(int32_t worker_id) override;
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return kProjectOp; }

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -24,7 +24,6 @@
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/util/log_adapter.h"
namespace mindspore {
@ -171,10 +170,5 @@ Status RenameOp::EoeReceived(int32_t) {
return Status::OK();
}
// Visitor accept method for NodePass
Status RenameOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<RenameOp>(), modified);
}
} // namespace dataset
} // namespace mindspore

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -110,12 +110,6 @@ class RenameOp : public PipelineOp {
// @return Status The status code returned
Status operator()() override;
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return kRenameOp; }

16
mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,8 +20,6 @@
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/datasetops/repeat_op.h"
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/util/log_adapter.h"
namespace mindspore {
@ -175,18 +173,6 @@ int32_t RepeatOp::num_producers() const {
}
}
// Pre-Visitor accept method for NodePass
Status RepeatOp::PreAccept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call the pre-visitation
return p->PreRunOnNode(shared_from_base<RepeatOp>(), modified);
}
// Visitor accept method for NodePass
Status RepeatOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<RepeatOp>(), modified);
}
int64_t RepeatOp::GetTreeRepeatCount() { return num_repeats_; }
} // namespace dataset
} // namespace mindspore

14
mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -109,18 +109,6 @@ class RepeatOp : public PipelineOp {
// @param workerId - The worker id
int32_t num_producers() const override;
/// \brief Base-class override for NodePass pre-visit acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status PreAccept(NodePass *p, bool *const modified) override;
/// \brief Base-class override for NodePass visitor acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return kRepeatOp; }

12
mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,12 +16,9 @@
#if defined(_WIN32) || defined(_WIN64)
#include <stdlib.h>
#endif
#include <securec.h>
#include <algorithm>
#include <chrono>
#include <iomanip>
#include <iostream>
#include <limits>
#include <random>
#include <utility>
@ -30,7 +27,6 @@
#include "minddata/dataset/engine/dataset_iterator.h"
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/util/log_adapter.h"
#include "minddata/dataset/util/random.h"
#include "minddata/dataset/util/status.h"
@ -291,11 +287,5 @@ Status ShuffleOp::EoeReceived(int32_t worker_id) {
state_ = OpState::kDeOpIdle;
return Status::OK();
}
// Visitor accept method for NodePass
Status ShuffleOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<ShuffleOp>(), modified);
}
} // namespace dataset
} // namespace mindspore

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -155,12 +155,6 @@ class ShuffleOp : public PipelineOp {
// @return Status The status code returned
Status EoeReceived(int32_t worker_id) override;
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return kShuffleOp; }

15
mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -22,7 +22,6 @@
#include "minddata/dataset/engine/datasetops/skip_op.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/util/log_adapter.h"
namespace mindspore {
@ -124,17 +123,5 @@ Status SkipOp::EofReceived(int32_t worker_id) {
return Status::OK();
}
// Visitor accept method for NodePass
Status SkipOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<SkipOp>(), modified);
}
// Visitor pre-accept method for NodePass
Status SkipOp::PreAccept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->PreRunOnNode(shared_from_base<SkipOp>(), modified);
}
} // namespace dataset
} // namespace mindspore

14
mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -74,18 +74,6 @@ class SkipOp : public PipelineOp {
// @param worker_id - The worker id
Status EofReceived(int32_t worker_id) override;
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
/// \brief Base-class override for NodePass pre-visit acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status PreAccept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return kSkipOp; }

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -21,7 +21,6 @@
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#ifndef ENABLE_ANDROID
#include "minddata/dataset/kernels/image/image_utils.h"
#else
@ -590,12 +589,6 @@ Status AlbumOp::LaunchThreadsAndInitOp() {
return Status::OK();
}
// Visitor accept method for NodePass
Status AlbumOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<AlbumOp>(), modified);
}
Status AlbumOp::ComputeColMap() {
// Set the column name map (base class field)
if (column_name_id_map_.empty()) {

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -192,12 +192,6 @@ class AlbumOp : public ParallelOp, public RandomAccessOp {
/// \return bool - if file is bad then return false
bool CheckImageType(const std::string &file_name, bool *valid);
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return "AlbumOp"; }

11
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc
View File

@ -1,6 +1,6 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
@ -23,7 +23,6 @@
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "minddata/dataset/engine/data_schema.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#ifndef ENABLE_ANDROID
#include "minddata/dataset/kernels/image/image_utils.h"
#else
@ -440,12 +439,6 @@ Status CelebAOp::Reset() {
return Status::OK();
}
// Visitor accept method for NodePass
Status CelebAOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<CelebAOp>(), modified);
}
Status CelebAOp::ComputeColMap() {
// Set the column name map (base class field)
if (column_name_id_map_.empty()) {

10
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.h
View File

@ -1,6 +1,6 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
@ -169,12 +169,6 @@ class CelebAOp : public ParallelOp, RandomAccessOp {
// @return Status The status code returned
Status AddIOBlock(std::unique_ptr<DataBuffer> *data_buffer);
/// \brief Base-class override for NodePass visitor acceptor
/// \param[in] p Pointer to the NodePass to be accepted
/// \param[out] modified Indicator if the node was changed at all
/// \return Status of the node visit
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return "CelebAOp"; }

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -26,7 +26,6 @@
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "utils/ms_utils.h"
namespace mindspore {
@ -500,12 +499,6 @@ Status CifarOp::CountTotalRows(const std::string &dir, const std::string &usage,
}
}
// Visitor accept method for NodePass
Status CifarOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<CifarOp>(), modified);
}
Status CifarOp::ComputeColMap() {
// set the column name map (base class field)
if (column_name_id_map_.empty()) {

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -165,12 +165,6 @@ class CifarOp : public ParallelOp, public RandomAccessOp {
// @return
static Status CountTotalRows(const std::string &dir, const std::string &usage, bool isCIFAR10, int64_t *count);
/// \brief Base-class override for NodePass visitor acceptor
/// \param[in] p Pointer to the NodePass to be accepted
/// \param[out] modified Indicator if the node was changed at all
/// \return Status of the node visit
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return "CifarOp"; }

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -26,7 +26,6 @@
#include "minddata/dataset/util/task_manager.h"
#include "minddata/dataset/engine/jagged_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/engine/datasetops/source/io_block.h"
#include "minddata/dataset/util/random.h"
@ -549,11 +548,5 @@ Status ClueOp::ComputeColMap() {
return Status::OK();
}
// Visitor accept method for NodePass
Status ClueOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<ClueOp>(), modified);
}
} // namespace dataset
} // namespace mindspore

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -177,12 +177,6 @@ class ClueOp : public ParallelOp {
// @return Name of the current Op
std::string Name() const override { return "ClueOp"; }
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
private:
// The entry point for when workers are launched.
// @param worker_id - the id of the worker that is executing this function.

10
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -22,7 +22,7 @@
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/engine/execution_tree.h"
namespace mindspore {
namespace dataset {
@ -674,12 +674,6 @@ Status CocoOp::GetClassIndexing(const std::string &dir, const std::string &file,
return Status::OK();
}
// Visitor accept method for NodePass
Status CocoOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<CocoOp>(), modified);
}
Status CocoOp::ComputeColMap() {
// Set the column name map (base class field)
if (column_name_id_map_.empty()) {

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -203,12 +203,6 @@ class CocoOp : public ParallelOp, public RandomAccessOp {
static Status GetClassIndexing(const std::string &dir, const std::string &task_type, const std::string &task_mode,
std::vector<std::pair<std::string, std::vector<int32_t>>> *output_class_indexing);
/// \brief Base-class override for NodePass visitor acceptor
/// \param[in] p Pointer to the NodePass to be accepted
/// \param[out] modified Indicator if the node was changed at all
/// \return Status of the node visit
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return "CocoOp"; }

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -23,7 +23,6 @@
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/engine/jagged_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/util/random.h"
namespace mindspore {
@ -916,11 +915,5 @@ Status CsvOp::ComputeColMap() {
return Status::OK();
}
// Visitor accept method for NodePass
Status CsvOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<CsvOp>(), modified);
}
} // namespace dataset
} // namespace mindspore

6
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.h
View File

@ -298,12 +298,6 @@ class CsvOp : public ParallelOp {
// @return Vector of the input file names
std::vector<std::string> FileNames() { return csv_files_list_; }
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return "CsvOp"; }

10
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,11 +16,9 @@
#include "minddata/dataset/engine/datasetops/source/generator_op.h"
#include <iomanip>
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/util/task_manager.h"
#include "minddata/dataset/engine/opt/pass.h"
namespace mindspore {
namespace dataset {
@ -278,12 +276,6 @@ Status GeneratorOp::Reset() {
return Status(StatusCode::kSuccess, "GeneratorOp Reset Succeed");
}
// Visitor accept method for NodePass
Status GeneratorOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<GeneratorOp>(), modified);
}
Status GeneratorOp::ComputeColMap() {
// Setup column names map (base class field)
if (column_name_id_map_.empty()) {

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -126,12 +126,6 @@ class GeneratorOp : public PipelineOp, public RandomAccessOp {
/// \return Status The status code returned
Status Reset() override;
/// Base-class override for NodePass visitor acceptor.
/// \param p - Pointer to the NodePass to be accepted.
/// \param modified - Whether this node visit modified the pipeline.
/// \return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
/// Op name getter
/// \return Name of the current Op
std::string Name() const override { return "GeneratorOp"; }

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -22,7 +22,6 @@
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
namespace mindspore {
namespace dataset {
@ -450,12 +449,6 @@ Status ImageFolderOp::CountRowsAndClasses(const std::string &path, const std::se
return Status::OK();
}
// Visitor accept method for NodePass
Status ImageFolderOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<ImageFolderOp>(), modified);
}
Status ImageFolderOp::ComputeColMap() {
// Set the column name map (base class field)
if (column_name_id_map_.empty()) {

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -207,12 +207,6 @@ class ImageFolderOp : public ParallelOp, public RandomAccessOp {
static Status CountRowsAndClasses(const std::string &path, const std::set<std::string> &exts, int64_t *num_rows,
int64_t *num_classes, std::map<std::string, int32_t> class_index);
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return "ImageFolderOp"; }

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -27,7 +27,6 @@
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
namespace mindspore {
namespace dataset {
@ -436,12 +435,6 @@ Status ManifestOp::GetClassIndexing(const std::string &file, const py::dict &dic
}
#endif
// Visitor accept method for NodePass
Status ManifestOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<ManifestOp>(), modified);
}
Status ManifestOp::ComputeColMap() {
// Set the column name map (base class field)
if (column_name_id_map_.empty()) {

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -180,12 +180,6 @@ class ManifestOp : public ParallelOp, public RandomAccessOp {
std::map<std::string, int32_t> *output_class_indexing);
#endif
/// \brief Base-class override for NodePass visitor acceptor
/// \param[in] p Pointer to the NodePass to be accepted
/// \param[out] modified Indicator if the node was changed at all
/// \return Status of the node visit
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return "ManifestOp"; }

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -29,7 +29,6 @@
#include "minddata/dataset/engine/datasetops/dataset_op.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/util/log_adapter.h"
namespace mindspore {
@ -461,12 +460,6 @@ Status MindRecordOp::CountTotalRows(const std::vector<std::string> dataset_path,
return Status::OK();
}
// Visitor accept method for NodePass
Status MindRecordOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<MindRecordOp>(), modified);
}
Status MindRecordOp::ComputeColMap() {
if (column_name_id_map_.empty()) {
for (int i = 0; i < static_cast<int>(columns_to_load_.size()); i++) {

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -202,12 +202,6 @@ class MindRecordOp : public ParallelOp {
Status Init();
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return "MindRecordOp"; }

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -24,7 +24,6 @@
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
namespace mindspore {
namespace dataset {
@ -459,12 +458,6 @@ Status MnistOp::CountTotalRows(const std::string &dir, const std::string &usage,
return Status::OK();
}
// Visitor accept method for NodePass
Status MnistOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<MnistOp>(), modified);
}
Status MnistOp::ComputeColMap() {
// set the column name map (base class field)
if (column_name_id_map_.empty()) {

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -158,12 +158,6 @@ class MnistOp : public ParallelOp, public RandomAccessOp {
// @return
static Status CountTotalRows(const std::string &dir, const std::string &usage, int64_t *count);
/// \brief Base-class override for NodePass visitor acceptor
/// \param[in] p Pointer to the NodePass to be accepted
/// \param[out] modified Indicator if the node was changed at all
/// \return Status of the node visit
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return "MnistOp"; }

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -24,7 +24,6 @@
#include "minddata/dataset/util/random.h"
#include "minddata/dataset/util/wait_post.h"
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "minddata/dataset/engine/opt/pass.h"
namespace mindspore {
namespace dataset {
@ -406,12 +405,6 @@ Status RandomDataOp::Reset() {
return Status::OK();
}
// Visitor accept method for NodePass
Status RandomDataOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<RandomDataOp>(), modified);
}
Status RandomDataOp::ComputeColMap() {
// Extract the column name mapping from the schema and save it in the class.
if (column_name_id_map_.empty()) {

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -249,12 +249,6 @@ class RandomDataOp : public ParallelOp {
return ++buffer_id_;
}
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
// Private function for computing the assignment of the column name map.
// @return - Status
Status ComputeColMap() override;

10
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -21,7 +21,6 @@
#include <string>
#include <utility>
#include "utils/ms_utils.h"
#include "minddata/dataset/engine/datasetops/source/text_file_op.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/util/task_manager.h"
@ -29,7 +28,6 @@
#include "minddata/dataset/util/random.h"
#include "minddata/dataset/engine/datasetops/source/io_block.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
namespace mindspore {
namespace dataset {
@ -499,11 +497,5 @@ Status TextFileOp::ComputeColMap() {
return Status::OK();
}
// Visitor accept method for NodePass
Status TextFileOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<TextFileOp>(), modified);
}
} // namespace dataset
} // namespace mindspore

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -177,12 +177,6 @@ class TextFileOp : public ParallelOp {
// @return Vector of the input file names
std::vector<std::string> FileNames() { return text_files_list_; }
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
private:
// The entry point for when workers are launched.
// @param worker_id - the id of the worker that is executing this function.

31
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -26,7 +26,6 @@
#include <vector>
#include "proto/example.pb.h"
#include "./securec.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/engine/data_schema.h"
@ -34,7 +33,6 @@
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/jagged_connector.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/util/random.h"
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/util/task_manager.h"
@ -1020,12 +1018,6 @@ int64_t TFReaderOp::CountTotalRowsSectioned(const std::vector<std::string> &file
return rows_read;
}
// Visitor accept method for NodePass
Status TFReaderOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<TFReaderOp>(), modified);
}
Status TFReaderOp::ComputeColMap() {
// Construct the column name map for this operator (base class field)
if (column_name_id_map_.empty()) {
@ -1038,26 +1030,5 @@ Status TFReaderOp::ComputeColMap() {
return Status::OK();
}
// During tree prepare phase, operators may have specific post-operations to perform depending on
// their role.
Status TFReaderOp::PrepareNodePostAction() {
// Run common code from super class before adding TFReaderOp specific handling
RETURN_IF_NOT_OK(ParallelOp::PrepareNodePostAction());
// Now that the sampler has been saved for the cache, we need to adjust the TFReaderOp to turn it into
// a simpler producer of all data (no shuffling or sharding or anything)
if (!BitTest(tree_->PrepareFlags(), ExecutionTree::kDePrepCache)) {
// This sanity check had been delayed until now in the prepare loop.
// If we are not in a cache path, then we can validate the file-based sharding config.
// If we are in a cache path, there is no file-based sharding so the check is not correct in that
// situation.
if (!equal_rows_per_shard_ && dataset_files_list_.size() < static_cast<uint32_t>(num_devices_)) {
RETURN_STATUS_UNEXPECTED("Invalid file, not enough tfrecord files provided.\n");
}
}
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

14
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -222,12 +222,6 @@ class TFReaderOp : public ParallelOp {
static Status CountTotalRows(int64_t *out_total_rows, const std::vector<std::string> &filenames, int64_t threads = 1,
bool estimate = false);
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return "TFReaderOp"; }
@ -236,12 +230,6 @@ class TFReaderOp : public ParallelOp {
// @return Vector of the input file names
std::vector<std::string> FileNames() { return dataset_files_list_; }
// During tree prepare phase, operators may have specific post-operations to perform depending on
// their role.
// @notes Derived versions of this function should always call it's superclass version first
// before providing their own implementations.
Status PrepareNodePostAction() override;
static bool ValidateFirstRowCrc(const std::string &filename);
private:

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -24,7 +24,6 @@
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "utils/ms_utils.h"
namespace mindspore {
@ -494,12 +493,6 @@ Status VOCOp::GetClassIndexing(const std::string &dir, const std::string &task_t
}
#endif
// Visitor accept method for NodePass
Status VOCOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<VOCOp>(), modified);
}
Status VOCOp::ComputeColMap() {
// Set the column name map (base class field)
if (column_name_id_map_.empty()) {

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -210,12 +210,6 @@ class VOCOp : public ParallelOp, public RandomAccessOp {
const py::dict &dict, std::map<std::string, int32_t> *output_class_indexing);
#endif
/// \brief Base-class override for NodePass visitor acceptor
/// \param[in] p Pointer to the NodePass to be accepted
/// \param[out] modified Indicator if the node was changed at all
/// \return Status of the node visit
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return "VOCOp"; }

15
mindspore/ccsrc/minddata/dataset/engine/datasetops/take_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -23,7 +23,6 @@
#include "minddata/dataset/engine/datasetops/take_op.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
namespace mindspore {
namespace dataset {
@ -127,17 +126,5 @@ Status TakeOp::FillBuffer(std::unique_ptr<DataBuffer> *buffer, std::unique_ptr<D
return Status::OK();
}
// Visitor accept method for NodePass
Status TakeOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<TakeOp>(), modified);
}
// Visitor pre-accept method for NodePass
Status TakeOp::PreAccept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->PreRunOnNode(shared_from_base<TakeOp>(), modified);
}
} // namespace dataset
} // namespace mindspore

14
mindspore/ccsrc/minddata/dataset/engine/datasetops/take_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -78,18 +78,6 @@ class TakeOp : public PipelineOp {
// @return Status The status code returned
Status operator()() override;
// Base-class override for NodePass visitor acceptor.
// @param p - Pointer to the NodePass to be accepted.
// @param modified - Whether this node visit modified the pipeline.
// @return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
/// \brief Base-class override for NodePass pre-visit acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status PreAccept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return kTakeOp; }

15
mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,7 +20,6 @@
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/util/log_adapter.h"
@ -238,18 +237,6 @@ Status ZipOp::EoeReceived(int32_t) {
return Status::OK();
}
// Visitor pre-accept method for NodePass
Status ZipOp::PreAccept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->PreRunOnNode(shared_from_base<ZipOp>(), modified);
}
// Visitor accept method for NodePass
Status ZipOp::Accept(NodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->RunOnNode(shared_from_base<ZipOp>(), modified);
}
Status ZipOp::ComputeColMap() {
if (column_name_id_map_.empty()) {
column_name_id_map_ = {};

14
mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.h
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -104,18 +104,6 @@ class ZipOp : public PipelineOp {
// @return Status The status code returned
Status operator()() override;
/// \brief Base-class override for NodePass pre-visit acceptor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status PreAccept(NodePass *p, bool *const modified) override;
/// \brief Base-class override for NodePass visitor acceptor.
/// \param[in] p Pointer to the NodePass to be accepted.
/// \param[out] modified Whether this node visit modified the pipeline.
/// \return - Status of the node visit.
Status Accept(NodePass *p, bool *const modified) override;
// Op name getter
// @return Name of the current Op
std::string Name() const override { return kZipOp; }

54
mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc
View File

@ -29,7 +29,7 @@
namespace mindspore {
namespace dataset {
// Constructor
ExecutionTree::ExecutionTree() : id_count_(0), tree_state_(kDeTStateInit), prepare_flags_(kDePrepNone) {
ExecutionTree::ExecutionTree() : id_count_(0), tree_state_(kDeTStateInit) {
tg_ = std::make_unique<TaskGroup>();
profiling_manager_ = std::make_unique<ProfilingManager>(this);
#if defined(ENABLE_GPUQUE) || defined(ENABLE_TDTQUE)
@ -66,11 +66,11 @@ Status ExecutionTree::AssociateNode(const std::shared_ptr<DatasetOp> &op) {
if (op->tree_ == this) {
return Status::OK();
}
if (tree_state_ != kDeTStateInit && tree_state_ != kDeTStateBuilding && tree_state_ != kDeTStatePrepare) {
if (tree_state_ != kDeTStateInit && tree_state_ != kDeTStateBuilding) {
std::string err_msg =
"Invalid tree state for adding a node. Current state: " + std::to_string(static_cast<int>(tree_state_)) +
" Expected states: " + std::to_string(static_cast<int>(kDeTStateInit)) + " or " +
std::to_string(static_cast<int>(kDeTStateBuilding)) + " or " + std::to_string(static_cast<int>(kDeTStatePrepare));
std::to_string(static_cast<int>(kDeTStateBuilding));
RETURN_STATUS_UNEXPECTED(err_msg);
}
@ -172,10 +172,10 @@ Status ExecutionTree::Launch() {
#endif
// Tree must be built and prepared before it can be launched!
if (tree_state_ != kDeTStateReady) {
if (tree_state_ != kDeTStatePrepared) {
std::string err_msg =
"Invalid tree state for launching tree. Current state: " + std::to_string(static_cast<int>(tree_state_)) +
" Expected state: " + std::to_string(static_cast<int>(kDeTStateReady));
" Expected state: " + std::to_string(static_cast<int>(kDeTStatePrepared));
RETURN_STATUS_UNEXPECTED(err_msg);
}
@ -244,43 +244,31 @@ Status ExecutionTree::LaunchWorkers(int32_t num_workers, std::function<Status(ui
return Status::OK();
}
// walk the tree to perform modifications to the tree or specific nodes within the tree to get
// it ready for execution.
//
// This driver is deprecated.
// Walks the tree to perform modifications to the tree in post-order to get it ready for execution.
Status ExecutionTree::Prepare() {
// The tree is ready to be prepared.
tree_state_ = kDeTStatePrepare;
if (root_ == nullptr) {
RETURN_STATUS_UNEXPECTED("Please assign one operator as the root of this tree.");
}
// Start the recursive prepare
RETURN_IF_NOT_OK(this->PrepareNode(root_));
tree_state_ = kDeTStateReady;
return Status::OK();
}
std::vector<std::shared_ptr<DatasetOp>> fifo;
std::shared_ptr<DatasetOp> op = root_;
size_t index = 0;
// Recursive function used during prepare phase to visit a node and drive any pre- and post-
// node actions during a tree walk.
Status ExecutionTree::PrepareNode(const std::shared_ptr<DatasetOp> &dataset_op) {
// Before going down into children, make any prepare flags updates based on this operator.
uint32_t op_prep_flags = dataset_op->PrepareFlags();
BitSet(&prepare_flags_, op_prep_flags);
// Build a FIFO queue with the root at the beginning and continue adding its descendants to the queue.
fifo.push_back(op);
do {
op = fifo[index];
fifo.insert(fifo.end(), op->child_.begin(), op->child_.end());
++index;
} while (index < fifo.size());
// Now, descend to children
for (const auto &i : dataset_op->child_) {
RETURN_IF_NOT_OK(this->PrepareNode(i));
// By iterating from the end of the FIFO queue, we simulate the post-order walk.
for (auto rit = fifo.crbegin(); rit != fifo.crend(); ++rit) {
RETURN_IF_NOT_OK((*rit)->PrepareOperator());
}
// No more children, now we execute any prepare actions before going back up the
// the tree on recursive function
RETURN_IF_NOT_OK(dataset_op->PrepareNodePostAction());
// Then clear the flags from this op now that we have prepared it.
BitClear(&prepare_flags_, op_prep_flags);
// The tree is prepared.
tree_state_ = kDeTStatePrepared;
return Status::OK();
}
} // namespace dataset

141
mindspore/ccsrc/minddata/dataset/engine/execution_tree.h
View File

@ -39,19 +39,11 @@ class Pass;
using OptPass = std::vector<std::unique_ptr<Pass>>;
class ExecutionTree {
public:
// Prepare flags used during tree prepare phase
enum PrepareFlags {
kDePrepNone = 0,
kDePrepRepeat = 1, // Processing a repeat operation
kDePrepCache = 2 // Processing a cache operation
};
// State flags for the lifecycle of the tree
enum TreeState {
kDeTStateInit = 0, // The freshly initialized state after construction
kDeTStateBuilding, // The tree is being built, nodes are being added
kDeTStatePrepare, // The tree has been assigned a root node and is pending prepare
kDeTStateReady, // The tree has been prepared and is ready to be launched
kDeTStatePrepared, // The tree has been prepared and is ready to be launched
kDeTStateExecuting, // The tree has been launched and is executing
kDeTStateEpochEnd, // The tree has been received end of epoch signal, just for profiling
kDeTStateFinished // The tree has been drained, dataset iterator received EOF
@ -111,122 +103,101 @@ class ExecutionTree {
// Destructor
~ExecutionTree();
// Associates a DatasetOp with this tree. This assigns a valid node id to the operator and
// provides it with a link to the tree. A node cannot form any relationships (parent/child) with
// other nodes unless they are associated with the same tree.
// @param op - The operator to associate
// @return Status The status code returned
/// \brief Associates a DatasetOp with this tree. This assigns a valid node id to the operator and
/// provides it with a link to the tree. A node cannot form any relationships (parent/child) with
/// other nodes unless they are associated with the same tree.
/// \param op - The operator to associate
/// \return Status The status code returned
Status AssociateNode(const std::shared_ptr<DatasetOp> &op);
// Sets the root node of the tree
// @param op - The operator to assign as root
// @return Status The status code returned
/// \brief Set the root node of the tree
/// \param op - The operator to assign as root
/// \return Status The status code returned
Status AssignRoot(const std::shared_ptr<DatasetOp> &op);
// Start the execution of the tree
// @return Status The status code returned
/// \brief Start the execution of the tree
/// \return Status The status code returned
Status Launch();
/// A print method typically used for debugging
/// /brief A print method typically used for debugging
/// \param out - The output stream to write output to
void Print(std::ostream &out, const std::shared_ptr<DatasetOp> &op = nullptr) const;
// Returns an iterator positioned at the start
// @return Iterator - The iterator
/// \brief Return an iterator positioned at the start
/// \return Iterator - The iterator
ExecutionTree::Iterator begin(const std::shared_ptr<DatasetOp> &root = nullptr) const {
return Iterator(root == nullptr ? root_ : root);
}
// Returns an iterator positioned at the end
// @return Iterator - The iterator
/// \brief Return an iterator positioned at the end
/// \return Iterator - The iterator
ExecutionTree::Iterator end() const { return Iterator(nullptr); }
// << Stream output operator overload
// @notes This allows you to write the debug print info using stream operators
// @param out - reference to the output stream being overloaded
// @param exe_tree - reference to the execution tree to display
// @return - the output stream must be returned
/// \brief << Stream output operator overload
/// \notes This allows you to write the debug print info using stream operators
/// \param out - reference to the output stream being overloaded
/// \param exe_tree - reference to the execution tree to display
/// \return - the output stream must be returned
friend std::ostream &operator<<(std::ostream &out, ExecutionTree &exe_tree) {
exe_tree.Print(out);
return out;
}
// Given the number of workers, launches the worker entry function for each. Essentially a
// wrapper for the TaskGroup handling that is stored inside the execution tree.
// @param num_workers - The number of workers to launch
// @param func - The function entry point that workers will execute
// @param name - The description of worker to launch
// @param op_id - The id of corresponding operator, if not inherit from dataset op then it is -1.
// @return Status The status code returned
/// \brief Given the number of workers, launches the worker entry function for each. Essentially a
/// wrapper for the TaskGroup handling that is stored inside the execution tree.
/// \param num_workers - The number of workers to launch
/// \param func - The function entry point that workers will execute
/// \param name - The description of worker to launch
/// \param op_id - The id of corresponding operator, if not inherit from dataset op then it is -1.
/// \return Status The status code returned
Status LaunchWorkers(int32_t num_workers, std::function<Status(uint32_t)> func, std::string name = "",
int32_t operator_id = -1);
// Getter method
// @return shared_ptr to the root operator
/// \brief Getter method
/// \return shared_ptr to the root operator
std::shared_ptr<DatasetOp> root() const { return root_; }
// Getter method
// @return the prepare flags
uint32_t PrepareFlags() const { return prepare_flags_; }
// Compulsory transformation/action pre optimization.
// @return Status The status code returned
Status PreAction();
// Compulsory transformation/action post optimization.
// @return Status The status code returned
Status PostAction();
// The DEPRECATED driver of the prepare phase of the execution tree. The prepare phase will recursively
// walk the tree to perform modifications to the tree or specific nodes within the tree to get
// it ready for execution.
// @param Total number of epochs that will be run on this tree
// @return Status The status code returned
/// \brief The prepare phase walks the tree in post-order to perform modifications to get it ready for execution.
/// \return Status The status code returned
Status Prepare();
// Recursive function used during prepare phase to visit a node and drive any pre- and post-
// node actions during a tree walk.
// @param op - The dataset op to work on
// @return Status The status code returned
Status PrepareNode(const std::shared_ptr<DatasetOp> &dataset_op);
// Return the pointer to the TaskGroup
// @return raw pointer to the TaskGroup
/// \brief Return the pointer to the TaskGroup
/// \return raw pointer to the TaskGroup
TaskGroup *const AllTasks() const { return tg_.get(); }
// Return if the ExecutionTree is at end of epoch status
// @return bool - true is ExecutionTree is end of epoch status
/// \brief Return if the ExecutionTree is at end of epoch status
/// \return bool - true is ExecutionTree is end of epoch status
bool IsEpochEnd() const { return tree_state_ == TreeState::kDeTStateEpochEnd; }
// Set the ExecutionTree to EOE state
/// \brief Set the ExecutionTree to EOE state
void SetEpochEnd() { tree_state_ = TreeState::kDeTStateEpochEnd; }
// Set the ExecutionTree to executing state
/// \brief Set the ExecutionTree to executing state
void SetExecuting() { tree_state_ = TreeState::kDeTStateExecuting; }
// Return if the ExecutionTree is finished (iterator receives EOF).
// @return Bool - true is ExecutionTree is finished
bool isFinished() const { return tree_state_ == TreeState::kDeTStateFinished; }
// Return if the ExecutionTree is ready.
// @return Bool - true is ExecutionTree is ready
bool isPrepared() const {
return tree_state_ == TreeState::kDeTStateReady || tree_state_ == kDeTStateExecuting ||
tree_state_ == kDeTStateFinished;
}
// Set the ExecutionTree to Finished state.
/// \brief Set the ExecutionTree to Finished state.
void SetFinished() { tree_state_ = TreeState::kDeTStateFinished; }
// Getter for profiling manager, no ownership
/// \brief Return if the ExecutionTree is finished (iterator receives EOF).
/// \return Bool - true is ExecutionTree is finished
bool isFinished() const { return tree_state_ == TreeState::kDeTStateFinished; }
/// \brief Return if the ExecutionTree is ready.
/// \return Bool - true is ExecutionTree is ready
bool isPrepared() const {
return tree_state_ == TreeState::kDeTStatePrepared || tree_state_ == TreeState::kDeTStateExecuting ||
tree_state_ == TreeState::kDeTStateFinished;
}
/// \brief Getter for profiling manager, no ownership
ProfilingManager *GetProfilingManager() { return profiling_manager_.get(); }
private:
// A helper functions for doing the recursive printing
// @param dataset_op - The dataset op to print
// @param indent - an indent string for aligning child levels in output
// @param last - an indicator if it's the last child or not
// @param detailed - should it display the detailed node output or the summary line
/// \brief A helper functions for doing the recursive printing
/// \param dataset_op - The dataset op to print
/// \param indent - an indent string for aligning child levels in output
/// \param last - an indicator if it's the last child or not
/// \param detailed - should it display the detailed node output or the summary line
void PrintNode(std::ostream &out, const std::shared_ptr<DatasetOp> &dataset_op, std::string indent, bool last,
bool detailed) const;

12
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc
View File

@ -24,6 +24,7 @@
#include "minddata/dataset/engine/datasetops/source/tf_reader_op.h"
#include "minddata/dataset/engine/jagged_connector.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/util/status.h"
#include "utils/system/crc32c.h"
@ -246,5 +247,16 @@ Status TFRecordNode::MakeSimpleProducer() {
return Status::OK();
}
// Visitor accepting method for IRNodePass
Status TFRecordNode::Accept(IRNodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->Visit(shared_from_base<TFRecordNode>(), modified);
}
// Visitor accepting method for IRNodePass
Status TFRecordNode::AcceptAfter(IRNodePass *p, bool *const modified) {
// Downcast shared pointer then call visitor
return p->VisitAfter(shared_from_base<TFRecordNode>(), modified);
}
} // namespace dataset
} // namespace mindspore

14
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.h
View File

@ -30,6 +30,8 @@ namespace dataset {
/// \class TFRecordNode
/// \brief A Dataset derived class to represent TFRecord dataset
class TFRecordNode : public NonMappableSourceNode {
friend class CacheValidationPass;
public:
/// \brief Constructor
/// \note Parameter 'schema' is the path to the schema file
@ -140,6 +142,18 @@ class TFRecordNode : public NonMappableSourceNode {
/// \return Status of the function
Status MakeSimpleProducer() override;
/// \brief Base-class override for accepting IRNodePass visitor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status Accept(IRNodePass *p, bool *const modified) override;
/// \brief Base-class override for accepting IRNodePass visitor
/// \param[in] p The node to visit
/// \param[out] modified Indicator if the node was modified
/// \return Status of the node visit
Status AcceptAfter(IRNodePass *p, bool *const modified) override;
private:
std::vector<std::string> dataset_files_;
std::string schema_path_; // schema_path_ path to schema file. It is set when type of schema parameter is string

336
mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc
View File

@ -43,6 +43,7 @@
#include "minddata/dataset/engine/ir/datasetops/source/generator_node.h"
#endif
#include "minddata/dataset/engine/ir/datasetops/source/random_node.h"
#include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h"
#ifdef ENABLE_PYTHON
#include "minddata/dataset/engine/ir/datasetops/sync_wait_node.h"
#endif
@ -50,53 +51,6 @@
#include "minddata/dataset/engine/ir/datasetops/transfer_node.h"
#include "minddata/dataset/engine/ir/datasetops/zip_node.h"
//////////////////////////////////
// This section of code will be removed once the migration of optimizer from DatasetOp to DatasetNode is done.
#include "minddata/dataset/engine/datasetops/batch_op.h"
#include "minddata/dataset/engine/datasetops/build_vocab_op.h"
#ifndef ENABLE_ANDROID
#include "minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h"
#include "minddata/dataset/engine/datasetops/cache_op.h"
#include "minddata/dataset/engine/datasetops/cache_merge_op.h"
#include "minddata/dataset/engine/datasetops/cache_lookup_op.h"
#endif
#include "minddata/dataset/engine/datasetops/concat_op.h"
#include "minddata/dataset/engine/datasetops/dataset_op.h"
#include "minddata/dataset/engine/datasetops/device_queue_op.h"
#include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h"
#include "minddata/dataset/engine/datasetops/map_op/map_op.h"
#include "minddata/dataset/engine/datasetops/project_op.h"
#include "minddata/dataset/engine/datasetops/rename_op.h"
#include "minddata/dataset/engine/datasetops/repeat_op.h"
#include "minddata/dataset/engine/datasetops/skip_op.h"
#include "minddata/dataset/engine/datasetops/shuffle_op.h"
#include "minddata/dataset/engine/datasetops/source/album_op.h"
#include "minddata/dataset/engine/datasetops/source/celeba_op.h"
#include "minddata/dataset/engine/datasetops/source/cifar_op.h"
#include "minddata/dataset/engine/datasetops/source/coco_op.h"
#include "minddata/dataset/engine/datasetops/source/manifest_op.h"
#ifndef ENABLE_ANDROID
#include "minddata/dataset/engine/datasetops/source/mindrecord_op.h"
#endif
#include "minddata/dataset/engine/datasetops/source/mnist_op.h"
#include "minddata/dataset/engine/datasetops/source/random_data_op.h"
#ifndef ENABLE_ANDROID
#include "minddata/dataset/engine/datasetops/source/tf_reader_op.h"
#include "minddata/dataset/engine/datasetops/source/clue_op.h"
#include "minddata/dataset/engine/datasetops/source/csv_op.h"
#include "minddata/dataset/engine/datasetops/source/text_file_op.h"
#endif
#ifdef ENABLE_PYTHON
#include "minddata/dataset/engine/datasetops/source/voc_op.h"
#include "minddata/dataset/engine/datasetops/filter_op.h"
#include "minddata/dataset/engine/datasetops/source/generator_op.h"
#endif
#include "minddata/dataset/engine/datasetops/source/image_folder_op.h"
#include "minddata/dataset/engine/datasetops/take_op.h"
#include "minddata/dataset/engine/datasetops/zip_op.h"
//////////////////////////////////
namespace mindspore {
namespace dataset {
@ -295,6 +249,12 @@ Status IRNodePass::Visit(std::shared_ptr<TakeNode> node, bool *const modified) {
Status IRNodePass::VisitAfter(std::shared_ptr<TakeNode> node, bool *const modified) {
return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);
}
Status IRNodePass::Visit(std::shared_ptr<TFRecordNode> node, bool *const modified) {
return Visit(std::static_pointer_cast<NonMappableSourceNode>(node), modified);
}
Status IRNodePass::VisitAfter(std::shared_ptr<TFRecordNode> node, bool *const modified) {
return VisitAfter(std::static_pointer_cast<NonMappableSourceNode>(node), modified);
}
Status IRNodePass::Visit(std::shared_ptr<TransferNode> node, bool *const modified) {
return Visit(std::static_pointer_cast<DatasetNode>(node), modified);
}
@ -332,287 +292,5 @@ Status IRNodePass::Visit(std::shared_ptr<MappableSourceNode> node, bool *const m
Status IRNodePass::Visit(std::shared_ptr<NonMappableSourceNode> node, bool *const modified) {
return Visit(std::static_pointer_cast<DatasetNode>(node), modified);
}
//////////////////////////////////
// This section of code will be removed once the migration of optimizer from DatasetOp to DatasetNode is done.
// Driver method for TreePass
Status TreePass::Run(ExecutionTree *tree, bool *const modified) {
if (tree == nullptr || modified == nullptr) {
return Status(StatusCode::kMDUnexpectedError, "Null pointer passed to TreePass");
}
// Initialize modified flag
*modified = false;
return this->RunOnTree(tree, modified);
}
// Driver method for NodePass
Status NodePass::Run(ExecutionTree *tree, bool *const modified) {
if (tree == nullptr || modified == nullptr) {
return Status(StatusCode::kMDUnexpectedError, "Null pointer passed to NodePass");
}
// Initialize modified flag
*modified = false;
std::shared_ptr<DatasetOp> root = tree->root();
if (traversalOrder_ == Order::DFS) {
// DFS
return DFSNodeVisit(root, modified);
} else if (traversalOrder_ == Order::BFS) {
// BFS
return BFSNodeVisit(root, modified);
}
return Status::OK();
}
// Helper function to perform DFS visit
Status NodePass::DFSNodeVisit(std::shared_ptr<DatasetOp> node, bool *const modified) {
RETURN_IF_NOT_OK(node->PreAccept(this, modified));
for (const auto &c : node->Children()) {
RETURN_IF_NOT_OK(this->DFSNodeVisit(c, modified));
}
return node->Accept(this, modified);
}
// Helper function to perform BFS visit
Status NodePass::BFSNodeVisit(std::shared_ptr<DatasetOp> root, bool *const modified) {
// Initialize bfs queue with root
std::queue<std::shared_ptr<DatasetOp>> bfsQueue;
bfsQueue.push(root);
// BFS loop
while (!bfsQueue.empty()) {
// Pop the front of the bfs queue
auto curNode = bfsQueue.front();
bfsQueue.pop();
// Run node pass
RETURN_IF_NOT_OK(curNode->Accept(this, modified));
// Push children into bfs queue
for (const auto &c : curNode->Children()) {
bfsQueue.push(c);
}
}
return Status::OK();
}
Status NodePass::RunOnNode(std::shared_ptr<BatchOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<MapOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<ProjectOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<RenameOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<SkipOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<ShuffleOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<RandomDataOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<TakeOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<ZipOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<DeviceQueueOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<ImageFolderOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<AlbumOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<MnistOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<CifarOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<CelebAOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<CocoOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<RepeatOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<EpochCtrlOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::PreRunOnNode(std::shared_ptr<RepeatOp> node, bool *const modified) {
// Fallback to base class visitor by default
return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::PreRunOnNode(std::shared_ptr<EpochCtrlOp> node, bool *const modified) {
// Fallback to base class visitor by default
return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::PreRunOnNode(std::shared_ptr<BuildVocabOp> node, bool *const modified) {
// Fallback to base class visitor by default
return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::PreRunOnNode(std::shared_ptr<ZipOp> node, bool *const modified) {
// Fallback to base class visitor by default
return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::PreRunOnNode(std::shared_ptr<MapOp> node, bool *const modified) {
// Fallback to base class visitor by default
return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::PreRunOnNode(std::shared_ptr<ConcatOp> node, bool *const modified) {
// Fallback to base class visitor by default
return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::PreRunOnNode(std::shared_ptr<TakeOp> node, bool *const modified) {
// Fallback to base class visitor by default
return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::PreRunOnNode(std::shared_ptr<SkipOp> node, bool *const modified) {
// Fallback to base class visitor by default
return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::PreRunOnNode(std::shared_ptr<BatchOp> node, bool *const modified) {
// Fallback to base class visitor by default
return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
#ifndef ENABLE_ANDROID
Status NodePass::RunOnNode(std::shared_ptr<MindRecordOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<TFReaderOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<CacheOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<CacheMergeOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<CacheLookupOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<ClueOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<CsvOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<TextFileOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::PreRunOnNode(std::shared_ptr<CacheOp> node, bool *const modified) {
// Fallback to base class visitor by default
return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::PreRunOnNode(std::shared_ptr<CacheMergeOp> node, bool *const modified) {
// Fallback to base class visitor by default
return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::PreRunOnNode(std::shared_ptr<BuildSentencePieceVocabOp> node, bool *const modified) {
// Fallback to base class visitor by default
return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
#endif
#ifdef ENABLE_PYTHON
Status NodePass::RunOnNode(std::shared_ptr<FilterOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<GeneratorOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<ManifestOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<VOCOp> node, bool *const modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::PreRunOnNode(std::shared_ptr<FilterOp> node, bool *const modified) {
// Fallback to base class visitor by default
return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
#endif
//////////////////////////////////
} // namespace dataset
} // namespace mindspore

199
mindspore/ccsrc/minddata/dataset/engine/opt/pass.h
View File

@ -46,6 +46,7 @@ class RootNode;
class ShuffleNode;
class SkipNode;
class TakeNode;
class TFRecordNode;
class TransferNode;
class ZipNode;
#ifdef ENABLE_PYTHON
@ -76,47 +77,6 @@ class TextFileNode;
class TFRecordNode;
#endif
//////////////////////////////////
// This section of code will be removed once the migration of optimizer from DatasetOp to DatasetNode is done.
class BatchOp;
class MapOp;
class ProjectOp;
class RenameOp;
class SkipOp;
class ShuffleOp;
class AlbumOp;
class RandomDataOp;
class RepeatOp;
class TakeOp;
class ZipOp;
class DeviceQueueOp;
class ImageFolderOp;
class MnistOp;
class ManifestOp;
class CifarOp;
class VOCOp;
class CocoOp;
class CelebAOp;
class EpochCtrlOp;
class BuildVocabOp;
class ConcatOp;
#ifndef ENABLE_ANDROID
class MindRecordOp;
class TFReaderOp;
class CacheOp;
class CacheMergeOp;
class CacheLookupOp;
class BuildSentencePieceVocabOp;
class ClueOp;
class CsvOp;
class TextFileOp;
#endif
#ifdef ENABLE_PYTHON
class FilterOp;
class GeneratorOp;
#endif
//////////////////////////////////
// The base class Pass is the basic unit of tree transformation.
// The actual implementation of the passes will be derived from here.
class IRPass : public std::enable_shared_from_this<IRPass> {
@ -200,6 +160,10 @@ class IRNodePass : public IRPass {
virtual Status VisitAfter(std::shared_ptr<BatchNode> node, bool *const modified);
virtual Status Visit(std::shared_ptr<BucketBatchByLengthNode> node, bool *const modified);
virtual Status VisitAfter(std::shared_ptr<BucketBatchByLengthNode> node, bool *const modified);
#ifndef ENABLE_ANDROID
virtual Status Visit(std::shared_ptr<BuildSentenceVocabNode> node, bool *const modified);
virtual Status VisitAfter(std::shared_ptr<BuildSentenceVocabNode> node, bool *const modified);
#endif
virtual Status Visit(std::shared_ptr<BuildVocabNode> node, bool *const modified);
virtual Status VisitAfter(std::shared_ptr<BuildVocabNode> node, bool *const modified);
virtual Status Visit(std::shared_ptr<ConcatNode> node, bool *const modified);
@ -240,20 +204,18 @@ class IRNodePass : public IRPass {
virtual Status VisitAfter(std::shared_ptr<ShuffleNode> node, bool *const modified);
virtual Status Visit(std::shared_ptr<SkipNode> node, bool *const modified);
virtual Status VisitAfter(std::shared_ptr<SkipNode> node, bool *const modified);
virtual Status Visit(std::shared_ptr<TakeNode> node, bool *const modified);
virtual Status VisitAfter(std::shared_ptr<TakeNode> node, bool *const modified);
virtual Status Visit(std::shared_ptr<TransferNode> node, bool *const modified);
virtual Status VisitAfter(std::shared_ptr<TransferNode> node, bool *const modified);
virtual Status Visit(std::shared_ptr<ZipNode> node, bool *const modified);
virtual Status VisitAfter(std::shared_ptr<ZipNode> node, bool *const modified);
#ifdef ENABLE_PYTHON
virtual Status Visit(std::shared_ptr<SyncWaitNode> node, bool *const modified);
virtual Status VisitAfter(std::shared_ptr<SyncWaitNode> node, bool *const modified);
#endif
#ifndef ENABLE_ANDROID
virtual Status Visit(std::shared_ptr<BuildSentenceVocabNode> node, bool *const modified);
virtual Status VisitAfter(std::shared_ptr<BuildSentenceVocabNode> node, bool *const modified);
#endif
virtual Status Visit(std::shared_ptr<TakeNode> node, bool *const modified);
virtual Status VisitAfter(std::shared_ptr<TakeNode> node, bool *const modified);
virtual Status Visit(std::shared_ptr<TFRecordNode> node, bool *const modified);
virtual Status VisitAfter(std::shared_ptr<TFRecordNode> node, bool *const modified);
virtual Status Visit(std::shared_ptr<TransferNode> node, bool *const modified);
virtual Status VisitAfter(std::shared_ptr<TransferNode> node, bool *const modified);
virtual Status Visit(std::shared_ptr<ZipNode> node, bool *const modified);
virtual Status VisitAfter(std::shared_ptr<ZipNode> node, bool *const modified);
// leaf-IR Node
virtual Status Visit(std::shared_ptr<MappableSourceNode> node, bool *const modified);
@ -269,141 +231,6 @@ class IRNodePass : public IRPass {
// Tree traversal order of the NodePass
Order traversalOrder_;
};
//////////////////////////////////
// This section of code will be removed once the migration of optimizer from DatasetOp to DatasetNode is done.
// The base class Pass is the basic unit of tree transformation.
// The actual implementation of the passes will be derived from here.
class Pass : public std::enable_shared_from_this<Pass> {
public:
// Run the transformation pass against the execution tree.
// @param tree - Pointer to the execution tree to be transformed.
// @param modified - Pointer to the modified flag,
virtual Status Run(ExecutionTree *tree, bool *const modified) = 0;
virtual ~Pass() = default;
};
// TreePass is a basic Pass class which performs transformation on ExecutionTree directly.
class TreePass : public Pass {
public:
/// \brief Run the transformation pass against the execution tree.
/// \param[in,out] tree Pointer to the execution tree to be transformed.
/// \param[in,out] modified Indicate if the tree was modified
Status Run(ExecutionTree *tree, bool *const modified) final;
/// \brief Derived classes may implement the runOnTree function to implement tree transformation.
/// "modified" flag needs to be set to true if tree is modified during the pass execution.
/// \param[in,out] tree The tree to operate on.
/// \param[in,out] Indicate of the tree was modified.
/// \return Status The status code returned
virtual Status RunOnTree(ExecutionTree *tree, bool *const modified) { return Status::OK(); }
};
// NodePass is a base Pass class which performs transformation on node visiting.
// NodePass implements Visitor design pattern.
// The visiting happens twice for each node in the DFS traversal, one on the way down of the traversal,
// and the other when all the descending nodes are visited.
// Actual transformation is done by implementing a new derived class of NodePass.
// The derived class will implement the method Visit()/VisitAfter() passing specified node types
// it wants to action on them, overriding the ones defined in NodePass.
// If the derived class wants to perform the same action on all node types,
// it can simply implement the method Visit()/VisitAfter() passing the base class DatasetNode.
// This is made possible by overloading the method Visit()/VisitAfter() on each node type to fall back
// to call the Visit()/VisitAfter() in this parent NodePass class.
class NodePass : public Pass {
public:
// Tree traversal order
enum Order { DFS, BFS };
// Constructor
// Default DFS traversal
explicit NodePass(Order order = Order::DFS) { traversalOrder_ = order; }
~NodePass() = default;
/// \brief Run the transformation pass against the execution tree
/// \param[in,out] tree Pointer to the execution tree to be transformed
/// \param[in,out] modified Indicator if the tree was changed
Status Run(ExecutionTree *tree, bool *const modified) final;
/// \brief Derived classes may implement the PreRunOnNode function to implement any initial visit work on the way down
/// a tree traversal. "modified" flag needs to be set to true if tree is modified during the pass execution
/// \param[in] node The node being visited
/// \param[out] modified Indicator if the node was changed at all
/// \return Status The status code returned
virtual Status PreRunOnNode(std::shared_ptr<DatasetOp> node, bool *const modified) { return Status::OK(); }
/// \brief Derived classes may implement the RunOnNode function to implement node level tree transformation
/// "modified" flag needs to be set to true if tree is modified during the pass execution
/// \param[in] node The node being visited
/// \param[out] modified Indicator if the node was changed at all.
/// \return Status The status code returned
virtual Status RunOnNode(std::shared_ptr<DatasetOp> node, bool *const modified) { return Status::OK(); }
// Visit methods to be overridden.
// Note that member template can not be virtual, any op which wants to work with NodePass should declare RunOnNode
// of its own type and override "Accept" from DatasetOp.
virtual Status RunOnNode(std::shared_ptr<BatchOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<MapOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<ProjectOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<RenameOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<SkipOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<ShuffleOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<RandomDataOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<AlbumOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<TakeOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<ZipOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<DeviceQueueOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<ImageFolderOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<MnistOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<CifarOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<CocoOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<CelebAOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<RepeatOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<EpochCtrlOp> node, bool *const modified);
virtual Status PreRunOnNode(std::shared_ptr<RepeatOp> node, bool *const modified);
virtual Status PreRunOnNode(std::shared_ptr<EpochCtrlOp> node, bool *const modified);
virtual Status PreRunOnNode(std::shared_ptr<BuildVocabOp> node, bool *const modified);
virtual Status PreRunOnNode(std::shared_ptr<ZipOp> node, bool *const modified);
virtual Status PreRunOnNode(std::shared_ptr<MapOp> node, bool *const modified);
virtual Status PreRunOnNode(std::shared_ptr<ConcatOp> node, bool *const modified);
virtual Status PreRunOnNode(std::shared_ptr<TakeOp> node, bool *const modified);
virtual Status PreRunOnNode(std::shared_ptr<SkipOp> node, bool *const modified);
virtual Status PreRunOnNode(std::shared_ptr<BatchOp> node, bool *const modified);
#ifndef ENABLE_ANDROID
virtual Status RunOnNode(std::shared_ptr<MindRecordOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<TFReaderOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<CacheMergeOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<CacheLookupOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<CacheOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<ClueOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<CsvOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<TextFileOp> node, bool *const modified);
virtual Status PreRunOnNode(std::shared_ptr<CacheOp> node, bool *const modified);
virtual Status PreRunOnNode(std::shared_ptr<CacheMergeOp> node, bool *const modified);
virtual Status PreRunOnNode(std::shared_ptr<BuildSentencePieceVocabOp> node, bool *const modified);
#endif
#ifdef ENABLE_PYTHON
virtual Status RunOnNode(std::shared_ptr<FilterOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<ManifestOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<GeneratorOp> node, bool *const modified);
virtual Status RunOnNode(std::shared_ptr<VOCOp> node, bool *const modified);
virtual Status PreRunOnNode(std::shared_ptr<FilterOp> node, bool *const modified);
#endif
private:
// Helper function to perform DFS visit
Status DFSNodeVisit(std::shared_ptr<DatasetOp> node, bool *const modified);
// Helper function to perform BFS visit
Status BFSNodeVisit(std::shared_ptr<DatasetOp> root, bool *const modified);
// Tree traversal order of the NodePass
Order traversalOrder_;
};
//////////////////////////////////
} // namespace dataset
} // namespace mindspore

17
mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.cc
View File

@ -23,6 +23,7 @@
#include "minddata/dataset/engine/ir/datasetops/map_node.h"
#include "minddata/dataset/engine/ir/datasetops/repeat_node.h"
#include "minddata/dataset/engine/ir/datasetops/skip_node.h"
#include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h"
#include "minddata/dataset/engine/ir/datasetops/take_node.h"
#include "minddata/dataset/engine/ir/datasetops/zip_node.h"
@ -157,6 +158,22 @@ Status CacheValidationPass::VisitAfter(std::shared_ptr<RepeatNode> node, bool *c
return Status::OK();
}
Status CacheValidationPass::VisitAfter(std::shared_ptr<TFRecordNode> node, bool *const modified) {
MS_LOG(DEBUG) << "CacheValidationPass::VisitAfter(<TFRecordNode>): visiting " << node->Name() << ".";
if (!is_cached_) {
// If we are not in a cache path, then we must validate the file-based sharding config.
// If we are in a cache path, there is no file-based sharding so the check is not required.
if (!node->shard_equal_rows_ && node->dataset_files_.size() < static_cast<uint32_t>(node->num_shards_)) {
RETURN_STATUS_UNEXPECTED("Invalid file, not enough tfrecord files provided.\n");
}
}
// Reset the flag when this node is cached and is already visited
if (node->IsCached()) {
is_cached_ = false;
}
return Status::OK();
}
Status CacheValidationPass::VisitAfter(std::shared_ptr<DatasetNode> node, bool *const modified) {
MS_LOG(DEBUG) << "CacheValidationPass::VisitAfter(<DatasetNode>): visiting " << node->Name() << ".";
// Reset the flag when all descendants are visited

26
mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.h
View File

@ -37,61 +37,67 @@ class CacheValidationPass : public IRNodePass {
/// \brief Returns an error if BatchNode exists under a cache
/// \param[in] node The node being visited
/// \param[inout] modified Indicator if the node was changed at all
/// \param[in,out] modified Indicator if the node was changed at all
/// \return Status The status code returned
Status Visit(std::shared_ptr<BatchNode> node, bool *const modified) override;
/// \brief Returns an error if ConcatNode exists under a cache
/// \param[in] node The node being visited
/// \param[inout] modified Indicator if the node was changed at all
/// \param[in,out] modified Indicator if the node was changed at all
/// \return Status The status code returned
Status Visit(std::shared_ptr<ConcatNode> node, bool *const modified) override;
/// \brief Returns an error if FilterNode exists under a cache
/// \param[in] node The node being visited
/// \param[inout] modified Indicator if the node was changed at all
/// \param[in,out] modified Indicator if the node was changed at all
/// \return Status The status code returned
Status Visit(std::shared_ptr<FilterNode> node, bool *const modified) override;
/// \brief Returns an error if SkipNode exists under a cache
/// \param[in] node The node being visited
/// \param[inout] modified Indicator if the node was changed at all
/// \param[in,out] modified Indicator if the node was changed at all
/// \return Status The status code returned
Status Visit(std::shared_ptr<SkipNode> node, bool *const modified) override;
/// \brief Returns an error if TakeNode exists under a cache
/// \param[in] node The node being visited
/// \param[inout] modified Indicator if the node was changed at all
/// \param[in,out] modified Indicator if the node was changed at all
/// \return Status The status code returned
Status Visit(std::shared_ptr<TakeNode> node, bool *const modified) override;
/// \brief Returns an error if ZipNode exists under a cache
/// \param[in] node The node being visited
/// \param[inout] modified Indicator if the node was changed at all
/// \param[in,out] modified Indicator if the node was changed at all
/// \return Status The status code returned
Status Visit(std::shared_ptr<ZipNode> node, bool *const modified) override;
/// \brief Returns an error if MapNode with non-deterministic tensor operations exists under a cache
/// \param[in] node The node being visited
/// \param[inout] modified Indicator if the node was changed at all
/// \param[in,out] modified Indicator if the node was changed at all
/// \return Status The status code returned
Status Visit(std::shared_ptr<MapNode> node, bool *const modified) override;
/// \brief Returns an error if there is a cache over another cache
/// \param[in] node The node being visited
/// \param[inout] modified Indicator if the node was changed at all
/// \param[in,out] modified Indicator if the node was changed at all
/// \return Status The status code returned
Status Visit(std::shared_ptr<DatasetNode> node, bool *const modified) override;
/// \brief Identifies and block repeat under cache scenarios
/// \param[in] node The node being visited
/// \param[inout] modified Indicator if the node was changed at all
/// \param[in,out] modified Indicator if the node was changed at all
/// \return Status The status code returned
Status VisitAfter(std::shared_ptr<RepeatNode> node, bool *const modified) override;
/// \brief Identifies the subtree above this node as not being cached
/// \param[in] node The node being visited
/// \param[inout] modified Indicator if the node was changed at all
/// \param[in,out] modified Indicator if the node was changed at all
/// \return Status The status code returned
Status VisitAfter(std::shared_ptr<TFRecordNode> node, bool *const modified) override;
/// \brief Identifies the subtree above this node as not being cached
/// \param[in] node The node being visited
/// \param[in,out] modified Indicator if the node was changed at all
/// \return Status The status code returned
Status VisitAfter(std::shared_ptr<DatasetNode> node, bool *const modified) override;