From 46483d89b1082e7604d8c2cb2bcc4fdf78bf8e59 Mon Sep 17 00:00:00 2001 From: Nat Sutyanyong Date: Thu, 28 Jan 2021 17:12:08 -0500 Subject: [PATCH] Completely remove unused ExecTree optimizer code, rewrite ExecutionTree::Prepare() --- .../dataset/engine/datasetops/batch_op.cc | 16 +- .../dataset/engine/datasetops/batch_op.h | 14 +- .../datasetops/bucket_batch_by_length_op.cc | 9 +- .../build_sentence_piece_vocab_op.cc | 9 +- .../build_sentence_piece_vocab_op.h | 9 +- .../engine/datasetops/build_vocab_op.cc | 9 +- .../engine/datasetops/build_vocab_op.h | 8 +- .../engine/datasetops/cache_lookup_op.cc | 9 +- .../engine/datasetops/cache_lookup_op.h | 8 +- .../engine/datasetops/cache_merge_op.cc | 21 +- .../engine/datasetops/cache_merge_op.h | 25 +- .../dataset/engine/datasetops/cache_op.cc | 23 +- .../dataset/engine/datasetops/cache_op.h | 38 +- .../dataset/engine/datasetops/concat_op.cc | 37 +- .../dataset/engine/datasetops/concat_op.h | 8 +- .../dataset/engine/datasetops/dataset_op.cc | 28 +- .../dataset/engine/datasetops/dataset_op.h | 68 ++-- .../engine/datasetops/device_queue_op.cc | 15 +- .../engine/datasetops/device_queue_op.h | 8 +- .../engine/datasetops/epoch_ctrl_op.cc | 12 - .../dataset/engine/datasetops/epoch_ctrl_op.h | 16 +- .../dataset/engine/datasetops/filter_op.cc | 18 +- .../dataset/engine/datasetops/filter_op.h | 14 +- .../engine/datasetops/map_op/map_op.cc | 16 +- .../dataset/engine/datasetops/map_op/map_op.h | 14 +- .../dataset/engine/datasetops/parallel_op.h | 22 +- .../dataset/engine/datasetops/pipeline_op.h | 20 +- .../dataset/engine/datasetops/project_op.cc | 10 +- .../dataset/engine/datasetops/project_op.h | 8 +- .../dataset/engine/datasetops/rename_op.cc | 8 +- .../dataset/engine/datasetops/rename_op.h | 8 +- .../dataset/engine/datasetops/repeat_op.cc | 16 +- .../dataset/engine/datasetops/repeat_op.h | 14 +- .../dataset/engine/datasetops/shuffle_op.cc | 12 +- .../dataset/engine/datasetops/shuffle_op.h | 8 +- .../dataset/engine/datasetops/skip_op.cc | 15 +- .../dataset/engine/datasetops/skip_op.h | 14 +- .../engine/datasetops/source/album_op.cc | 9 +- .../engine/datasetops/source/album_op.h | 8 +- .../engine/datasetops/source/celeba_op.cc | 11 +- .../engine/datasetops/source/celeba_op.h | 10 +- .../engine/datasetops/source/cifar_op.cc | 9 +- .../engine/datasetops/source/cifar_op.h | 8 +- .../engine/datasetops/source/clue_op.cc | 9 +- .../engine/datasetops/source/clue_op.h | 8 +- .../engine/datasetops/source/coco_op.cc | 10 +- .../engine/datasetops/source/coco_op.h | 8 +- .../engine/datasetops/source/csv_op.cc | 9 +- .../dataset/engine/datasetops/source/csv_op.h | 6 - .../engine/datasetops/source/generator_op.cc | 10 +- .../engine/datasetops/source/generator_op.h | 8 +- .../datasetops/source/image_folder_op.cc | 9 +- .../datasetops/source/image_folder_op.h | 8 +- .../engine/datasetops/source/manifest_op.cc | 9 +- .../engine/datasetops/source/manifest_op.h | 8 +- .../engine/datasetops/source/mindrecord_op.cc | 9 +- .../engine/datasetops/source/mindrecord_op.h | 8 +- .../engine/datasetops/source/mnist_op.cc | 9 +- .../engine/datasetops/source/mnist_op.h | 8 +- .../datasetops/source/random_data_op.cc | 9 +- .../engine/datasetops/source/random_data_op.h | 8 +- .../engine/datasetops/source/text_file_op.cc | 10 +- .../engine/datasetops/source/text_file_op.h | 8 +- .../engine/datasetops/source/tf_reader_op.cc | 31 +- .../engine/datasetops/source/tf_reader_op.h | 14 +- .../engine/datasetops/source/voc_op.cc | 9 +- .../dataset/engine/datasetops/source/voc_op.h | 8 +- .../dataset/engine/datasetops/take_op.cc | 15 +- .../dataset/engine/datasetops/take_op.h | 14 +- .../dataset/engine/datasetops/zip_op.cc | 15 +- .../dataset/engine/datasetops/zip_op.h | 14 +- .../minddata/dataset/engine/execution_tree.cc | 54 ++- .../minddata/dataset/engine/execution_tree.h | 141 +++----- .../ir/datasetops/source/tf_record_node.cc | 12 + .../ir/datasetops/source/tf_record_node.h | 14 + .../ccsrc/minddata/dataset/engine/opt/pass.cc | 336 +----------------- .../ccsrc/minddata/dataset/engine/opt/pass.h | 199 +---------- .../engine/opt/pre/cache_validation_pass.cc | 17 + .../engine/opt/pre/cache_validation_pass.h | 26 +- 79 files changed, 303 insertions(+), 1449 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc index 5590ae71d80..eb711b3344e 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ #include "minddata/dataset/engine/datasetops/batch_op.h" #include -#include #include "utils/ms_utils.h" #ifdef ENABLE_PYTHON @@ -24,7 +23,6 @@ #endif #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/kernels/data/data_utils.h" #include "minddata/dataset/util/status.h" @@ -498,18 +496,6 @@ Status BatchOp::UnpackPadInfo(const PadInfo &pad_info, return Status::OK(); } -// Visitor accept method for NodePass -Status BatchOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - -// Visitor pre-accept method for NodePass -Status BatchOp::PreAccept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->PreRunOnNode(shared_from_base(), modified); -} - Status BatchOp::ComputeColMap() { CHECK_FAIL_RETURN_UNEXPECTED(child_.size() == 1, "Batch has " + std::to_string(child_.size()) + " child/children, expects only 1 child."); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.h index baae2a384ea..db82de9cbd7 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -193,18 +193,6 @@ class BatchOp : public ParallelOp { // @return Status The status code returned Status operator()() override; - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status PreAccept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return kBatchOp; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/bucket_batch_by_length_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/bucket_batch_by_length_op.cc index 03f9cf4fbff..f6a81ff1c2c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/bucket_batch_by_length_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/bucket_batch_by_length_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,22 +15,15 @@ */ #include "minddata/dataset/engine/datasetops/bucket_batch_by_length_op.h" -#include #include #include #include #include -#include "pybind11/numpy.h" -#include "pybind11/pybind11.h" -#include "pybind11/stl.h" -#include "minddata/dataset/core/pybind_support.h" #include "minddata/dataset/core/config_manager.h" -#include "minddata/dataset/core/tensor.h" #include "minddata/dataset/core/tensor_shape.h" #include "minddata/dataset/engine/dataset_iterator.h" #include "minddata/dataset/engine/datasetops/parallel_op.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/util/status.h" namespace py = pybind11; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.cc index 8053baa5676..98ea0aca1a4 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,6 @@ #include #include "minddata/dataset/core/config_manager.h" -#include "minddata/dataset/engine/opt/pass.h" namespace mindspore { namespace dataset { @@ -160,12 +159,6 @@ void BuildSentencePieceVocabOp::Next(std::string *sentence) { ret_status_ = Status::OK(); } -// Pre-Visitor accept method for NodePass -Status BuildSentencePieceVocabOp::PreAccept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call the pre-visitation - return p->PreRunOnNode(shared_from_base(), modified); -} - Status BuildSentencePieceVocabOp::Builder::Build(std::shared_ptr *op) { (*op) = std::make_shared(builder_vocab_, builder_col_names_, builder_vocab_size_, builder_character_coverage_, builder_model_type_, builder_params_, diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h index 073122e1ce9..7583c306090 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -63,7 +63,7 @@ class BuildSentencePieceVocabOp : public PipelineOp { } // Setter method - // @param float charactor corverage - to determine the minimum symbols + // @param float character corverage - to determine the minimum symbols // @return Builder & reference to builder class object Builder &SetCharacterCoverage(float character_coverage) { builder_character_coverage_ = character_coverage; @@ -166,11 +166,6 @@ class BuildSentencePieceVocabOp : public PipelineOp { bool Done(); void Next(std::string *sentence); - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status PreAccept(NodePass *p, bool *const modified) override; - private: bool read_done_; Status ret_status_; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.cc index 6d915abe889..3d5f9299525 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,6 @@ #include #include #include "minddata/dataset/core/config_manager.h" -#include "minddata/dataset/engine/opt/pass.h" namespace mindspore { namespace dataset { @@ -230,11 +229,5 @@ void BuildVocabOp::Print(std::ostream &out, bool show_all) const { << "\n\n"; } } - -// Pre-Visitor accept method for NodePass -Status BuildVocabOp::PreAccept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call the pre-visitation - return p->PreRunOnNode(shared_from_base(), modified); -} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.h index 757a1d12cbe..2034bb708d4 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -168,12 +168,6 @@ class BuildVocabOp : public ParallelOp { Status Reset() override { RETURN_STATUS_UNEXPECTED("Reset shouldn't be called in BuildVocabOp"); } - /// \brief Base-class override for NodePass pre-visit acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status PreAccept(NodePass *p, bool *const modified) override; - private: const int32_t interval_; bool special_first_; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc index 7beb6c15b07..cafe185af2b 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,6 @@ * limitations under the License. */ #include "minddata/dataset/engine/datasetops/cache_lookup_op.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/constants.h" #include "minddata/dataset/core/global_context.h" @@ -133,11 +132,5 @@ Status CacheLookupOp::ComputeColMap() { // So we will just return OK and let MergeOp (our parent) to handle it. return Status::OK(); } - -// Visitor accept method for NodePass -Status CacheLookupOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.h index 9905419047e..f3211ddd384 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -103,12 +103,6 @@ class CacheLookupOp : public CacheBase, public SamplerRT { bool AllowCacheMiss() override { return true; } std::string Name() const override { return kCacheLookupOp; } - /// \brief Base-class override for NodePass visitor acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status Accept(NodePass *p, bool *const modified) override; - protected: Status ComputeColMap() override; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc index e02eecbd7c9..e40ac6dc9bf 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,6 @@ #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/constants.h" #include "minddata/dataset/core/global_context.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/util/system_pool.h" #include "minddata/dataset/util/task_manager.h" @@ -205,10 +204,10 @@ Status CacheMergeOp::Cleaner() { return Status::OK(); } -Status CacheMergeOp::PrepareNodePostAction() { // Run any common code from super class first before adding our own - // specific logic +Status CacheMergeOp::PrepareOperator() { // Run any common code from super class first before adding our own + // specific logic CHECK_FAIL_RETURN_UNEXPECTED(child_.size() == 2, "Incorrect number of children"); - RETURN_IF_NOT_OK(ParallelOp::PrepareNodePostAction()); + RETURN_IF_NOT_OK(DatasetOp::PrepareOperator()); // Get the computed check sum from all ops in the cache miss class uint32_t cache_crc = DatasetOp::GenerateCRC(child_[kCacheMissChildIdx]); // This is a mappable cache op so the id's need to be generated. @@ -263,18 +262,6 @@ Status CacheMergeOp::Builder::Build(std::shared_ptr *ptr) { return Status::OK(); } -// Pre-Visitor accept method for NodePass -Status CacheMergeOp::PreAccept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call the pre-visitation - return p->PreRunOnNode(shared_from_base(), modified); -} - -// Visitor accept method for NodePass -Status CacheMergeOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - Status CacheMergeOp::EoeReceived(int32_t worker_id) { // Send the eoe up. MS_LOG(DEBUG) << "Cache merge sending eoe"; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.h index f3dc3a0190f..42f33dd1eca 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -144,7 +144,7 @@ class CacheMergeOp : public ParallelOp { /// \param numWorkers Number of parallel workers as a derived class of ParallelOp /// \param opConnector Size Connector size as a derived class of ParallelOp /// \param numCleaners Number of cleaners to move cache miss rows into the cache server - /// \param cache_client CacheClient to commmunicate with the Cache server + /// \param cache_client CacheClient to communicate with the Cache server /// \param sampler as a derived class of ParallelOp CacheMergeOp(int32_t numWorkers, int32_t opConnectorSize, int32_t numCleaners, std::shared_ptr cache_client, const std::shared_ptr &sampler); @@ -157,31 +157,24 @@ class CacheMergeOp : public ParallelOp { return out; } /// \brief Master thread responsible to spawn all the necessary worker threads for the two streams and - /// the threads for the cleaners. + /// the threads for the cleaners. /// \return Status operator()() override; + /// \brief Entry function for worker thread that fetch rows from CacheLookupOp /// \param workerId /// \return Status object Status WorkerEntry(int32_t workerId) override; - Status PrepareNodePostAction() override; + + /// \brief Perform specific post-operations on CacheOp + /// \return Status The status code returned + Status PrepareOperator() override; + /// \brief Entry function for worker thread that fetch rows from the cache miss stream /// \param workerId /// \return Status object Status CacheMissWorkerEntry(int32_t workerId); - /// \brief Base-class override for NodePass pre-visit acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status PreAccept(NodePass *p, bool *const modified) override; - - /// \brief Base-class override for NodePass visitor acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status Accept(NodePass *p, bool *const modified) override; - /// \brief Base-class override for eoe handling /// \param worker_id /// \return Status object diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc index 5c80a5523aa..ebd00d153cd 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,14 +17,12 @@ #include #include -#include #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/constants.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/datasetops/repeat_op.h" #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/task_manager.h" @@ -207,9 +205,6 @@ Status CacheOp::RegisterResources() { return Status::OK(); } -// Base-class override for setting specific CacheOp configurations. This code will be called -// during the execution tree prepare phase BEFORE traversing down to child operators. -uint32_t CacheOp::PrepareFlags() const { return ExecutionTree::kDePrepCache; } // Base-class override for special eoe handler. // CacheOp must override this because it shall not perform default handling of eoe. Instead // the CacheOp manages actions related to the end of the epoch. @@ -225,21 +220,9 @@ Status CacheOp::EofReceived(int32_t worker_id) { return Status::OK(); } -// Pre-Visitor accept method for NodePass -Status CacheOp::PreAccept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call the pre-visitation - return p->PreRunOnNode(shared_from_base(), modified); -} - -// Visitor accept method for NodePass -Status CacheOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - -Status CacheOp::PrepareNodePostAction() { +Status CacheOp::PrepareOperator() { // Run any common code from super class first before adding our own - RETURN_IF_NOT_OK(ParallelOp::PrepareNodePostAction()); + RETURN_IF_NOT_OK(DatasetOp::PrepareOperator()); // Get the computed check sum from all ops in our cache path below us and ask the cache op to create it's cache uint32_t cache_crc = DatasetOp::GenerateCRC(shared_from_this()); // This is a non-mappable cache op so the id's need to be generated. diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.h index 30d7ea6ab6d..fd7045ca595 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -113,35 +113,35 @@ class CacheOp : public CacheBase, public RandomAccessOp { // Destructor ~CacheOp(); - /// \brief Base-class override for setting specific CacheOp configurations. This code will be called - /// during the execution tree prepare phase BEFORE traversing down to child operators. - uint32_t PrepareFlags() const override; /// \brief Base-class override for special eoe handler. - /// CacheOp must override this because it shall not perform default handling of eoe. Instead - /// the CacheOp manages actions related to the end of the epoch. + /// \notes CacheOp must override this because it shall not perform default handling of eoe. Instead + /// the CacheOp manages actions related to the end of the epoch. /// \return Status The status code returned Status EoeReceived(int32_t worker_id) override; - /// \brief Base-class override for NodePass pre-visit acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status PreAccept(NodePass *p, bool *const modified) override; - /// \brief Base-class override for NodePass visitor acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status Accept(NodePass *p, bool *const modified) override; + /// \brief Base-class override for handling cases when an eof is received. /// \param worker_id - The worker id /// \return Status The status code returned Status EofReceived(int32_t worker_id) override; + + // \brief Class functor operator (). + /// \return Status The status code returned Status operator()() override; + + /// \brief Entry function for worker thread that fetch rows from CacheLookupOp + /// \param workerId + /// \return Status The status code returned Status WorkerEntry(int32_t worker_id) override; - /// \brief Base-class override for handling cases if we allow cache miss + + /// \brief Base-class override for handling cases if we allow cache miss. bool AllowCacheMiss() override { return false; } - /// \brief Base-class override for the name of this operator + + /// \brief Base-class override for the name of this operator. std::string Name() const override { return kCacheOp; } - Status PrepareNodePostAction() override; + + /// \brief Perform specific post-operations on CacheOp + /// \return Status The status code returned + Status PrepareOperator() override; private: WaitPost rows_cache_done_; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.cc index 9284346147a..abb808c91ed 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,6 @@ #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" -#include "minddata/dataset/engine/opt/pass.h" #include "utils/ms_utils.h" namespace mindspore { @@ -70,6 +69,13 @@ void ConcatOp::Print(std::ostream &out, bool show_all) const { } } +// This definition is added to pass the cyclomatic complexity rule of <= 20 units +// The NOLINT directive is to disable cpplint check. +// Clang format and cpplint give conflicting recommendations on this line below. +#define f(fv, sv, shard_index) \ + (((fv) == -1 && (sv) == -1) || ((fv) < (sv) && (shard_index) >= (fv) && (shard_index) < (sv)) || \ + ((fv) > (sv) && ((shard_index) >= (fv) || (shard_index) < (sv)))) // NOLINT + // Main entry point for Concat Status ConcatOp::operator()() { children_num_ = static_cast(child_.size()); @@ -78,6 +84,7 @@ Status ConcatOp::operator()() { int eof_count = 0; int sample_number = 0; bool is_not_mappable = true; + bool is_not_mappable_or_second_ne_zero = true; int num_shard = 1; int shard_index = 0; std::shared_ptr distribute_sampler = std::dynamic_pointer_cast(sampler_); @@ -99,26 +106,28 @@ Status ConcatOp::operator()() { RETURN_IF_NOT_OK(Verify(i, buf)); } // 3. Put the data into output_connector - if (!children_flag_and_nums_.empty()) is_not_mappable = children_flag_and_nums_[i].first; + if (!children_flag_and_nums_.empty()) { + is_not_mappable = children_flag_and_nums_[i].first; + is_not_mappable_or_second_ne_zero = is_not_mappable || (!children_flag_and_nums_[i].second); + } while (!buf->eoe() && !buf->eof()) { // if dataset is not mappable or generator dataset which source is yield, cannot get the number of samples in // python layer), we use filtering to get data - if (sample_number % num_shard == shard_index && (is_not_mappable || !children_flag_and_nums_[i].second)) { + if (sample_number % num_shard == shard_index && is_not_mappable_or_second_ne_zero) { RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(buf))); - } else if (!is_not_mappable && children_flag_and_nums_[i].second) { // if dataset is mappable or generator - // dataset which source is not yield + } else if (!is_not_mappable_or_second_ne_zero) { + // if dataset is mappable or generator dataset which source is not yield, // get the start and end subscripts of valid values int fv = children_start_end_index_[i].first, sv = children_start_end_index_[i].second; // determine whether the data allocated to the current shard id is false data - if ((fv == -1 && sv == -1) || (fv < sv && shard_index >= fv && shard_index < sv) || - (fv > sv && (shard_index >= fv || shard_index < sv))) { + if (f(fv, sv, shard_index)) { RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(buf))); } } - // if dataSet is no mappable or generator dataset` which source is yeild, sample_number+=1 - if (is_not_mappable || !children_flag_and_nums_[i].second) { + // if dataset is not mappable or generator dataset which source is yield, sample_number+=1 + if (is_not_mappable_or_second_ne_zero) { sample_number++; } @@ -127,7 +136,7 @@ Status ConcatOp::operator()() { // if dataset is mappable,We don't use filtering to pick data. // so sample_number plus the length of the entire dataset - if (!is_not_mappable && children_flag_and_nums_[i].second) { + if (!is_not_mappable_or_second_ne_zero) { sample_number += children_flag_and_nums_[i].second; } } @@ -190,12 +199,6 @@ Status ConcatOp::ComputeColMap() { return Status::OK(); } -// Visitor pre-accept method for NodePass -Status ConcatOp::PreAccept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->PreRunOnNode(shared_from_base(), modified); -} - // Gets the number of classes Status ConcatOp::GetNumClasses(int64_t *num_classes) { int64_t max_num_classes = -1; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.h index 79a24746b48..6222b32a535 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -105,12 +105,6 @@ class ConcatOp : public PipelineOp { // @return - Status Status ComputeColMap() override; - /// \brief Base-class override for NodePass pre-visit acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status PreAccept(NodePass *p, bool *const modified) override; - /// \brief Gets the number of classes /// \param[out] num_classes the number of classes /// \return Status - The status code return diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc index 68f194d346d..5367955f78c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,10 +26,8 @@ #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/engine/datasetops/device_queue_op.h" #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h" -#include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h" #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" -#include "minddata/dataset/engine/opt/pass.h" #ifndef ENABLE_ANDROID #include "utils/system/crc32c.h" #include "utils/log_adapter.h" @@ -338,13 +336,8 @@ Status DatasetOp::EofReceived(int32_t worker_id) { return (out_connector_->Add(static_cast(worker_id), std::move(eof_buffer))); } -// During tree prepare phase, operators may have specific pre-operations to perform depending on -// their role. -Status DatasetOp::PrepareNodePreAction() { return Status::OK(); } - -// During tree prepare phase, operators may have specific post-operations to perform depending on -// their role. -Status DatasetOp::PrepareNodePostAction() { +// During tree prepare phase, operators may have specific post-operations to perform depending on their role. +Status DatasetOp::PrepareOperator() { // Creating Connector object for each op. // The consumer of the root node is assumed to be one thread. // If multiple threads are consuming from the root node, they will get the ordered data in round robin fashion. @@ -364,9 +357,6 @@ Status DatasetOp::PrepareNodePostAction() { return Status::OK(); } -// Getter function. Base class does not have any special flags setting. -uint32_t DatasetOp::PrepareFlags() const { return ExecutionTree::kDePrepNone; } - // Derived classes may implement the reset function if the operator is stateful and needs // specific reset handling that is not contained in this common code version of the reset. Status DatasetOp::Reset() { @@ -402,18 +392,6 @@ Status DatasetOp::ComputeColMap() { return Status::OK(); } -Status DatasetOp::PreAccept(NodePass *p, bool *const modified) { - // DatasetOp is the base class of visitor target pre-visit. - // This method will only be called if its derived class does not implement one. - return p->PreRunOnNode(shared_from_this(), modified); -} - -Status DatasetOp::Accept(NodePass *p, bool *const modified) { - // DatasetOp is the base class of visitor target. - // This method will only be called if its derived class does not implement one. - return p->RunOnNode(shared_from_this(), modified); -} - // Getter for the sampler, and it also removes the sampler from the op Status DatasetOp::FetchRemoveSampler(std::shared_ptr *sampler) { *sampler = sampler_; // It's okay if it sampler_ points to nullptr diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h index 7f7dabc9250..026ab51a080 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h @@ -78,19 +78,19 @@ class DatasetOp : public std::enable_shared_from_this { // Flags that control operator runtime behaviours enum OpState { kDeOpRunning = 0, kDeOpIdle = 1, kDeOpTerminated }; - /// Constructor + /// \brief Constructor /// \param op_connector_size - The size for the output connector of this operator. /// \param sampler - The sampler for the op DatasetOp(int32_t op_connector_size, std::shared_ptr sampler); - /// Destructor + /// \brief Destructor virtual ~DatasetOp() { tree_ = nullptr; } - /// Adds a operator to become our child. + /// \brief Adds a operator to become our child. /// \param child - shared pointer to the child to add. Status AddChild(std::shared_ptr child); - /// Remove a operator from our children. + /// \brief Remove a operator from our children. /// \param child - shared pointer to the child to remove. Status RemoveChild(std::shared_ptr child); @@ -114,9 +114,9 @@ class DatasetOp : public std::enable_shared_from_this { // Getter function to get all of our parents. std::vector parents() const; - // Inserts a operator as the parent current op. - // Inserted op will become the sole parent of the current op. - // The existing parent of the current op will be transferred to the inserted op. + // \brief Inserts a operator as the parent current op. + // \notes Inserted op will become the sole parent of the current op. + // The existing parent of the current op will be transferred to the inserted op. Status InsertAsParent(std::shared_ptr to_add); /// \brief Creates the connector within this operator @@ -140,9 +140,9 @@ class DatasetOp : public std::enable_shared_from_this { } /// \brief Class functor operator (). - /// DatasetOps operate by launching a thread (see ExecutionTree). - /// This pure virtual version makes the requirement that derived classes must provide a functor - /// that will execute their main runtime loop code. + /// \notes DatasetOps operate by launching a thread (see ExecutionTree). + /// This pure virtual version makes the requirement that derived classes must provide a functor + /// that will execute their main runtime loop code. /// \return Status The status code returned virtual Status operator()() = 0; @@ -170,8 +170,8 @@ class DatasetOp : public std::enable_shared_from_this { virtual Status GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id, bool retry_if_eoe); /// \brief Gets the next buffer from the given child . This function also has built-in eoe and eof - /// message handling so that child classes don't have to manually code pass-through logic when - /// those messages are received. + /// message handling so that child classes don't have to manually code pass-through logic when + /// those messages are received. /// \param p_buffer - The shared pointer for the fetched buffer to return (by reference) /// \param worker_id - The worker id /// \return Status The status code returned @@ -194,44 +194,34 @@ class DatasetOp : public std::enable_shared_from_this { virtual Status GetClassIndexing(std::vector>> *output_class_indexing); /// \brief Performs handling for when an eoe message is received. - /// The base class implementation simply flows the eoe message to output. Derived classes - /// may override if they need to perform special eoe handling. + /// The base class implementation simply flows the eoe message to output. Derived classes + /// may override if they need to perform special eoe handling. /// \param worker_id - The worker id /// \return Status The status code returned virtual Status EoeReceived(int32_t worker_id); /// \brief Performs handling for when an eof message is received. - /// The base class implementation simply flows the eof message to output. Derived classes - /// may override if they need to perform special eof handling. + /// The base class implementation simply flows the eof message to output. Derived classes + /// may override if they need to perform special eof handling. /// \param worker_id - The worker id /// \return Status The status code returned virtual Status EofReceived(int32_t worker_id); /// \brief Derived classes may implement the reset function if the operator is stateful and needs - /// specific reset handling that is not contained in this common code version of the reset + /// specific reset handling that is not contained in this common code version of the reset /// \return Status The status code returned virtual Status Reset(); - /// \brief During tree prepare phase, operators may have specific pre-operations to perform depending on - /// their role. - /// \notes Derived versions of this function should always call it's superclass version first - /// before providing their own implementations. - virtual Status PrepareNodePreAction(); - /// \brief During tree prepare phase, operators may have specific post-operations to perform depending on - /// their role. + /// their role. /// \notes Derived versions of this function should always call it's superclass version first - /// before providing their own implementations. - virtual Status PrepareNodePostAction(); + /// before providing their own implementations. + virtual Status PrepareOperator(); /// \brief Getter function /// \return The operator id int32_t id() const { return operator_id_; } - /// \brief Getter function - /// \return The prepare flags - virtual uint32_t PrepareFlags() const; - /// \brief Getter function /// \return The number of workers in this op virtual int32_t num_workers() const = 0; @@ -315,22 +305,6 @@ class DatasetOp : public std::enable_shared_from_this { /// \return Vector of Children std::vector> Children() const { return child_; } - /// \brief Base method for NodePass pre-visit. A tree walk consists of walking down the tree and also walking back up - /// in a depth-first order. PreAccept is the node visit on the way down, whereas the regular Accept is the main - /// visit on the way back up the tree during a post-order traversal. Subclass needs to override this if it - /// requires special node visit access. Check "dataset/engine/opt/pass.h" for more details. - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - virtual Status PreAccept(NodePass *p, bool *const modified); - - /// \brief Base method for NodePass visit. Subclass needs to override this if it requires special node visit access. - /// Check "dataset/engine/opt/pass.h" for more details. - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - virtual Status Accept(NodePass *p, bool *const modified); - /// Op name getter /// \return Name of the current Op virtual std::string Name() const = 0; @@ -401,7 +375,7 @@ class DatasetOp : public std::enable_shared_from_this { void AddParent(DatasetOp *parent); /// Compute the current op's column map using its child's column map. - /// Get called during the tree post-prepare phase in PrepareNodePostAction. + /// Get called during the tree post-prepare phase in PrepareOperator. /// This base implementation just inherits the map from child 0, and can only be used if the number of children is 1. /// Operations changing the column map it inherits from the child must overwrite this function. /// \return - Status diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc index 05714d88659..624aa06f907 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,17 +16,10 @@ #include "minddata/dataset/engine/datasetops/device_queue_op.h" -#include #include #include -#include -#include "minddata/dataset/core/config_manager.h" -#include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/dataset_iterator.h" -#include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h" -#include "minddata/dataset/engine/opt/pass.h" -#include "minddata/dataset/engine/perf/profiling.h" #include "minddata/dataset/util/status.h" #include "minddata/dataset/util/task_manager.h" @@ -490,12 +483,6 @@ void DeviceQueueOp::Print(std::ostream &out, bool show_all) const { } } -// Visitor accept method for NodePass -Status DeviceQueueOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - void DeviceQueueOp::ProfilingRecorder(bool isProfilingEnable, std::shared_ptr profiling_node, int64_t send_batch, int32_t tdt_cost, uint64_t *batch_start_time, uint64_t *end_time, int32_t connector_capacity, int32_t connector_size) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.h index dcf77cd262f..c3d48c2b458 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -168,12 +168,6 @@ class DeviceQueueOp : public PipelineOp { Status operator()() override; - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - // Record the pipeline profiling info void ProfilingRecorder(bool isProfilingEnable, std::shared_ptr profiling_node, int64_t send_batch, int32_t tdt_cost, uint64_t *batch_start_time, uint64_t *end_time, int32_t connector_capacity, diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc index 5efa38b6838..273ea606334 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc @@ -19,7 +19,6 @@ #include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h" #include "minddata/dataset/engine/data_buffer.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/util/log_adapter.h" namespace mindspore { @@ -102,17 +101,6 @@ Status EpochCtrlOp::EoeReceived(int32_t worker_id) { return Status::OK(); } -// Pre-Visitor accept method for NodePass -Status EpochCtrlOp::PreAccept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call the pre-visitation - return p->PreRunOnNode(shared_from_base(), modified); -} - -// Visitor accept method for NodePass -Status EpochCtrlOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call the pre-visitation - return p->RunOnNode(shared_from_base(), modified); -} int64_t EpochCtrlOp::GetTreeRepeatCount() { return child_[0]->GetTreeRepeatCount(); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.h index 9605658cdf5..f2ef8f6697a 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,7 +42,7 @@ class EpochCtrlOp : public RepeatOp { Status Build(std::shared_ptr *); }; - // Contructor + // Constructor explicit EpochCtrlOp(int32_t num_epoch); // Destructor @@ -65,18 +65,6 @@ class EpochCtrlOp : public RepeatOp { // @param worker_id - The worker id Status EoeReceived(int32_t worker_id) override; - /// \brief Base-class override for NodePass pre-visit acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status PreAccept(NodePass *p, bool *const modified) override; - - /// \brief Base-class override for NodePass visitor acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status Accept(NodePass *p, bool *const modified) override; - int64_t GetTreeRepeatCount() override; }; } // namespace dataset diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc index f674582c265..d9c9d2acf66 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,18 +16,14 @@ #include "minddata/dataset/engine/datasetops/filter_op.h" #include #include -#include #include #include #include #include "minddata/dataset/core/config_manager.h" -#include "minddata/dataset/core/constants.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/engine/data_buffer.h" -#include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/kernels/tensor_op.h" #include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/task_manager.h" @@ -247,17 +243,5 @@ Status FilterOp::InvokePredicateFunc(const TensorRow &input, bool *out_predicate return Status(StatusCode::kSuccess, "FilterOp predicate func call succeed"); } -// Visitor accept method for NodePass -Status FilterOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - -// Visitor pre-accept method for NodePass -Status FilterOp::PreAccept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->PreRunOnNode(shared_from_base(), modified); -} - } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.h index 89063f31987..ab09ac12299 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -121,18 +121,6 @@ class FilterOp : public ParallelOp { // @param show_all A bool to control if you want to show all info or just a summary. void Print(std::ostream &out, bool show_all) const override; - /// \brief Base-class override for NodePass pre-visit acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status PreAccept(NodePass *p, bool *const modified) override; - - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return kFilterOp; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc index c951a61b80b..18f69ff2370 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/datasetops/map_op/cpu_map_job.h" #include "minddata/dataset/engine/datasetops/map_op/gpu_map_job.h" -#include "minddata/dataset/engine/opt/pass.h" +#include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/kernels/tensor_op.h" #include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/task_manager.h" @@ -445,18 +445,6 @@ void MapOp::CreateFinalColMap(std::unordered_map *col_name } } -// Visitor accept method for NodePass -Status MapOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - -// Visitor pre-accept method for NodePass -Status MapOp::PreAccept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->PreRunOnNode(shared_from_base(), modified); -} - Status MapOp::WaitForWorkers() { // reset num_paused workers to 0 num_workers_paused_ = 0; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.h index 80d1069adfb..1eca6895916 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -177,18 +177,6 @@ class MapOp : public ParallelOp { // @return the number of threads consuming data from previous op's output Connector. int32_t num_consumers() const override; - /// \brief Base-class override for NodePass pre-visit acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status PreAccept(NodePass *p, bool *const modified) override; - - /// \brief Base-class override for NodePass visitor acceptor. - /// \param[in] p Pointer to the NodePass to be accepted. - /// \param[out] modified Whether this node visit modified the pipeline. - /// \return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return kMapOp; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.h index f119c870176..b5dd73a174c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -68,26 +68,6 @@ class ParallelOp : public DatasetOp { return out; } - // During tree prepare phase, operators may have specific pre-operations to perform depending on - // their role. - // @notes Derived versions of this function should always call it's superclass version first - // before providing their own implementations. - // @return Status - The error return code - Status PrepareNodePreAction() override { - // Run common code from super class before adding ParallelOp specific logic - return (DatasetOp::PrepareNodePreAction()); - } - - // During tree prepare phase, operators may have specific post-operations to perform depending on - // their role. - // @notes Derived versions of this function should always call it's superclass version first - // before providing their own implementations. - // @return Status - The error return code - Status PrepareNodePostAction() override { - // Run common code from super class before adding ParallelOp specific logic - return (DatasetOp::PrepareNodePostAction()); - } - // Override base class reset to provide reset actions specific to the ParallelOp class. // @return Status The status code returned Status Reset() override; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/pipeline_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/pipeline_op.h index 3c137c49d56..0af0b0d8896 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/pipeline_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/pipeline_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -67,24 +67,6 @@ class PipelineOp : public DatasetOp { // @return The number of threads that push data to the output connector int32_t num_producers() const override { return 1; } - // During tree prepare phase, operators may have specific pre-operations to perform depending on - // their role. - // @notes Derived versions of this function should always call it's superclass version first - // before providing their own implementations. - Status PrepareNodePreAction() override { - // Run common code from super class before adding PipelineOp specific logic - return (DatasetOp::PrepareNodePreAction()); - } - - // During tree prepare phase, operators may have specific post-operations to perform depending on - // their role. - // @notes Derived versions of this function should always call it's superclass version first - // before providing their own implementations. - Status PrepareNodePostAction() override { - // Run common code from super class before adding PipelineOp specific logic - return (DatasetOp::PrepareNodePostAction()); - } - protected: // ******************************************************************************* // I'm predicting there will be common arguments or functionality for pipeline ops, diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc index 573732a7f20..98159b91220 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,9 +23,7 @@ #include #include #include "minddata/dataset/engine/data_buffer.h" -#include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/util/log_adapter.h" namespace mindspore { @@ -132,12 +130,6 @@ Status ProjectOp::EoeReceived(int32_t worker_id) { Status ProjectOp::EofReceived(int32_t worker_id) { return Status::OK(); } -// Visitor accept method for NodePass -Status ProjectOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - // Compute the column map and save it into our own column name map // We cannot use the super class ComputeColMap here because we're making a modification of the // map from the child map. diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.h index f74957a13d3..3a33430af51 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -101,12 +101,6 @@ class ProjectOp : public PipelineOp { // @return Status The status code returned Status EofReceived(int32_t worker_id) override; - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return kProjectOp; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc index 32dc2cf1c57..127015a4261 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,6 @@ #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/util/log_adapter.h" namespace mindspore { @@ -171,10 +170,5 @@ Status RenameOp::EoeReceived(int32_t) { return Status::OK(); } -// Visitor accept method for NodePass -Status RenameOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.h index af628c1c9e6..6b35618deb3 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -110,12 +110,6 @@ class RenameOp : public PipelineOp { // @return Status The status code returned Status operator()() override; - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return kRenameOp; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc index 95a858d8d62..8fd3d2a8838 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,8 +20,6 @@ #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/engine/datasetops/repeat_op.h" #include "minddata/dataset/engine/data_buffer.h" -#include "minddata/dataset/engine/db_connector.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/util/log_adapter.h" namespace mindspore { @@ -175,18 +173,6 @@ int32_t RepeatOp::num_producers() const { } } -// Pre-Visitor accept method for NodePass -Status RepeatOp::PreAccept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call the pre-visitation - return p->PreRunOnNode(shared_from_base(), modified); -} - -// Visitor accept method for NodePass -Status RepeatOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - int64_t RepeatOp::GetTreeRepeatCount() { return num_repeats_; } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.h index f2fdf98cea6..09c305073bb 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -109,18 +109,6 @@ class RepeatOp : public PipelineOp { // @param workerId - The worker id int32_t num_producers() const override; - /// \brief Base-class override for NodePass pre-visit acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status PreAccept(NodePass *p, bool *const modified) override; - - /// \brief Base-class override for NodePass visitor acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return kRepeatOp; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc index 0eaa31d2a07..0d8cd168f73 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,9 @@ #if defined(_WIN32) || defined(_WIN64) #include #endif -#include -#include #include #include #include -#include #include #include @@ -30,7 +27,6 @@ #include "minddata/dataset/engine/dataset_iterator.h" #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/random.h" #include "minddata/dataset/util/status.h" @@ -291,11 +287,5 @@ Status ShuffleOp::EoeReceived(int32_t worker_id) { state_ = OpState::kDeOpIdle; return Status::OK(); } - -// Visitor accept method for NodePass -Status ShuffleOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.h index 58f4b6ea46e..9b733a79818 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -155,12 +155,6 @@ class ShuffleOp : public PipelineOp { // @return Status The status code returned Status EoeReceived(int32_t worker_id) override; - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return kShuffleOp; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc index 609b5819261..3a260755f4a 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,6 @@ #include "minddata/dataset/engine/datasetops/skip_op.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/util/log_adapter.h" namespace mindspore { @@ -124,17 +123,5 @@ Status SkipOp::EofReceived(int32_t worker_id) { return Status::OK(); } -// Visitor accept method for NodePass -Status SkipOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - -// Visitor pre-accept method for NodePass -Status SkipOp::PreAccept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->PreRunOnNode(shared_from_base(), modified); -} - } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.h index a8818bb4863..05d95c880e2 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -74,18 +74,6 @@ class SkipOp : public PipelineOp { // @param worker_id - The worker id Status EofReceived(int32_t worker_id) override; - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - - /// \brief Base-class override for NodePass pre-visit acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status PreAccept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return kSkipOp; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc index 4e0482588fd..fbe7136390e 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,6 @@ #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" #ifndef ENABLE_ANDROID #include "minddata/dataset/kernels/image/image_utils.h" #else @@ -590,12 +589,6 @@ Status AlbumOp::LaunchThreadsAndInitOp() { return Status::OK(); } -// Visitor accept method for NodePass -Status AlbumOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - Status AlbumOp::ComputeColMap() { // Set the column name map (base class field) if (column_name_id_map_.empty()) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.h index 764e666da22..b12946731d9 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -192,12 +192,6 @@ class AlbumOp : public ParallelOp, public RandomAccessOp { /// \return bool - if file is bad then return false bool CheckImageType(const std::string &file_name, bool *valid); - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return "AlbumOp"; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc index 54c8cf6da96..cca95841377 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc @@ -1,6 +1,6 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd - + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -23,7 +23,6 @@ #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" #include "minddata/dataset/engine/data_schema.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" #ifndef ENABLE_ANDROID #include "minddata/dataset/kernels/image/image_utils.h" #else @@ -440,12 +439,6 @@ Status CelebAOp::Reset() { return Status::OK(); } -// Visitor accept method for NodePass -Status CelebAOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - Status CelebAOp::ComputeColMap() { // Set the column name map (base class field) if (column_name_id_map_.empty()) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.h index f13b44e2789..e1e0a644ede 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.h @@ -1,6 +1,6 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd - + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -169,12 +169,6 @@ class CelebAOp : public ParallelOp, RandomAccessOp { // @return Status The status code returned Status AddIOBlock(std::unique_ptr *data_buffer); - /// \brief Base-class override for NodePass visitor acceptor - /// \param[in] p Pointer to the NodePass to be accepted - /// \param[out] modified Indicator if the node was changed at all - /// \return Status of the node visit - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return "CelebAOp"; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc index 84ca6f35513..5631f131225 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,6 @@ #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" #include "utils/ms_utils.h" namespace mindspore { @@ -500,12 +499,6 @@ Status CifarOp::CountTotalRows(const std::string &dir, const std::string &usage, } } -// Visitor accept method for NodePass -Status CifarOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - Status CifarOp::ComputeColMap() { // set the column name map (base class field) if (column_name_id_map_.empty()) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.h index 5c5f392eee6..abbb5da7c26 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -165,12 +165,6 @@ class CifarOp : public ParallelOp, public RandomAccessOp { // @return static Status CountTotalRows(const std::string &dir, const std::string &usage, bool isCIFAR10, int64_t *count); - /// \brief Base-class override for NodePass visitor acceptor - /// \param[in] p Pointer to the NodePass to be accepted - /// \param[out] modified Indicator if the node was changed at all - /// \return Status of the node visit - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return "CifarOp"; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc index f3d23c10221..d5bac81e3c7 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,6 @@ #include "minddata/dataset/util/task_manager.h" #include "minddata/dataset/engine/jagged_connector.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/engine/datasetops/source/io_block.h" #include "minddata/dataset/util/random.h" @@ -549,11 +548,5 @@ Status ClueOp::ComputeColMap() { return Status::OK(); } -// Visitor accept method for NodePass -Status ClueOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.h index a95b8a16e56..f5457d5abc0 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -177,12 +177,6 @@ class ClueOp : public ParallelOp { // @return Name of the current Op std::string Name() const override { return "ClueOp"; } - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - private: // The entry point for when workers are launched. // @param worker_id - the id of the worker that is executing this function. diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc index 01a167f006f..91892b49ee3 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ #include "minddata/dataset/core/tensor_shape.h" #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" #include "minddata/dataset/engine/db_connector.h" -#include "minddata/dataset/engine/opt/pass.h" +#include "minddata/dataset/engine/execution_tree.h" namespace mindspore { namespace dataset { @@ -674,12 +674,6 @@ Status CocoOp::GetClassIndexing(const std::string &dir, const std::string &file, return Status::OK(); } -// Visitor accept method for NodePass -Status CocoOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - Status CocoOp::ComputeColMap() { // Set the column name map (base class field) if (column_name_id_map_.empty()) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.h index 808823721b2..e152cbef8f5 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -203,12 +203,6 @@ class CocoOp : public ParallelOp, public RandomAccessOp { static Status GetClassIndexing(const std::string &dir, const std::string &task_type, const std::string &task_mode, std::vector>> *output_class_indexing); - /// \brief Base-class override for NodePass visitor acceptor - /// \param[in] p Pointer to the NodePass to be accepted - /// \param[out] modified Indicator if the node was changed at all - /// \return Status of the node visit - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return "CocoOp"; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc index 8575d06006e..0723abbd9d1 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,6 @@ #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/engine/jagged_connector.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/util/random.h" namespace mindspore { @@ -916,11 +915,5 @@ Status CsvOp::ComputeColMap() { return Status::OK(); } -// Visitor accept method for NodePass -Status CsvOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.h index af52d91a297..d101c565453 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.h @@ -298,12 +298,6 @@ class CsvOp : public ParallelOp { // @return Vector of the input file names std::vector FileNames() { return csv_files_list_; } - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return "CsvOp"; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc index 4bbecff4311..fbd6ec87b74 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,11 +16,9 @@ #include "minddata/dataset/engine/datasetops/source/generator_op.h" #include #include "minddata/dataset/core/global_context.h" -#include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/util/task_manager.h" -#include "minddata/dataset/engine/opt/pass.h" namespace mindspore { namespace dataset { @@ -278,12 +276,6 @@ Status GeneratorOp::Reset() { return Status(StatusCode::kSuccess, "GeneratorOp Reset Succeed"); } -// Visitor accept method for NodePass -Status GeneratorOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - Status GeneratorOp::ComputeColMap() { // Setup column names map (base class field) if (column_name_id_map_.empty()) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.h index 1c9a7f9a958..b2c5abdcd41 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -126,12 +126,6 @@ class GeneratorOp : public PipelineOp, public RandomAccessOp { /// \return Status The status code returned Status Reset() override; - /// Base-class override for NodePass visitor acceptor. - /// \param p - Pointer to the NodePass to be accepted. - /// \param modified - Whether this node visit modified the pipeline. - /// \return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - /// Op name getter /// \return Name of the current Op std::string Name() const override { return "GeneratorOp"; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc index 1e806fc45c6..2a969c1f493 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,6 @@ #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" namespace mindspore { namespace dataset { @@ -450,12 +449,6 @@ Status ImageFolderOp::CountRowsAndClasses(const std::string &path, const std::se return Status::OK(); } -// Visitor accept method for NodePass -Status ImageFolderOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - Status ImageFolderOp::ComputeColMap() { // Set the column name map (base class field) if (column_name_id_map_.empty()) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.h index 0b91ad68d37..2b4e0902631 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -207,12 +207,6 @@ class ImageFolderOp : public ParallelOp, public RandomAccessOp { static Status CountRowsAndClasses(const std::string &path, const std::set &exts, int64_t *num_rows, int64_t *num_classes, std::map class_index); - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return "ImageFolderOp"; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc index eff09d53c8a..325769b74b3 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,7 +27,6 @@ #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" namespace mindspore { namespace dataset { @@ -436,12 +435,6 @@ Status ManifestOp::GetClassIndexing(const std::string &file, const py::dict &dic } #endif -// Visitor accept method for NodePass -Status ManifestOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - Status ManifestOp::ComputeColMap() { // Set the column name map (base class field) if (column_name_id_map_.empty()) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.h index 0a1d4b386e9..b8417fa554f 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -180,12 +180,6 @@ class ManifestOp : public ParallelOp, public RandomAccessOp { std::map *output_class_indexing); #endif - /// \brief Base-class override for NodePass visitor acceptor - /// \param[in] p Pointer to the NodePass to be accepted - /// \param[out] modified Indicator if the node was changed at all - /// \return Status of the node visit - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return "ManifestOp"; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc index 80f2a09eee1..42d9635a215 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,7 +29,6 @@ #include "minddata/dataset/engine/datasetops/dataset_op.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/util/log_adapter.h" namespace mindspore { @@ -461,12 +460,6 @@ Status MindRecordOp::CountTotalRows(const std::vector dataset_path, return Status::OK(); } -// Visitor accept method for NodePass -Status MindRecordOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - Status MindRecordOp::ComputeColMap() { if (column_name_id_map_.empty()) { for (int i = 0; i < static_cast(columns_to_load_.size()); i++) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.h index 038e7dd4969..acc90fd879a 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -202,12 +202,6 @@ class MindRecordOp : public ParallelOp { Status Init(); - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return "MindRecordOp"; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.cc index bdf70066582..397f1cee234 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,6 @@ #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" namespace mindspore { namespace dataset { @@ -459,12 +458,6 @@ Status MnistOp::CountTotalRows(const std::string &dir, const std::string &usage, return Status::OK(); } -// Visitor accept method for NodePass -Status MnistOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - Status MnistOp::ComputeColMap() { // set the column name map (base class field) if (column_name_id_map_.empty()) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.h index c02ff7f0d3c..1791adfadd0 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -158,12 +158,6 @@ class MnistOp : public ParallelOp, public RandomAccessOp { // @return static Status CountTotalRows(const std::string &dir, const std::string &usage, int64_t *count); - /// \brief Base-class override for NodePass visitor acceptor - /// \param[in] p Pointer to the NodePass to be accepted - /// \param[out] modified Indicator if the node was changed at all - /// \return Status of the node visit - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return "MnistOp"; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc index eccedcecd3a..ee7d5532a14 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,6 @@ #include "minddata/dataset/util/random.h" #include "minddata/dataset/util/wait_post.h" #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" -#include "minddata/dataset/engine/opt/pass.h" namespace mindspore { namespace dataset { @@ -406,12 +405,6 @@ Status RandomDataOp::Reset() { return Status::OK(); } -// Visitor accept method for NodePass -Status RandomDataOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - Status RandomDataOp::ComputeColMap() { // Extract the column name mapping from the schema and save it in the class. if (column_name_id_map_.empty()) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.h index bec8cb2f540..50d4ec49882 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -249,12 +249,6 @@ class RandomDataOp : public ParallelOp { return ++buffer_id_; } - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - // Private function for computing the assignment of the column name map. // @return - Status Status ComputeColMap() override; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc index 97eaf216045..dceae4beb8e 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,6 @@ #include #include -#include "utils/ms_utils.h" #include "minddata/dataset/engine/datasetops/source/text_file_op.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/util/task_manager.h" @@ -29,7 +28,6 @@ #include "minddata/dataset/util/random.h" #include "minddata/dataset/engine/datasetops/source/io_block.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" namespace mindspore { namespace dataset { @@ -499,11 +497,5 @@ Status TextFileOp::ComputeColMap() { return Status::OK(); } -// Visitor accept method for NodePass -Status TextFileOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.h index f253a4e3acd..2259e374836 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -177,12 +177,6 @@ class TextFileOp : public ParallelOp { // @return Vector of the input file names std::vector FileNames() { return text_files_list_; } - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - private: // The entry point for when workers are launched. // @param worker_id - the id of the worker that is executing this function. diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc index 0f85128ebf9..b20a12de3d2 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,6 @@ #include #include "proto/example.pb.h" -#include "./securec.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/data_schema.h" @@ -34,7 +33,6 @@ #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/engine/jagged_connector.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/util/random.h" #include "minddata/dataset/util/status.h" #include "minddata/dataset/util/task_manager.h" @@ -1020,12 +1018,6 @@ int64_t TFReaderOp::CountTotalRowsSectioned(const std::vector &file return rows_read; } -// Visitor accept method for NodePass -Status TFReaderOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - Status TFReaderOp::ComputeColMap() { // Construct the column name map for this operator (base class field) if (column_name_id_map_.empty()) { @@ -1038,26 +1030,5 @@ Status TFReaderOp::ComputeColMap() { return Status::OK(); } -// During tree prepare phase, operators may have specific post-operations to perform depending on -// their role. -Status TFReaderOp::PrepareNodePostAction() { - // Run common code from super class before adding TFReaderOp specific handling - RETURN_IF_NOT_OK(ParallelOp::PrepareNodePostAction()); - - // Now that the sampler has been saved for the cache, we need to adjust the TFReaderOp to turn it into - // a simpler producer of all data (no shuffling or sharding or anything) - if (!BitTest(tree_->PrepareFlags(), ExecutionTree::kDePrepCache)) { - // This sanity check had been delayed until now in the prepare loop. - // If we are not in a cache path, then we can validate the file-based sharding config. - // If we are in a cache path, there is no file-based sharding so the check is not correct in that - // situation. - if (!equal_rows_per_shard_ && dataset_files_list_.size() < static_cast(num_devices_)) { - RETURN_STATUS_UNEXPECTED("Invalid file, not enough tfrecord files provided.\n"); - } - } - - return Status::OK(); -} - } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.h index cf038a392a7..e4d4625f681 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -222,12 +222,6 @@ class TFReaderOp : public ParallelOp { static Status CountTotalRows(int64_t *out_total_rows, const std::vector &filenames, int64_t threads = 1, bool estimate = false); - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return "TFReaderOp"; } @@ -236,12 +230,6 @@ class TFReaderOp : public ParallelOp { // @return Vector of the input file names std::vector FileNames() { return dataset_files_list_; } - // During tree prepare phase, operators may have specific post-operations to perform depending on - // their role. - // @notes Derived versions of this function should always call it's superclass version first - // before providing their own implementations. - Status PrepareNodePostAction() override; - static bool ValidateFirstRowCrc(const std::string &filename); private: diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc index 5346dc69372..11d6b24ea2c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,6 @@ #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" #include "utils/ms_utils.h" namespace mindspore { @@ -494,12 +493,6 @@ Status VOCOp::GetClassIndexing(const std::string &dir, const std::string &task_t } #endif -// Visitor accept method for NodePass -Status VOCOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - Status VOCOp::ComputeColMap() { // Set the column name map (base class field) if (column_name_id_map_.empty()) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.h index 31e900c90d7..355de2266c9 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -210,12 +210,6 @@ class VOCOp : public ParallelOp, public RandomAccessOp { const py::dict &dict, std::map *output_class_indexing); #endif - /// \brief Base-class override for NodePass visitor acceptor - /// \param[in] p Pointer to the NodePass to be accepted - /// \param[out] modified Indicator if the node was changed at all - /// \return Status of the node visit - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return "VOCOp"; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/take_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/take_op.cc index aecd7658dcf..bea084a0d0c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/take_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/take_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,6 @@ #include "minddata/dataset/engine/datasetops/take_op.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" -#include "minddata/dataset/engine/opt/pass.h" namespace mindspore { namespace dataset { @@ -127,17 +126,5 @@ Status TakeOp::FillBuffer(std::unique_ptr *buffer, std::unique_ptrRunOnNode(shared_from_base(), modified); -} - -// Visitor pre-accept method for NodePass -Status TakeOp::PreAccept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->PreRunOnNode(shared_from_base(), modified); -} - } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/take_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/take_op.h index 149cde80e39..77173c6ee2a 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/take_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/take_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -78,18 +78,6 @@ class TakeOp : public PipelineOp { // @return Status The status code returned Status operator()() override; - // Base-class override for NodePass visitor acceptor. - // @param p - Pointer to the NodePass to be accepted. - // @param modified - Whether this node visit modified the pipeline. - // @return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - - /// \brief Base-class override for NodePass pre-visit acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status PreAccept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return kTakeOp; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc index 2ce1d895456..eb0da1d0638 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,6 @@ #include "minddata/dataset/core/constants.h" #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/util/log_adapter.h" @@ -238,18 +237,6 @@ Status ZipOp::EoeReceived(int32_t) { return Status::OK(); } -// Visitor pre-accept method for NodePass -Status ZipOp::PreAccept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->PreRunOnNode(shared_from_base(), modified); -} - -// Visitor accept method for NodePass -Status ZipOp::Accept(NodePass *p, bool *const modified) { - // Downcast shared pointer then call visitor - return p->RunOnNode(shared_from_base(), modified); -} - Status ZipOp::ComputeColMap() { if (column_name_id_map_.empty()) { column_name_id_map_ = {}; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.h index 1ef13ac4d03..55f57ba496d 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -104,18 +104,6 @@ class ZipOp : public PipelineOp { // @return Status The status code returned Status operator()() override; - /// \brief Base-class override for NodePass pre-visit acceptor - /// \param[in] p The node to visit - /// \param[out] modified Indicator if the node was modified - /// \return Status of the node visit - Status PreAccept(NodePass *p, bool *const modified) override; - - /// \brief Base-class override for NodePass visitor acceptor. - /// \param[in] p Pointer to the NodePass to be accepted. - /// \param[out] modified Whether this node visit modified the pipeline. - /// \return - Status of the node visit. - Status Accept(NodePass *p, bool *const modified) override; - // Op name getter // @return Name of the current Op std::string Name() const override { return kZipOp; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc b/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc index 83ac5a495f9..2aebc458569 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc @@ -29,7 +29,7 @@ namespace mindspore { namespace dataset { // Constructor -ExecutionTree::ExecutionTree() : id_count_(0), tree_state_(kDeTStateInit), prepare_flags_(kDePrepNone) { +ExecutionTree::ExecutionTree() : id_count_(0), tree_state_(kDeTStateInit) { tg_ = std::make_unique(); profiling_manager_ = std::make_unique(this); #if defined(ENABLE_GPUQUE) || defined(ENABLE_TDTQUE) @@ -66,11 +66,11 @@ Status ExecutionTree::AssociateNode(const std::shared_ptr &op) { if (op->tree_ == this) { return Status::OK(); } - if (tree_state_ != kDeTStateInit && tree_state_ != kDeTStateBuilding && tree_state_ != kDeTStatePrepare) { + if (tree_state_ != kDeTStateInit && tree_state_ != kDeTStateBuilding) { std::string err_msg = "Invalid tree state for adding a node. Current state: " + std::to_string(static_cast(tree_state_)) + " Expected states: " + std::to_string(static_cast(kDeTStateInit)) + " or " + - std::to_string(static_cast(kDeTStateBuilding)) + " or " + std::to_string(static_cast(kDeTStatePrepare)); + std::to_string(static_cast(kDeTStateBuilding)); RETURN_STATUS_UNEXPECTED(err_msg); } @@ -172,10 +172,10 @@ Status ExecutionTree::Launch() { #endif // Tree must be built and prepared before it can be launched! - if (tree_state_ != kDeTStateReady) { + if (tree_state_ != kDeTStatePrepared) { std::string err_msg = "Invalid tree state for launching tree. Current state: " + std::to_string(static_cast(tree_state_)) + - " Expected state: " + std::to_string(static_cast(kDeTStateReady)); + " Expected state: " + std::to_string(static_cast(kDeTStatePrepared)); RETURN_STATUS_UNEXPECTED(err_msg); } @@ -244,43 +244,31 @@ Status ExecutionTree::LaunchWorkers(int32_t num_workers, std::functionPrepareNode(root_)); - tree_state_ = kDeTStateReady; - return Status::OK(); -} + std::vector> fifo; + std::shared_ptr op = root_; + size_t index = 0; -// Recursive function used during prepare phase to visit a node and drive any pre- and post- -// node actions during a tree walk. -Status ExecutionTree::PrepareNode(const std::shared_ptr &dataset_op) { - // Before going down into children, make any prepare flags updates based on this operator. - uint32_t op_prep_flags = dataset_op->PrepareFlags(); - BitSet(&prepare_flags_, op_prep_flags); + // Build a FIFO queue with the root at the beginning and continue adding its descendants to the queue. + fifo.push_back(op); + do { + op = fifo[index]; + fifo.insert(fifo.end(), op->child_.begin(), op->child_.end()); + ++index; + } while (index < fifo.size()); - // Now, descend to children - for (const auto &i : dataset_op->child_) { - RETURN_IF_NOT_OK(this->PrepareNode(i)); + // By iterating from the end of the FIFO queue, we simulate the post-order walk. + for (auto rit = fifo.crbegin(); rit != fifo.crend(); ++rit) { + RETURN_IF_NOT_OK((*rit)->PrepareOperator()); } - // No more children, now we execute any prepare actions before going back up the - // the tree on recursive function - RETURN_IF_NOT_OK(dataset_op->PrepareNodePostAction()); - - // Then clear the flags from this op now that we have prepared it. - BitClear(&prepare_flags_, op_prep_flags); - + // The tree is prepared. + tree_state_ = kDeTStatePrepared; return Status::OK(); } } // namespace dataset diff --git a/mindspore/ccsrc/minddata/dataset/engine/execution_tree.h b/mindspore/ccsrc/minddata/dataset/engine/execution_tree.h index 059a53aadd5..8353dde9432 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/execution_tree.h +++ b/mindspore/ccsrc/minddata/dataset/engine/execution_tree.h @@ -39,19 +39,11 @@ class Pass; using OptPass = std::vector>; class ExecutionTree { public: - // Prepare flags used during tree prepare phase - enum PrepareFlags { - kDePrepNone = 0, - kDePrepRepeat = 1, // Processing a repeat operation - kDePrepCache = 2 // Processing a cache operation - }; - // State flags for the lifecycle of the tree enum TreeState { kDeTStateInit = 0, // The freshly initialized state after construction kDeTStateBuilding, // The tree is being built, nodes are being added - kDeTStatePrepare, // The tree has been assigned a root node and is pending prepare - kDeTStateReady, // The tree has been prepared and is ready to be launched + kDeTStatePrepared, // The tree has been prepared and is ready to be launched kDeTStateExecuting, // The tree has been launched and is executing kDeTStateEpochEnd, // The tree has been received end of epoch signal, just for profiling kDeTStateFinished // The tree has been drained, dataset iterator received EOF @@ -111,122 +103,101 @@ class ExecutionTree { // Destructor ~ExecutionTree(); - // Associates a DatasetOp with this tree. This assigns a valid node id to the operator and - // provides it with a link to the tree. A node cannot form any relationships (parent/child) with - // other nodes unless they are associated with the same tree. - // @param op - The operator to associate - // @return Status The status code returned + /// \brief Associates a DatasetOp with this tree. This assigns a valid node id to the operator and + /// provides it with a link to the tree. A node cannot form any relationships (parent/child) with + /// other nodes unless they are associated with the same tree. + /// \param op - The operator to associate + /// \return Status The status code returned Status AssociateNode(const std::shared_ptr &op); - // Sets the root node of the tree - // @param op - The operator to assign as root - // @return Status The status code returned + /// \brief Set the root node of the tree + /// \param op - The operator to assign as root + /// \return Status The status code returned Status AssignRoot(const std::shared_ptr &op); - // Start the execution of the tree - // @return Status The status code returned + /// \brief Start the execution of the tree + /// \return Status The status code returned Status Launch(); - /// A print method typically used for debugging + /// /brief A print method typically used for debugging /// \param out - The output stream to write output to void Print(std::ostream &out, const std::shared_ptr &op = nullptr) const; - // Returns an iterator positioned at the start - // @return Iterator - The iterator + /// \brief Return an iterator positioned at the start + /// \return Iterator - The iterator ExecutionTree::Iterator begin(const std::shared_ptr &root = nullptr) const { return Iterator(root == nullptr ? root_ : root); } - // Returns an iterator positioned at the end - // @return Iterator - The iterator + /// \brief Return an iterator positioned at the end + /// \return Iterator - The iterator ExecutionTree::Iterator end() const { return Iterator(nullptr); } - // << Stream output operator overload - // @notes This allows you to write the debug print info using stream operators - // @param out - reference to the output stream being overloaded - // @param exe_tree - reference to the execution tree to display - // @return - the output stream must be returned + /// \brief << Stream output operator overload + /// \notes This allows you to write the debug print info using stream operators + /// \param out - reference to the output stream being overloaded + /// \param exe_tree - reference to the execution tree to display + /// \return - the output stream must be returned friend std::ostream &operator<<(std::ostream &out, ExecutionTree &exe_tree) { exe_tree.Print(out); return out; } - // Given the number of workers, launches the worker entry function for each. Essentially a - // wrapper for the TaskGroup handling that is stored inside the execution tree. - // @param num_workers - The number of workers to launch - // @param func - The function entry point that workers will execute - // @param name - The description of worker to launch - // @param op_id - The id of corresponding operator, if not inherit from dataset op then it is -1. - // @return Status The status code returned + /// \brief Given the number of workers, launches the worker entry function for each. Essentially a + /// wrapper for the TaskGroup handling that is stored inside the execution tree. + /// \param num_workers - The number of workers to launch + /// \param func - The function entry point that workers will execute + /// \param name - The description of worker to launch + /// \param op_id - The id of corresponding operator, if not inherit from dataset op then it is -1. + /// \return Status The status code returned Status LaunchWorkers(int32_t num_workers, std::function func, std::string name = "", int32_t operator_id = -1); - // Getter method - // @return shared_ptr to the root operator + /// \brief Getter method + /// \return shared_ptr to the root operator std::shared_ptr root() const { return root_; } - // Getter method - // @return the prepare flags - uint32_t PrepareFlags() const { return prepare_flags_; } - - // Compulsory transformation/action pre optimization. - // @return Status The status code returned - Status PreAction(); - - // Compulsory transformation/action post optimization. - // @return Status The status code returned - Status PostAction(); - - // The DEPRECATED driver of the prepare phase of the execution tree. The prepare phase will recursively - // walk the tree to perform modifications to the tree or specific nodes within the tree to get - // it ready for execution. - // @param Total number of epochs that will be run on this tree - // @return Status The status code returned + /// \brief The prepare phase walks the tree in post-order to perform modifications to get it ready for execution. + /// \return Status The status code returned Status Prepare(); - // Recursive function used during prepare phase to visit a node and drive any pre- and post- - // node actions during a tree walk. - // @param op - The dataset op to work on - // @return Status The status code returned - Status PrepareNode(const std::shared_ptr &dataset_op); - - // Return the pointer to the TaskGroup - // @return raw pointer to the TaskGroup + /// \brief Return the pointer to the TaskGroup + /// \return raw pointer to the TaskGroup TaskGroup *const AllTasks() const { return tg_.get(); } - // Return if the ExecutionTree is at end of epoch status - // @return bool - true is ExecutionTree is end of epoch status + /// \brief Return if the ExecutionTree is at end of epoch status + /// \return bool - true is ExecutionTree is end of epoch status bool IsEpochEnd() const { return tree_state_ == TreeState::kDeTStateEpochEnd; } - // Set the ExecutionTree to EOE state + /// \brief Set the ExecutionTree to EOE state void SetEpochEnd() { tree_state_ = TreeState::kDeTStateEpochEnd; } - // Set the ExecutionTree to executing state + /// \brief Set the ExecutionTree to executing state void SetExecuting() { tree_state_ = TreeState::kDeTStateExecuting; } - // Return if the ExecutionTree is finished (iterator receives EOF). - // @return Bool - true is ExecutionTree is finished - bool isFinished() const { return tree_state_ == TreeState::kDeTStateFinished; } - - // Return if the ExecutionTree is ready. - // @return Bool - true is ExecutionTree is ready - bool isPrepared() const { - return tree_state_ == TreeState::kDeTStateReady || tree_state_ == kDeTStateExecuting || - tree_state_ == kDeTStateFinished; - } - - // Set the ExecutionTree to Finished state. + /// \brief Set the ExecutionTree to Finished state. void SetFinished() { tree_state_ = TreeState::kDeTStateFinished; } - // Getter for profiling manager, no ownership + /// \brief Return if the ExecutionTree is finished (iterator receives EOF). + /// \return Bool - true is ExecutionTree is finished + bool isFinished() const { return tree_state_ == TreeState::kDeTStateFinished; } + + /// \brief Return if the ExecutionTree is ready. + /// \return Bool - true is ExecutionTree is ready + bool isPrepared() const { + return tree_state_ == TreeState::kDeTStatePrepared || tree_state_ == TreeState::kDeTStateExecuting || + tree_state_ == TreeState::kDeTStateFinished; + } + + /// \brief Getter for profiling manager, no ownership ProfilingManager *GetProfilingManager() { return profiling_manager_.get(); } private: - // A helper functions for doing the recursive printing - // @param dataset_op - The dataset op to print - // @param indent - an indent string for aligning child levels in output - // @param last - an indicator if it's the last child or not - // @param detailed - should it display the detailed node output or the summary line + /// \brief A helper functions for doing the recursive printing + /// \param dataset_op - The dataset op to print + /// \param indent - an indent string for aligning child levels in output + /// \param last - an indicator if it's the last child or not + /// \param detailed - should it display the detailed node output or the summary line void PrintNode(std::ostream &out, const std::shared_ptr &dataset_op, std::string indent, bool last, bool detailed) const; diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc index de9fdbfc0e2..2d2ce251446 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc @@ -24,6 +24,7 @@ #include "minddata/dataset/engine/datasetops/source/tf_reader_op.h" #include "minddata/dataset/engine/jagged_connector.h" +#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/util/status.h" #include "utils/system/crc32c.h" @@ -246,5 +247,16 @@ Status TFRecordNode::MakeSimpleProducer() { return Status::OK(); } +// Visitor accepting method for IRNodePass +Status TFRecordNode::Accept(IRNodePass *p, bool *const modified) { + // Downcast shared pointer then call visitor + return p->Visit(shared_from_base(), modified); +} + +// Visitor accepting method for IRNodePass +Status TFRecordNode::AcceptAfter(IRNodePass *p, bool *const modified) { + // Downcast shared pointer then call visitor + return p->VisitAfter(shared_from_base(), modified); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.h b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.h index ad5d259e57d..c56f205b580 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.h +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.h @@ -30,6 +30,8 @@ namespace dataset { /// \class TFRecordNode /// \brief A Dataset derived class to represent TFRecord dataset class TFRecordNode : public NonMappableSourceNode { + friend class CacheValidationPass; + public: /// \brief Constructor /// \note Parameter 'schema' is the path to the schema file @@ -140,6 +142,18 @@ class TFRecordNode : public NonMappableSourceNode { /// \return Status of the function Status MakeSimpleProducer() override; + /// \brief Base-class override for accepting IRNodePass visitor + /// \param[in] p The node to visit + /// \param[out] modified Indicator if the node was modified + /// \return Status of the node visit + Status Accept(IRNodePass *p, bool *const modified) override; + + /// \brief Base-class override for accepting IRNodePass visitor + /// \param[in] p The node to visit + /// \param[out] modified Indicator if the node was modified + /// \return Status of the node visit + Status AcceptAfter(IRNodePass *p, bool *const modified) override; + private: std::vector dataset_files_; std::string schema_path_; // schema_path_ path to schema file. It is set when type of schema parameter is string diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc b/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc index 396c3a7a7ab..e74d770d1f2 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc @@ -43,6 +43,7 @@ #include "minddata/dataset/engine/ir/datasetops/source/generator_node.h" #endif #include "minddata/dataset/engine/ir/datasetops/source/random_node.h" +#include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h" #ifdef ENABLE_PYTHON #include "minddata/dataset/engine/ir/datasetops/sync_wait_node.h" #endif @@ -50,53 +51,6 @@ #include "minddata/dataset/engine/ir/datasetops/transfer_node.h" #include "minddata/dataset/engine/ir/datasetops/zip_node.h" -////////////////////////////////// -// This section of code will be removed once the migration of optimizer from DatasetOp to DatasetNode is done. -#include "minddata/dataset/engine/datasetops/batch_op.h" -#include "minddata/dataset/engine/datasetops/build_vocab_op.h" -#ifndef ENABLE_ANDROID -#include "minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h" -#include "minddata/dataset/engine/datasetops/cache_op.h" -#include "minddata/dataset/engine/datasetops/cache_merge_op.h" -#include "minddata/dataset/engine/datasetops/cache_lookup_op.h" -#endif -#include "minddata/dataset/engine/datasetops/concat_op.h" -#include "minddata/dataset/engine/datasetops/dataset_op.h" -#include "minddata/dataset/engine/datasetops/device_queue_op.h" -#include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h" -#include "minddata/dataset/engine/datasetops/map_op/map_op.h" -#include "minddata/dataset/engine/datasetops/project_op.h" -#include "minddata/dataset/engine/datasetops/rename_op.h" -#include "minddata/dataset/engine/datasetops/repeat_op.h" -#include "minddata/dataset/engine/datasetops/skip_op.h" -#include "minddata/dataset/engine/datasetops/shuffle_op.h" -#include "minddata/dataset/engine/datasetops/source/album_op.h" -#include "minddata/dataset/engine/datasetops/source/celeba_op.h" -#include "minddata/dataset/engine/datasetops/source/cifar_op.h" -#include "minddata/dataset/engine/datasetops/source/coco_op.h" -#include "minddata/dataset/engine/datasetops/source/manifest_op.h" -#ifndef ENABLE_ANDROID -#include "minddata/dataset/engine/datasetops/source/mindrecord_op.h" -#endif -#include "minddata/dataset/engine/datasetops/source/mnist_op.h" -#include "minddata/dataset/engine/datasetops/source/random_data_op.h" -#ifndef ENABLE_ANDROID -#include "minddata/dataset/engine/datasetops/source/tf_reader_op.h" -#include "minddata/dataset/engine/datasetops/source/clue_op.h" -#include "minddata/dataset/engine/datasetops/source/csv_op.h" -#include "minddata/dataset/engine/datasetops/source/text_file_op.h" -#endif - -#ifdef ENABLE_PYTHON -#include "minddata/dataset/engine/datasetops/source/voc_op.h" -#include "minddata/dataset/engine/datasetops/filter_op.h" -#include "minddata/dataset/engine/datasetops/source/generator_op.h" -#endif -#include "minddata/dataset/engine/datasetops/source/image_folder_op.h" -#include "minddata/dataset/engine/datasetops/take_op.h" -#include "minddata/dataset/engine/datasetops/zip_op.h" -////////////////////////////////// - namespace mindspore { namespace dataset { @@ -295,6 +249,12 @@ Status IRNodePass::Visit(std::shared_ptr node, bool *const modified) { Status IRNodePass::VisitAfter(std::shared_ptr node, bool *const modified) { return VisitAfter(std::static_pointer_cast(node), modified); } +Status IRNodePass::Visit(std::shared_ptr node, bool *const modified) { + return Visit(std::static_pointer_cast(node), modified); +} +Status IRNodePass::VisitAfter(std::shared_ptr node, bool *const modified) { + return VisitAfter(std::static_pointer_cast(node), modified); +} Status IRNodePass::Visit(std::shared_ptr node, bool *const modified) { return Visit(std::static_pointer_cast(node), modified); } @@ -332,287 +292,5 @@ Status IRNodePass::Visit(std::shared_ptr node, bool *const m Status IRNodePass::Visit(std::shared_ptr node, bool *const modified) { return Visit(std::static_pointer_cast(node), modified); } - -////////////////////////////////// -// This section of code will be removed once the migration of optimizer from DatasetOp to DatasetNode is done. -// Driver method for TreePass -Status TreePass::Run(ExecutionTree *tree, bool *const modified) { - if (tree == nullptr || modified == nullptr) { - return Status(StatusCode::kMDUnexpectedError, "Null pointer passed to TreePass"); - } - // Initialize modified flag - *modified = false; - return this->RunOnTree(tree, modified); -} - -// Driver method for NodePass -Status NodePass::Run(ExecutionTree *tree, bool *const modified) { - if (tree == nullptr || modified == nullptr) { - return Status(StatusCode::kMDUnexpectedError, "Null pointer passed to NodePass"); - } - // Initialize modified flag - *modified = false; - std::shared_ptr root = tree->root(); - if (traversalOrder_ == Order::DFS) { - // DFS - return DFSNodeVisit(root, modified); - } else if (traversalOrder_ == Order::BFS) { - // BFS - return BFSNodeVisit(root, modified); - } - return Status::OK(); -} - -// Helper function to perform DFS visit -Status NodePass::DFSNodeVisit(std::shared_ptr node, bool *const modified) { - RETURN_IF_NOT_OK(node->PreAccept(this, modified)); - for (const auto &c : node->Children()) { - RETURN_IF_NOT_OK(this->DFSNodeVisit(c, modified)); - } - return node->Accept(this, modified); -} - -// Helper function to perform BFS visit -Status NodePass::BFSNodeVisit(std::shared_ptr root, bool *const modified) { - // Initialize bfs queue with root - std::queue> bfsQueue; - bfsQueue.push(root); - - // BFS loop - while (!bfsQueue.empty()) { - // Pop the front of the bfs queue - auto curNode = bfsQueue.front(); - bfsQueue.pop(); - - // Run node pass - RETURN_IF_NOT_OK(curNode->Accept(this, modified)); - - // Push children into bfs queue - for (const auto &c : curNode->Children()) { - bfsQueue.push(c); - } - } - return Status::OK(); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::PreRunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return PreRunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::PreRunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return PreRunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::PreRunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return PreRunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::PreRunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return PreRunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::PreRunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return PreRunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::PreRunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return PreRunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::PreRunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return PreRunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::PreRunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return PreRunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::PreRunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return PreRunOnNode(std::static_pointer_cast(node), modified); -} - -#ifndef ENABLE_ANDROID -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::PreRunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return PreRunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::PreRunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return PreRunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::PreRunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return PreRunOnNode(std::static_pointer_cast(node), modified); -} -#endif - -#ifdef ENABLE_PYTHON -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::RunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - -Status NodePass::PreRunOnNode(std::shared_ptr node, bool *const modified) { - // Fallback to base class visitor by default - return PreRunOnNode(std::static_pointer_cast(node), modified); -} -#endif -////////////////////////////////// } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pass.h b/mindspore/ccsrc/minddata/dataset/engine/opt/pass.h index 19c9a4989b7..6a0c2f3641c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pass.h +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pass.h @@ -46,6 +46,7 @@ class RootNode; class ShuffleNode; class SkipNode; class TakeNode; +class TFRecordNode; class TransferNode; class ZipNode; #ifdef ENABLE_PYTHON @@ -76,47 +77,6 @@ class TextFileNode; class TFRecordNode; #endif -////////////////////////////////// -// This section of code will be removed once the migration of optimizer from DatasetOp to DatasetNode is done. -class BatchOp; -class MapOp; -class ProjectOp; -class RenameOp; -class SkipOp; -class ShuffleOp; -class AlbumOp; -class RandomDataOp; -class RepeatOp; -class TakeOp; -class ZipOp; -class DeviceQueueOp; -class ImageFolderOp; -class MnistOp; -class ManifestOp; -class CifarOp; -class VOCOp; -class CocoOp; -class CelebAOp; -class EpochCtrlOp; -class BuildVocabOp; -class ConcatOp; -#ifndef ENABLE_ANDROID -class MindRecordOp; -class TFReaderOp; -class CacheOp; -class CacheMergeOp; -class CacheLookupOp; -class BuildSentencePieceVocabOp; -class ClueOp; -class CsvOp; -class TextFileOp; -#endif -#ifdef ENABLE_PYTHON -class FilterOp; -class GeneratorOp; -#endif -////////////////////////////////// - // The base class Pass is the basic unit of tree transformation. // The actual implementation of the passes will be derived from here. class IRPass : public std::enable_shared_from_this { @@ -200,6 +160,10 @@ class IRNodePass : public IRPass { virtual Status VisitAfter(std::shared_ptr node, bool *const modified); virtual Status Visit(std::shared_ptr node, bool *const modified); virtual Status VisitAfter(std::shared_ptr node, bool *const modified); +#ifndef ENABLE_ANDROID + virtual Status Visit(std::shared_ptr node, bool *const modified); + virtual Status VisitAfter(std::shared_ptr node, bool *const modified); +#endif virtual Status Visit(std::shared_ptr node, bool *const modified); virtual Status VisitAfter(std::shared_ptr node, bool *const modified); virtual Status Visit(std::shared_ptr node, bool *const modified); @@ -240,20 +204,18 @@ class IRNodePass : public IRPass { virtual Status VisitAfter(std::shared_ptr node, bool *const modified); virtual Status Visit(std::shared_ptr node, bool *const modified); virtual Status VisitAfter(std::shared_ptr node, bool *const modified); - virtual Status Visit(std::shared_ptr node, bool *const modified); - virtual Status VisitAfter(std::shared_ptr node, bool *const modified); - virtual Status Visit(std::shared_ptr node, bool *const modified); - virtual Status VisitAfter(std::shared_ptr node, bool *const modified); - virtual Status Visit(std::shared_ptr node, bool *const modified); - virtual Status VisitAfter(std::shared_ptr node, bool *const modified); #ifdef ENABLE_PYTHON virtual Status Visit(std::shared_ptr node, bool *const modified); virtual Status VisitAfter(std::shared_ptr node, bool *const modified); #endif -#ifndef ENABLE_ANDROID - virtual Status Visit(std::shared_ptr node, bool *const modified); - virtual Status VisitAfter(std::shared_ptr node, bool *const modified); -#endif + virtual Status Visit(std::shared_ptr node, bool *const modified); + virtual Status VisitAfter(std::shared_ptr node, bool *const modified); + virtual Status Visit(std::shared_ptr node, bool *const modified); + virtual Status VisitAfter(std::shared_ptr node, bool *const modified); + virtual Status Visit(std::shared_ptr node, bool *const modified); + virtual Status VisitAfter(std::shared_ptr node, bool *const modified); + virtual Status Visit(std::shared_ptr node, bool *const modified); + virtual Status VisitAfter(std::shared_ptr node, bool *const modified); // leaf-IR Node virtual Status Visit(std::shared_ptr node, bool *const modified); @@ -269,141 +231,6 @@ class IRNodePass : public IRPass { // Tree traversal order of the NodePass Order traversalOrder_; }; - -////////////////////////////////// -// This section of code will be removed once the migration of optimizer from DatasetOp to DatasetNode is done. -// The base class Pass is the basic unit of tree transformation. -// The actual implementation of the passes will be derived from here. -class Pass : public std::enable_shared_from_this { - public: - // Run the transformation pass against the execution tree. - // @param tree - Pointer to the execution tree to be transformed. - // @param modified - Pointer to the modified flag, - virtual Status Run(ExecutionTree *tree, bool *const modified) = 0; - - virtual ~Pass() = default; -}; - -// TreePass is a basic Pass class which performs transformation on ExecutionTree directly. -class TreePass : public Pass { - public: - /// \brief Run the transformation pass against the execution tree. - /// \param[in,out] tree Pointer to the execution tree to be transformed. - /// \param[in,out] modified Indicate if the tree was modified - Status Run(ExecutionTree *tree, bool *const modified) final; - - /// \brief Derived classes may implement the runOnTree function to implement tree transformation. - /// "modified" flag needs to be set to true if tree is modified during the pass execution. - /// \param[in,out] tree The tree to operate on. - /// \param[in,out] Indicate of the tree was modified. - /// \return Status The status code returned - virtual Status RunOnTree(ExecutionTree *tree, bool *const modified) { return Status::OK(); } -}; - -// NodePass is a base Pass class which performs transformation on node visiting. -// NodePass implements Visitor design pattern. -// The visiting happens twice for each node in the DFS traversal, one on the way down of the traversal, -// and the other when all the descending nodes are visited. -// Actual transformation is done by implementing a new derived class of NodePass. -// The derived class will implement the method Visit()/VisitAfter() passing specified node types -// it wants to action on them, overriding the ones defined in NodePass. -// If the derived class wants to perform the same action on all node types, -// it can simply implement the method Visit()/VisitAfter() passing the base class DatasetNode. -// This is made possible by overloading the method Visit()/VisitAfter() on each node type to fall back -// to call the Visit()/VisitAfter() in this parent NodePass class. -class NodePass : public Pass { - public: - // Tree traversal order - enum Order { DFS, BFS }; - - // Constructor - // Default DFS traversal - explicit NodePass(Order order = Order::DFS) { traversalOrder_ = order; } - - ~NodePass() = default; - - /// \brief Run the transformation pass against the execution tree - /// \param[in,out] tree Pointer to the execution tree to be transformed - /// \param[in,out] modified Indicator if the tree was changed - Status Run(ExecutionTree *tree, bool *const modified) final; - - /// \brief Derived classes may implement the PreRunOnNode function to implement any initial visit work on the way down - /// a tree traversal. "modified" flag needs to be set to true if tree is modified during the pass execution - /// \param[in] node The node being visited - /// \param[out] modified Indicator if the node was changed at all - /// \return Status The status code returned - virtual Status PreRunOnNode(std::shared_ptr node, bool *const modified) { return Status::OK(); } - - /// \brief Derived classes may implement the RunOnNode function to implement node level tree transformation - /// "modified" flag needs to be set to true if tree is modified during the pass execution - /// \param[in] node The node being visited - /// \param[out] modified Indicator if the node was changed at all. - /// \return Status The status code returned - virtual Status RunOnNode(std::shared_ptr node, bool *const modified) { return Status::OK(); } - - // Visit methods to be overridden. - // Note that member template can not be virtual, any op which wants to work with NodePass should declare RunOnNode - // of its own type and override "Accept" from DatasetOp. - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status PreRunOnNode(std::shared_ptr node, bool *const modified); - virtual Status PreRunOnNode(std::shared_ptr node, bool *const modified); - virtual Status PreRunOnNode(std::shared_ptr node, bool *const modified); - virtual Status PreRunOnNode(std::shared_ptr node, bool *const modified); - virtual Status PreRunOnNode(std::shared_ptr node, bool *const modified); - virtual Status PreRunOnNode(std::shared_ptr node, bool *const modified); - virtual Status PreRunOnNode(std::shared_ptr node, bool *const modified); - virtual Status PreRunOnNode(std::shared_ptr node, bool *const modified); - virtual Status PreRunOnNode(std::shared_ptr node, bool *const modified); -#ifndef ENABLE_ANDROID - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status PreRunOnNode(std::shared_ptr node, bool *const modified); - virtual Status PreRunOnNode(std::shared_ptr node, bool *const modified); - virtual Status PreRunOnNode(std::shared_ptr node, bool *const modified); -#endif -#ifdef ENABLE_PYTHON - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status RunOnNode(std::shared_ptr node, bool *const modified); - virtual Status PreRunOnNode(std::shared_ptr node, bool *const modified); -#endif - - private: - // Helper function to perform DFS visit - Status DFSNodeVisit(std::shared_ptr node, bool *const modified); - - // Helper function to perform BFS visit - Status BFSNodeVisit(std::shared_ptr root, bool *const modified); - - // Tree traversal order of the NodePass - Order traversalOrder_; -}; -////////////////////////////////// - } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.cc b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.cc index 5ae54a99a94..c88d890c492 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.cc @@ -23,6 +23,7 @@ #include "minddata/dataset/engine/ir/datasetops/map_node.h" #include "minddata/dataset/engine/ir/datasetops/repeat_node.h" #include "minddata/dataset/engine/ir/datasetops/skip_node.h" +#include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h" #include "minddata/dataset/engine/ir/datasetops/take_node.h" #include "minddata/dataset/engine/ir/datasetops/zip_node.h" @@ -157,6 +158,22 @@ Status CacheValidationPass::VisitAfter(std::shared_ptr node, bool *c return Status::OK(); } +Status CacheValidationPass::VisitAfter(std::shared_ptr node, bool *const modified) { + MS_LOG(DEBUG) << "CacheValidationPass::VisitAfter(): visiting " << node->Name() << "."; + if (!is_cached_) { + // If we are not in a cache path, then we must validate the file-based sharding config. + // If we are in a cache path, there is no file-based sharding so the check is not required. + if (!node->shard_equal_rows_ && node->dataset_files_.size() < static_cast(node->num_shards_)) { + RETURN_STATUS_UNEXPECTED("Invalid file, not enough tfrecord files provided.\n"); + } + } + // Reset the flag when this node is cached and is already visited + if (node->IsCached()) { + is_cached_ = false; + } + return Status::OK(); +} + Status CacheValidationPass::VisitAfter(std::shared_ptr node, bool *const modified) { MS_LOG(DEBUG) << "CacheValidationPass::VisitAfter(): visiting " << node->Name() << "."; // Reset the flag when all descendants are visited diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.h b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.h index d75f8bd73b8..b47a2f05689 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.h +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.h @@ -37,61 +37,67 @@ class CacheValidationPass : public IRNodePass { /// \brief Returns an error if BatchNode exists under a cache /// \param[in] node The node being visited - /// \param[inout] modified Indicator if the node was changed at all + /// \param[in,out] modified Indicator if the node was changed at all /// \return Status The status code returned Status Visit(std::shared_ptr node, bool *const modified) override; /// \brief Returns an error if ConcatNode exists under a cache /// \param[in] node The node being visited - /// \param[inout] modified Indicator if the node was changed at all + /// \param[in,out] modified Indicator if the node was changed at all /// \return Status The status code returned Status Visit(std::shared_ptr node, bool *const modified) override; /// \brief Returns an error if FilterNode exists under a cache /// \param[in] node The node being visited - /// \param[inout] modified Indicator if the node was changed at all + /// \param[in,out] modified Indicator if the node was changed at all /// \return Status The status code returned Status Visit(std::shared_ptr node, bool *const modified) override; /// \brief Returns an error if SkipNode exists under a cache /// \param[in] node The node being visited - /// \param[inout] modified Indicator if the node was changed at all + /// \param[in,out] modified Indicator if the node was changed at all /// \return Status The status code returned Status Visit(std::shared_ptr node, bool *const modified) override; /// \brief Returns an error if TakeNode exists under a cache /// \param[in] node The node being visited - /// \param[inout] modified Indicator if the node was changed at all + /// \param[in,out] modified Indicator if the node was changed at all /// \return Status The status code returned Status Visit(std::shared_ptr node, bool *const modified) override; /// \brief Returns an error if ZipNode exists under a cache /// \param[in] node The node being visited - /// \param[inout] modified Indicator if the node was changed at all + /// \param[in,out] modified Indicator if the node was changed at all /// \return Status The status code returned Status Visit(std::shared_ptr node, bool *const modified) override; /// \brief Returns an error if MapNode with non-deterministic tensor operations exists under a cache /// \param[in] node The node being visited - /// \param[inout] modified Indicator if the node was changed at all + /// \param[in,out] modified Indicator if the node was changed at all /// \return Status The status code returned Status Visit(std::shared_ptr node, bool *const modified) override; /// \brief Returns an error if there is a cache over another cache /// \param[in] node The node being visited - /// \param[inout] modified Indicator if the node was changed at all + /// \param[in,out] modified Indicator if the node was changed at all /// \return Status The status code returned Status Visit(std::shared_ptr node, bool *const modified) override; /// \brief Identifies and block repeat under cache scenarios /// \param[in] node The node being visited - /// \param[inout] modified Indicator if the node was changed at all + /// \param[in,out] modified Indicator if the node was changed at all /// \return Status The status code returned Status VisitAfter(std::shared_ptr node, bool *const modified) override; /// \brief Identifies the subtree above this node as not being cached /// \param[in] node The node being visited - /// \param[inout] modified Indicator if the node was changed at all + /// \param[in,out] modified Indicator if the node was changed at all + /// \return Status The status code returned + Status VisitAfter(std::shared_ptr node, bool *const modified) override; + + /// \brief Identifies the subtree above this node as not being cached + /// \param[in] node The node being visited + /// \param[in,out] modified Indicator if the node was changed at all /// \return Status The status code returned Status VisitAfter(std::shared_ptr node, bool *const modified) override;