diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor_helpers.h b/mindspore/ccsrc/minddata/dataset/core/tensor_helpers.h index fa04d6be007..e8c1d4fd0eb 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor_helpers.h +++ b/mindspore/ccsrc/minddata/dataset/core/tensor_helpers.h @@ -33,7 +33,7 @@ class Slice { ~Slice() = default; - bool valid() const { return !(start_ == 0 && stop_ == 0 && step_ == 0); } + bool valid() const { return step_ != 0; } dsize_t start_; dsize_t stop_; dsize_t step_; diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/batch_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/batch_node.cc index 588f45ace6e..ca7cd1d0cf1 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/batch_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/batch_node.cc @@ -142,13 +142,13 @@ Status BatchNode::GetDatasetSize(const std::shared_ptr &size_ } // Visitor accepting method for IRNodePass -Status BatchNode::Accept(IRNodePass *p, bool *modified) { +Status BatchNode::Accept(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->Visit(shared_from_base(), modified); } // Visitor accepting method for IRNodePass -Status BatchNode::AcceptAfter(IRNodePass *p, bool *modified) { +Status BatchNode::AcceptAfter(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->VisitAfter(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/build_sentence_piece_vocab_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/build_sentence_piece_vocab_node.cc index d643ac8607f..c1ff38d9b03 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/build_sentence_piece_vocab_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/build_sentence_piece_vocab_node.cc @@ -92,13 +92,13 @@ Status BuildSentenceVocabNode::ValidateParams() { } // Visitor accepting method for IRNodePass -Status BuildSentenceVocabNode::Accept(IRNodePass *p, bool *modified) { +Status BuildSentenceVocabNode::Accept(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->Visit(shared_from_base(), modified); } // Visitor accepting method for IRNodePass -Status BuildSentenceVocabNode::AcceptAfter(IRNodePass *p, bool *modified) { +Status BuildSentenceVocabNode::AcceptAfter(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->VisitAfter(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/build_vocab_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/build_vocab_node.cc index d085c4c0fe9..bb0b80bd0a2 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/build_vocab_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/build_vocab_node.cc @@ -87,13 +87,13 @@ Status BuildVocabNode::ValidateParams() { } // Visitor accepting method for IRNodePass -Status BuildVocabNode::Accept(IRNodePass *p, bool *modified) { +Status BuildVocabNode::Accept(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->Visit(shared_from_base(), modified); } // Visitor accepting method for IRNodePass -Status BuildVocabNode::AcceptAfter(IRNodePass *p, bool *modified) { +Status BuildVocabNode::AcceptAfter(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->VisitAfter(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/concat_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/concat_node.cc index d255242ac2a..2349312f5dd 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/concat_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/concat_node.cc @@ -84,13 +84,13 @@ Status ConcatNode::Build(std::vector> *const node_ops } // Visitor accepting method for IRNodePass -Status ConcatNode::Accept(IRNodePass *p, bool *modified) { +Status ConcatNode::Accept(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->Visit(shared_from_base(), modified); } // Visitor accepting method for IRNodePass -Status ConcatNode::AcceptAfter(IRNodePass *p, bool *modified) { +Status ConcatNode::AcceptAfter(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->VisitAfter(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.cc index 55a84970040..78e298a8ffc 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.cc @@ -376,14 +376,14 @@ Status DatasetNode::Remove() { } // In DFS tree traversal, each node is visited twice. Accept is called on the first visit. -Status DatasetNode::Accept(IRNodePass *p, bool *modified) { +Status DatasetNode::Accept(IRNodePass *const p, bool *modified) { // This method will only be called if its derived class does not implement one. return p->Visit(shared_from_this(), modified); } // In DFS tree traversal, each node is visited twice. AcceptAfter is called on the second visit // after all child nodes are visited. -Status DatasetNode::AcceptAfter(IRNodePass *p, bool *modified) { +Status DatasetNode::AcceptAfter(IRNodePass *const p, bool *modified) { // This method will only be called if its derived class does not implement one. return p->VisitAfter(shared_from_this(), modified); } @@ -431,11 +431,11 @@ Status DatasetNode::ValidateParams() { return Status::OK(); } -Status MappableSourceNode::Accept(IRNodePass *p, bool *modified) { +Status MappableSourceNode::Accept(IRNodePass *const p, bool *modified) { return p->Visit(shared_from_base(), modified); } -Status NonMappableSourceNode::Accept(IRNodePass *p, bool *modified) { +Status NonMappableSourceNode::Accept(IRNodePass *const p, bool *modified) { return p->Visit(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.h b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.h index 9aef52bf407..8720b557e9e 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.h +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.h @@ -172,7 +172,7 @@ class DatasetNode : public std::enable_shared_from_this { /// \brief Getter function for the parent node /// \return The parent node (of a node from a cloned IR tree) - DatasetNode *Parent() const { return parent_; } + DatasetNode *const Parent() const { return parent_; } /// \brief Establish a parent-child relationship between this node and the input node. /// Used when building the IR tree. diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/filter_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/filter_node.cc index 5da04af1b1d..9355b0b1cfa 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/filter_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/filter_node.cc @@ -62,13 +62,13 @@ Status FilterNode::ValidateParams() { } // Visitor accepting method for IRNodePass -Status FilterNode::Accept(IRNodePass *p, bool *modified) { +Status FilterNode::Accept(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->Visit(shared_from_base(), modified); } // Visitor accepting method for IRNodePass -Status FilterNode::AcceptAfter(IRNodePass *p, bool *modified) { +Status FilterNode::AcceptAfter(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->VisitAfter(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.cc index e2e1fe073bd..2c78c9a9fbc 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.cc @@ -105,13 +105,13 @@ Status MapNode::ValidateParams() { } // Visitor accepting method for IRNodePass -Status MapNode::Accept(IRNodePass *p, bool *modified) { +Status MapNode::Accept(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->Visit(shared_from_base(), modified); } // Visitor accepting method for IRNodePass -Status MapNode::AcceptAfter(IRNodePass *p, bool *modified) { +Status MapNode::AcceptAfter(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->VisitAfter(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.h b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.h index 59a8b1a4b89..0f5c207d7eb 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.h +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.h @@ -61,7 +61,6 @@ class MapNode : public DatasetNode { /// \brief Getter of tensor operations /// \return Vector of operations the Map node will process const auto &TensorOperations() const { return operations_; } - auto &TensorOperations() { return operations_; } /// \brief Base-class override for accepting IRNodePass visitor /// \param[in] p The node to visit diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/repeat_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/repeat_node.cc index cd4b4109ed0..1825b8b9682 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/repeat_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/repeat_node.cc @@ -72,13 +72,13 @@ Status RepeatNode::GetDatasetSize(const std::shared_ptr &size } // Visitor accepting method for IRNodePass -Status RepeatNode::Accept(IRNodePass *p, bool *modified) { +Status RepeatNode::Accept(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->Visit(shared_from_base(), modified); } // Visitor accepting method for IRNodePass -Status RepeatNode::AcceptAfter(IRNodePass *p, bool *modified) { +Status RepeatNode::AcceptAfter(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->VisitAfter(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/root_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/root_node.cc index 0f116d85e9f..4e29b455e02 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/root_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/root_node.cc @@ -30,6 +30,7 @@ namespace dataset { RootNode::RootNode(std::shared_ptr child) : DatasetNode() { // The root node's parent must remain nullptr, which is set in the constructor of DatasetNode. AddChild(child); + num_epochs_ = 0; } std::shared_ptr RootNode::Copy() { @@ -75,13 +76,13 @@ Status RootNode::ValidateParams() { } // Visitor accepting method for IRNodePass -Status RootNode::Accept(IRNodePass *p, bool *modified) { +Status RootNode::Accept(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->Visit(shared_from_base(), modified); } // Visitor accepting method for IRNodePass -Status RootNode::AcceptAfter(IRNodePass *p, bool *modified) { +Status RootNode::AcceptAfter(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->VisitAfter(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/skip_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/skip_node.cc index a5e6d74ba48..13042e8e93f 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/skip_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/skip_node.cc @@ -73,13 +73,13 @@ Status SkipNode::GetDatasetSize(const std::shared_ptr &size_g } // Visitor accepting method for IRNodePass -Status SkipNode::Accept(IRNodePass *p, bool *modified) { +Status SkipNode::Accept(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->Visit(shared_from_base(), modified); } // Visitor accepting method for IRNodePass -Status SkipNode::AcceptAfter(IRNodePass *p, bool *modified) { +Status SkipNode::AcceptAfter(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->VisitAfter(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/take_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/take_node.cc index 62a98c86bdb..c270847900b 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/take_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/take_node.cc @@ -71,13 +71,13 @@ Status TakeNode::GetDatasetSize(const std::shared_ptr &size_g } // Visitor accepting method for IRNodePass -Status TakeNode::Accept(IRNodePass *p, bool *modified) { +Status TakeNode::Accept(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->Visit(shared_from_base(), modified); } // Visitor accepting method for IRNodePass -Status TakeNode::AcceptAfter(IRNodePass *p, bool *modified) { +Status TakeNode::AcceptAfter(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->VisitAfter(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/transfer_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/transfer_node.cc index 8d48d666025..c9d4f7c7063 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/transfer_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/transfer_node.cc @@ -106,13 +106,13 @@ Status TransferNode::Build(std::vector> *const node_o } // Visitor accepting method for IRNodePass -Status TransferNode::Accept(IRNodePass *p, bool *modified) { +Status TransferNode::Accept(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->Visit(shared_from_base(), modified); } // Visitor accepting method for IRNodePass -Status TransferNode::AcceptAfter(IRNodePass *p, bool *modified) { +Status TransferNode::AcceptAfter(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->VisitAfter(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/zip_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/zip_node.cc index 12543d908f1..2ad16c882d3 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/zip_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/zip_node.cc @@ -81,13 +81,13 @@ Status ZipNode::GetDatasetSize(const std::shared_ptr &size_ge } // Visitor accepting method for IRNodePass -Status ZipNode::Accept(IRNodePass *p, bool *modified) { +Status ZipNode::Accept(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->Visit(shared_from_base(), modified); } // Visitor accepting method for IRNodePass -Status ZipNode::AcceptAfter(IRNodePass *p, bool *modified) { +Status ZipNode::AcceptAfter(IRNodePass *const p, bool *modified) { // Downcast shared pointer then call visitor return p->VisitAfter(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc b/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc index d76069b2be8..ecb65eb5d0b 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc @@ -91,6 +91,8 @@ Status IRTreePass::Run(std::shared_ptr root_ir, bool *modified) { if (root_ir == nullptr || modified == nullptr) { return Status(StatusCode::kUnexpectedError, "Null pointer passed to TreePass"); } + // Initialize modified flag + *modified = false; return this->RunOnTree(root_ir, modified); } @@ -99,6 +101,8 @@ Status IRNodePass::Run(std::shared_ptr root_ir, bool *modified) { if (root_ir == nullptr || modified == nullptr) { return Status(StatusCode::kUnexpectedError, "Null pointer passed to NodePass"); } + // Initialize modified flag + *modified = false; if (traversalOrder_ == Order::DFS) { // DFS return DFSNodeVisit(root_ir, modified); @@ -114,13 +118,13 @@ Status IRNodePass::DFSNodeVisit(std::shared_ptr node_ir, bool *modi bool m = false; RETURN_IF_NOT_OK(node_ir->Accept(this, &m)); - *modified |= m; + *modified = *modified || m; for (const auto &c : node_ir->Children()) { RETURN_IF_NOT_OK(this->DFSNodeVisit(c, &m)); - *modified |= m; + *modified = *modified || m; } RETURN_IF_NOT_OK(node_ir->AcceptAfter(this, &m)); - *modified |= m; + *modified = *modified || m; return Status::OK(); } @@ -140,7 +144,7 @@ Status IRNodePass::BFSNodeVisit(std::shared_ptr node_ir, bool *modi // Run node pass RETURN_IF_NOT_OK(curNode->Accept(this, &m)); - *modified |= m; + *modified = *modified || m; // Push children into bfs queue for (const auto &c : curNode->Children()) { @@ -274,6 +278,8 @@ Status TreePass::Run(ExecutionTree *tree, bool *modified) { if (tree == nullptr || modified == nullptr) { return Status(StatusCode::kUnexpectedError, "Null pointer passed to TreePass"); } + // Initialize modified flag + *modified = false; return this->RunOnTree(tree, modified); } @@ -282,6 +288,8 @@ Status NodePass::Run(ExecutionTree *tree, bool *modified) { if (tree == nullptr || modified == nullptr) { return Status(StatusCode::kUnexpectedError, "Null pointer passed to NodePass"); } + // Initialize modified flag + *modified = false; std::shared_ptr root = tree->root(); if (traversalOrder_ == Order::DFS) { // DFS diff --git a/mindspore/ccsrc/minddata/dataset/engine/tree_adapter.cc b/mindspore/ccsrc/minddata/dataset/engine/tree_adapter.cc index 772aef9224a..9d2cc97efea 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/tree_adapter.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/tree_adapter.cc @@ -31,6 +31,11 @@ namespace dataset { TreeAdapter::TreeAdapter(UsageFlag usage) : usage_(usage), tree_state_(kCompileStateInit) { optimize_ = common::GetEnv("OPTIMIZE") == "true"; + + // Initialize profiling parameters + cur_batch_num_ = 0; + cur_connector_size_ = 0; + cur_connector_capacity_ = 0; } Status TreeAdapter::PrePass(std::shared_ptr ir) { @@ -98,7 +103,7 @@ Status TreeAdapter::PostPass(std::shared_ptr ir) { return Status::OK(); } -Status TreeAdapter::BuildExecutionTreeRecur(std::shared_ptr ir, std::shared_ptr *op) { +Status TreeAdapter::BuildExecutionTreeRecur(std::shared_ptr ir, std::shared_ptr *const op) { // Build the DatasetOp ExecutionTree from the optimized IR tree std::vector> ops; RETURN_IF_NOT_OK(ir->Build(&ops)); @@ -143,11 +148,6 @@ Status TreeAdapter::Build(std::shared_ptr root_ir, int32_t num_epoc // After the tree is prepared, the col_name_id_map can safely be obtained column_name_map_ = tree_->root()->column_name_id_map(); - // Profiling parameters init - cur_batch_num_ = 0; - cur_connector_size_ = 0; - cur_connector_capacity_ = 0; - return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/include/vision.h b/mindspore/ccsrc/minddata/dataset/include/vision.h index c4bbfa42572..082231fdf87 100644 --- a/mindspore/ccsrc/minddata/dataset/include/vision.h +++ b/mindspore/ccsrc/minddata/dataset/include/vision.h @@ -568,7 +568,6 @@ class CutOutOperation : public TensorOperation { private: int32_t length_; int32_t num_patches_; - ImageBatchFormat image_batch_format_; }; class DvppDecodeResizeCropOperation : public TensorOperation {