forked from mindspore-Ecosystem/mindspore
!11078 dataset: minor fixes which resolve codedex CI alarms
From: @cathwong Reviewed-by: @nsyca,@robingrosman Signed-off-by: @nsyca
This commit is contained in:
commit
d46279dba2
|
@ -33,7 +33,7 @@ class Slice {
|
|||
|
||||
~Slice() = default;
|
||||
|
||||
bool valid() const { return !(start_ == 0 && stop_ == 0 && step_ == 0); }
|
||||
bool valid() const { return step_ != 0; }
|
||||
dsize_t start_;
|
||||
dsize_t stop_;
|
||||
dsize_t step_;
|
||||
|
|
|
@ -142,13 +142,13 @@ Status BatchNode::GetDatasetSize(const std::shared_ptr<DatasetSizeGetter> &size_
|
|||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status BatchNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status BatchNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->Visit(shared_from_base<BatchNode>(), modified);
|
||||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status BatchNode::AcceptAfter(IRNodePass *p, bool *modified) {
|
||||
Status BatchNode::AcceptAfter(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->VisitAfter(shared_from_base<BatchNode>(), modified);
|
||||
}
|
||||
|
|
|
@ -92,13 +92,13 @@ Status BuildSentenceVocabNode::ValidateParams() {
|
|||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status BuildSentenceVocabNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status BuildSentenceVocabNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->Visit(shared_from_base<BuildSentenceVocabNode>(), modified);
|
||||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status BuildSentenceVocabNode::AcceptAfter(IRNodePass *p, bool *modified) {
|
||||
Status BuildSentenceVocabNode::AcceptAfter(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->VisitAfter(shared_from_base<BuildSentenceVocabNode>(), modified);
|
||||
}
|
||||
|
|
|
@ -87,13 +87,13 @@ Status BuildVocabNode::ValidateParams() {
|
|||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status BuildVocabNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status BuildVocabNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->Visit(shared_from_base<BuildVocabNode>(), modified);
|
||||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status BuildVocabNode::AcceptAfter(IRNodePass *p, bool *modified) {
|
||||
Status BuildVocabNode::AcceptAfter(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->VisitAfter(shared_from_base<BuildVocabNode>(), modified);
|
||||
}
|
||||
|
|
|
@ -84,13 +84,13 @@ Status ConcatNode::Build(std::vector<std::shared_ptr<DatasetOp>> *const node_ops
|
|||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status ConcatNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status ConcatNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->Visit(shared_from_base<ConcatNode>(), modified);
|
||||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status ConcatNode::AcceptAfter(IRNodePass *p, bool *modified) {
|
||||
Status ConcatNode::AcceptAfter(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->VisitAfter(shared_from_base<ConcatNode>(), modified);
|
||||
}
|
||||
|
|
|
@ -376,14 +376,14 @@ Status DatasetNode::Remove() {
|
|||
}
|
||||
|
||||
// In DFS tree traversal, each node is visited twice. Accept is called on the first visit.
|
||||
Status DatasetNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status DatasetNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
// This method will only be called if its derived class does not implement one.
|
||||
return p->Visit(shared_from_this(), modified);
|
||||
}
|
||||
|
||||
// In DFS tree traversal, each node is visited twice. AcceptAfter is called on the second visit
|
||||
// after all child nodes are visited.
|
||||
Status DatasetNode::AcceptAfter(IRNodePass *p, bool *modified) {
|
||||
Status DatasetNode::AcceptAfter(IRNodePass *const p, bool *modified) {
|
||||
// This method will only be called if its derived class does not implement one.
|
||||
return p->VisitAfter(shared_from_this(), modified);
|
||||
}
|
||||
|
@ -431,11 +431,11 @@ Status DatasetNode::ValidateParams() {
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
Status MappableSourceNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status MappableSourceNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
return p->Visit(shared_from_base<MappableSourceNode>(), modified);
|
||||
}
|
||||
|
||||
Status NonMappableSourceNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status NonMappableSourceNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
return p->Visit(shared_from_base<MappableSourceNode>(), modified);
|
||||
}
|
||||
|
||||
|
|
|
@ -172,7 +172,7 @@ class DatasetNode : public std::enable_shared_from_this<DatasetNode> {
|
|||
|
||||
/// \brief Getter function for the parent node
|
||||
/// \return The parent node (of a node from a cloned IR tree)
|
||||
DatasetNode *Parent() const { return parent_; }
|
||||
DatasetNode *const Parent() const { return parent_; }
|
||||
|
||||
/// \brief Establish a parent-child relationship between this node and the input node.
|
||||
/// Used when building the IR tree.
|
||||
|
|
|
@ -62,13 +62,13 @@ Status FilterNode::ValidateParams() {
|
|||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status FilterNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status FilterNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->Visit(shared_from_base<FilterNode>(), modified);
|
||||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status FilterNode::AcceptAfter(IRNodePass *p, bool *modified) {
|
||||
Status FilterNode::AcceptAfter(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->VisitAfter(shared_from_base<FilterNode>(), modified);
|
||||
}
|
||||
|
|
|
@ -105,13 +105,13 @@ Status MapNode::ValidateParams() {
|
|||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status MapNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status MapNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->Visit(shared_from_base<MapNode>(), modified);
|
||||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status MapNode::AcceptAfter(IRNodePass *p, bool *modified) {
|
||||
Status MapNode::AcceptAfter(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->VisitAfter(shared_from_base<MapNode>(), modified);
|
||||
}
|
||||
|
|
|
@ -61,7 +61,6 @@ class MapNode : public DatasetNode {
|
|||
/// \brief Getter of tensor operations
|
||||
/// \return Vector of operations the Map node will process
|
||||
const auto &TensorOperations() const { return operations_; }
|
||||
auto &TensorOperations() { return operations_; }
|
||||
|
||||
/// \brief Base-class override for accepting IRNodePass visitor
|
||||
/// \param[in] p The node to visit
|
||||
|
|
|
@ -72,13 +72,13 @@ Status RepeatNode::GetDatasetSize(const std::shared_ptr<DatasetSizeGetter> &size
|
|||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status RepeatNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status RepeatNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->Visit(shared_from_base<RepeatNode>(), modified);
|
||||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status RepeatNode::AcceptAfter(IRNodePass *p, bool *modified) {
|
||||
Status RepeatNode::AcceptAfter(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->VisitAfter(shared_from_base<RepeatNode>(), modified);
|
||||
}
|
||||
|
|
|
@ -30,6 +30,7 @@ namespace dataset {
|
|||
RootNode::RootNode(std::shared_ptr<DatasetNode> child) : DatasetNode() {
|
||||
// The root node's parent must remain nullptr, which is set in the constructor of DatasetNode.
|
||||
AddChild(child);
|
||||
num_epochs_ = 0;
|
||||
}
|
||||
|
||||
std::shared_ptr<DatasetNode> RootNode::Copy() {
|
||||
|
@ -75,13 +76,13 @@ Status RootNode::ValidateParams() {
|
|||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status RootNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status RootNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->Visit(shared_from_base<RootNode>(), modified);
|
||||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status RootNode::AcceptAfter(IRNodePass *p, bool *modified) {
|
||||
Status RootNode::AcceptAfter(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->VisitAfter(shared_from_base<RootNode>(), modified);
|
||||
}
|
||||
|
|
|
@ -73,13 +73,13 @@ Status SkipNode::GetDatasetSize(const std::shared_ptr<DatasetSizeGetter> &size_g
|
|||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status SkipNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status SkipNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->Visit(shared_from_base<SkipNode>(), modified);
|
||||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status SkipNode::AcceptAfter(IRNodePass *p, bool *modified) {
|
||||
Status SkipNode::AcceptAfter(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->VisitAfter(shared_from_base<SkipNode>(), modified);
|
||||
}
|
||||
|
|
|
@ -71,13 +71,13 @@ Status TakeNode::GetDatasetSize(const std::shared_ptr<DatasetSizeGetter> &size_g
|
|||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status TakeNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status TakeNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->Visit(shared_from_base<TakeNode>(), modified);
|
||||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status TakeNode::AcceptAfter(IRNodePass *p, bool *modified) {
|
||||
Status TakeNode::AcceptAfter(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->VisitAfter(shared_from_base<TakeNode>(), modified);
|
||||
}
|
||||
|
|
|
@ -106,13 +106,13 @@ Status TransferNode::Build(std::vector<std::shared_ptr<DatasetOp>> *const node_o
|
|||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status TransferNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status TransferNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->Visit(shared_from_base<TransferNode>(), modified);
|
||||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status TransferNode::AcceptAfter(IRNodePass *p, bool *modified) {
|
||||
Status TransferNode::AcceptAfter(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->VisitAfter(shared_from_base<TransferNode>(), modified);
|
||||
}
|
||||
|
|
|
@ -81,13 +81,13 @@ Status ZipNode::GetDatasetSize(const std::shared_ptr<DatasetSizeGetter> &size_ge
|
|||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status ZipNode::Accept(IRNodePass *p, bool *modified) {
|
||||
Status ZipNode::Accept(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->Visit(shared_from_base<ZipNode>(), modified);
|
||||
}
|
||||
|
||||
// Visitor accepting method for IRNodePass
|
||||
Status ZipNode::AcceptAfter(IRNodePass *p, bool *modified) {
|
||||
Status ZipNode::AcceptAfter(IRNodePass *const p, bool *modified) {
|
||||
// Downcast shared pointer then call visitor
|
||||
return p->VisitAfter(shared_from_base<ZipNode>(), modified);
|
||||
}
|
||||
|
|
|
@ -91,6 +91,8 @@ Status IRTreePass::Run(std::shared_ptr<DatasetNode> root_ir, bool *modified) {
|
|||
if (root_ir == nullptr || modified == nullptr) {
|
||||
return Status(StatusCode::kUnexpectedError, "Null pointer passed to TreePass");
|
||||
}
|
||||
// Initialize modified flag
|
||||
*modified = false;
|
||||
return this->RunOnTree(root_ir, modified);
|
||||
}
|
||||
|
||||
|
@ -99,6 +101,8 @@ Status IRNodePass::Run(std::shared_ptr<DatasetNode> root_ir, bool *modified) {
|
|||
if (root_ir == nullptr || modified == nullptr) {
|
||||
return Status(StatusCode::kUnexpectedError, "Null pointer passed to NodePass");
|
||||
}
|
||||
// Initialize modified flag
|
||||
*modified = false;
|
||||
if (traversalOrder_ == Order::DFS) {
|
||||
// DFS
|
||||
return DFSNodeVisit(root_ir, modified);
|
||||
|
@ -114,13 +118,13 @@ Status IRNodePass::DFSNodeVisit(std::shared_ptr<DatasetNode> node_ir, bool *modi
|
|||
bool m = false;
|
||||
|
||||
RETURN_IF_NOT_OK(node_ir->Accept(this, &m));
|
||||
*modified |= m;
|
||||
*modified = *modified || m;
|
||||
for (const auto &c : node_ir->Children()) {
|
||||
RETURN_IF_NOT_OK(this->DFSNodeVisit(c, &m));
|
||||
*modified |= m;
|
||||
*modified = *modified || m;
|
||||
}
|
||||
RETURN_IF_NOT_OK(node_ir->AcceptAfter(this, &m));
|
||||
*modified |= m;
|
||||
*modified = *modified || m;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
@ -140,7 +144,7 @@ Status IRNodePass::BFSNodeVisit(std::shared_ptr<DatasetNode> node_ir, bool *modi
|
|||
|
||||
// Run node pass
|
||||
RETURN_IF_NOT_OK(curNode->Accept(this, &m));
|
||||
*modified |= m;
|
||||
*modified = *modified || m;
|
||||
|
||||
// Push children into bfs queue
|
||||
for (const auto &c : curNode->Children()) {
|
||||
|
@ -274,6 +278,8 @@ Status TreePass::Run(ExecutionTree *tree, bool *modified) {
|
|||
if (tree == nullptr || modified == nullptr) {
|
||||
return Status(StatusCode::kUnexpectedError, "Null pointer passed to TreePass");
|
||||
}
|
||||
// Initialize modified flag
|
||||
*modified = false;
|
||||
return this->RunOnTree(tree, modified);
|
||||
}
|
||||
|
||||
|
@ -282,6 +288,8 @@ Status NodePass::Run(ExecutionTree *tree, bool *modified) {
|
|||
if (tree == nullptr || modified == nullptr) {
|
||||
return Status(StatusCode::kUnexpectedError, "Null pointer passed to NodePass");
|
||||
}
|
||||
// Initialize modified flag
|
||||
*modified = false;
|
||||
std::shared_ptr<DatasetOp> root = tree->root();
|
||||
if (traversalOrder_ == Order::DFS) {
|
||||
// DFS
|
||||
|
|
|
@ -31,6 +31,11 @@ namespace dataset {
|
|||
|
||||
TreeAdapter::TreeAdapter(UsageFlag usage) : usage_(usage), tree_state_(kCompileStateInit) {
|
||||
optimize_ = common::GetEnv("OPTIMIZE") == "true";
|
||||
|
||||
// Initialize profiling parameters
|
||||
cur_batch_num_ = 0;
|
||||
cur_connector_size_ = 0;
|
||||
cur_connector_capacity_ = 0;
|
||||
}
|
||||
|
||||
Status TreeAdapter::PrePass(std::shared_ptr<DatasetNode> ir) {
|
||||
|
@ -98,7 +103,7 @@ Status TreeAdapter::PostPass(std::shared_ptr<DatasetNode> ir) {
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
Status TreeAdapter::BuildExecutionTreeRecur(std::shared_ptr<DatasetNode> ir, std::shared_ptr<DatasetOp> *op) {
|
||||
Status TreeAdapter::BuildExecutionTreeRecur(std::shared_ptr<DatasetNode> ir, std::shared_ptr<DatasetOp> *const op) {
|
||||
// Build the DatasetOp ExecutionTree from the optimized IR tree
|
||||
std::vector<std::shared_ptr<DatasetOp>> ops;
|
||||
RETURN_IF_NOT_OK(ir->Build(&ops));
|
||||
|
@ -143,11 +148,6 @@ Status TreeAdapter::Build(std::shared_ptr<DatasetNode> root_ir, int32_t num_epoc
|
|||
// After the tree is prepared, the col_name_id_map can safely be obtained
|
||||
column_name_map_ = tree_->root()->column_name_id_map();
|
||||
|
||||
// Profiling parameters init
|
||||
cur_batch_num_ = 0;
|
||||
cur_connector_size_ = 0;
|
||||
cur_connector_capacity_ = 0;
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
|
@ -568,7 +568,6 @@ class CutOutOperation : public TensorOperation {
|
|||
private:
|
||||
int32_t length_;
|
||||
int32_t num_patches_;
|
||||
ImageBatchFormat image_batch_format_;
|
||||
};
|
||||
|
||||
class DvppDecodeResizeCropOperation : public TensorOperation {
|
||||
|
|
Loading…
Reference in New Issue