p12346870
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!830 Update the print methods for displaying execution tree info diagnostics 

Merge pull request !830 from Jamie/operator_print
This commit is contained in:
mindspore-ci-bot 2020-04-29 22:07:11 +08:00 committed by Gitee
parent 98939d839c 2facd80c8d
commit 323ea9d869
31 changed files with 483 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc
View File

@ -14,6 +14,7 @@
* limitations under the License.
*/
#include "dataset/engine/datasetops/barrier_op.h"
#include <iomanip>
#include <utility>
#include "dataset/core/constants.h"
#include "dataset/engine/data_buffer.h"
@ -214,10 +215,19 @@ Status BarrierOp::getNextTensorRow(TensorRow *new_row) {
// A function that prints info about the Operator
void BarrierOp::Print(std::ostream &out, bool show_all) const {
// Call base class printer first
PipelineOp::Print(out, show_all);
out << "\nBarrierOp:\n"
<< "\nCondition " << condition_name_ << "\n\n";
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <BarrierOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nCondition: " << condition_name_ << "\n\n";
}
}
// overwrite function and handle eof

18
mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc
View File

@ -15,6 +15,7 @@
*/
#include "dataset/engine/datasetops/batch_op.h"
#include <utility>
#include <iomanip>
#include "common/utils.h"
#include "dataset/engine/data_buffer.h"
#include "dataset/engine/db_connector.h"
@ -102,10 +103,19 @@ Status BatchOp::operator()() {
}
void BatchOp::Print(std::ostream &out, bool show_all) const {
ParallelOp::Print(out, show_all);
out << "\nBatchOp:\n"
<< "number of parallel workers: " << num_workers_ << "\nBatch size: " << start_batch_size_
<< "\nDrop remainder: " << (drop_ ? "yes" : "no") << "\n\n";
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <BatchOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << " [batch size: " << start_batch_size_ << "]\n";
} else {
// Call the super class for displaying any common detailed info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nStart batch size: " << start_batch_size_ << "\nDrop remainder: " << (drop_ ? "yes" : "no") << "\n\n";
}
}
Status BatchOp::BatchRows(const std::unique_ptr<TensorQTable> *source_table,

24
mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc
View File

@ -92,18 +92,24 @@ void DatasetOp::CreateConnector(int32_t num_producers, int32_t num_consumers) {
// A print method typically used for debugging. showAll of true will recursively descend to child prints
void DatasetOp::Print(std::ostream &out, bool show_all) const {
// When show_all is false, we display a 1 liner piece of text for the op.
// When show_all is true, we display more detailed output for the op.
// Derived printers should show their own header info, then call base class printer, followed by
// derived-specific items.
// For now, the base class doesn't have any summary info to show so it's a no-op in that case.
if (show_all) {
// The detailed display will show common base class info of the op. Allow the derived class to print
// it's own id and name though as the first line.
out << "\nNumber of children : " << child_.size();
for (size_t i = 0; i < child_.size(); i++) {
child_[i]->Print(out, show_all);
out << "\n Child[" << i << "] id: " << child_[i]->id();
}
}
out << "\n-------------------------"
<< "\nOperator # : " << operator_id_ << "\nNumber of children : " << child_.size()
<< "\nNumber of parents : " << parent_.size() << "\nConnector queue size : " << oc_queue_size_
<< "\nOperator control flags : 0x" << std::hex << std::setw(8) << std::setfill('0') << op_ctrl_flags_ << std::dec
<< std::setfill(' ') << "\nHas parents:\n";
for (size_t i = 0; i < parent_.size(); i++) {
out << "Parent[" << i << "] id: " << parent_[i]->id() << "\n";
out << "\nNumber of parents : " << parent_.size();
for (size_t i = 0; i < parent_.size(); i++) {
out << "\n Parent[" << i << "] id: " << parent_[i]->id();
}
out << "\nConnector queue size : " << oc_queue_size_ << "\nOperator control flags : 0x" << std::hex
<< std::setw(8) << std::setfill('0') << op_ctrl_flags_ << std::dec << std::setfill(' ');
}
}

18
mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
#include "dataset/engine/datasetops/device_queue_op.h"
#include <iomanip>
#include <iostream>
#include <memory>
@ -246,9 +246,19 @@ Status DeviceQueueOp::SendDataToCPU() {
}
void DeviceQueueOp::Print(std::ostream &out, bool show_all) const {
PipelineOp::Print(out, show_all);
out << "DeviceQueueOp: channelName: " << channel_name_ << ", prefetchSize: " << prefetch_size_ << '\n';
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <DeviceQueueOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nChannel name: " << channel_name_ << "\nPrefetch size: " << prefetch_size_ << "\n\n";
}
}
} // namespace dataset
} // namespace mindspore

25
mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc
View File

@ -16,6 +16,7 @@
#include "dataset/engine/datasetops/filter_op.h"
#include <algorithm>
#include <cstring>
#include <iomanip>
#include <iostream>
#include <memory>
#include <vector>
@ -88,14 +89,22 @@ Status FilterOp::ValidateInColumns(const std::unordered_map<std::string, int32_t
// A print method typically used for debugging.
void FilterOp::Print(std::ostream &out, bool show_all) const {
// Call base class printer first.
ParallelOp::Print(out, show_all);
// Then display our own stuff.
out << "\nFilterOp:";
out << "\n Input column names:";
for (size_t i = 0; i < in_columns_.size(); i++) {
out << " " << in_columns_[i];
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <FilterOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nInput column names:";
for (size_t i = 0; i < in_columns_.size(); i++) {
out << " " << in_columns_[i];
}
out << "\n\n";
}
}

34
mindspore/ccsrc/dataset/engine/datasetops/map_op.cc
View File

@ -15,6 +15,7 @@
*/
#include "dataset/engine/datasetops/map_op.h"
#include <cstring>
#include <iomanip>
#include <iostream>
#include <memory>
#include <vector>
@ -81,20 +82,27 @@ int32_t MapOp::num_consumers() const {
// A print method typically used for debugging
void MapOp::Print(std::ostream &out, bool show_all) const {
// Call base class printer first
ParallelOp::Print(out, show_all);
// Then display our own stuff
out << "\nMapOp:";
out << "\n Input column names:";
for (size_t i = 0; i < in_columns_.size(); i++) {
out << " " << in_columns_[i];
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <MapOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nInput column names:";
for (size_t i = 0; i < in_columns_.size(); i++) {
out << " " << in_columns_[i];
}
out << "\n TensorOps:";
for (size_t i = 0; i < tfuncs_.size(); i++) {
out << " " << tfuncs_[i];
}
out << "\n\n";
}
out << "\n TensorOps:";
for (size_t i = 0; i < tfuncs_.size(); i++) {
out << " " << tfuncs_[i];
}
out << "\n";
}
// This class functor will provide the master loop that drives the logic for performing the work

16
mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc
View File

@ -55,12 +55,16 @@ Status ParallelOp::CreateWorkerConnector(int32_t worker_connector_size) {
// A print method typically used for debugging
void ParallelOp::Print(std::ostream &out, bool show_all) const {
// Call base class printer first
DatasetOp::Print(out, show_all);
// Then show our own stuff
out << "ParallelOp:";
out << "\n Num workers : " << num_workers_ << "\n";
// Summary 1-liner print
if (!show_all) {
out << " [workers: " << num_workers_ << "]";
// Call super class printer
DatasetOp::Print(out, show_all);
} else {
// Detailed print
DatasetOp::Print(out, show_all);
out << "\nNum workers: " << num_workers_;
}
}
// Override base class reset to provide reset actions specific to the ParallelOp class.

26
mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc
View File

@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <iomanip>
#include <iostream>
#include "dataset/engine/datasetops/pipeline_op.h"
@ -23,11 +24,26 @@ PipelineOp::PipelineOp(int32_t op_connector_size) : DatasetOp(op_connector_size)
// A print method typically used for debugging
void PipelineOp::Print(std::ostream &out, bool show_all) const {
// Call base class printer first
DatasetOp::Print(out, show_all);
// Then display our own stuff for the pipeline op
// out << "This is a pipeline op print. nothing to display here at the moment.\n";
// Summary 1-liner print
if (!show_all) {
out << " [workers: ";
if (this->inlined()) {
out << "0 (inlined)]";
} else {
out << "1]"; // Pipeline ops only have 1 worker
}
// Call super class printer
DatasetOp::Print(out, show_all);
} else {
// Detailed print
DatasetOp::Print(out, show_all);
out << "\nNum workers: ";
if (this->inlined()) {
out << "0 (inlined)";
} else {
out << "1"; // Pipeline ops only have 1 worker
}
}
}
} // namespace dataset
} // namespace mindspore

22
mindspore/ccsrc/dataset/engine/datasetops/project_op.cc
View File

@ -16,6 +16,7 @@
#include "dataset/engine/datasetops/project_op.h"
#include <algorithm>
#include <iomanip>
#include <iostream>
#include <string>
#include <unordered_map>
@ -49,12 +50,23 @@ ProjectOp::ProjectOp(const std::vector<std::string> &columns_to_project)
: PipelineOp(0), columns_to_project_(columns_to_project) {}
void ProjectOp::Print(std::ostream &out, bool show_all) const {
PipelineOp::Print(out, show_all);
out << "ProjectOp: columns that are projected: ";
for (size_t i = 0; i < columns_to_project_.size(); i++) {
out << columns_to_project_[i] << " ";
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <ProjectOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nColumns that are projected:";
for (size_t i = 0; i < columns_to_project_.size(); i++) {
out << "\n" << columns_to_project_[i];
}
out << "\n\n";
}
out << '\n';
}
// Gets a buffer from the child operator and projects the buffer.

26
mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
#include "dataset/engine/datasetops/rename_op.h"
#include <iomanip>
#include <vector>
#include <utility>
#include <unordered_map>
@ -138,11 +138,25 @@ Status RenameOp::RenameBuffer(std::unique_ptr<DataBuffer> *input_buffer) {
// prints rename
void RenameOp::Print(std::ostream &out, // In: The output stream to print to
bool show_all) const { // In: T/F if it should print everything
// Call base class printer first
PipelineOp::Print(out, show_all);
out << "\nRenameOp:\n";
for (size_t i = 0; i < in_columns_.size(); ++i) {
out << "\nin Columns: " << in_columns_[i] << "\nOut Columns: " << out_columns_[i] << "\n\n";
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <RenameOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nIn columns:";
for (size_t i = 0; i < in_columns_.size(); ++i) {
out << "\n " << in_columns_[i];
}
for (size_t i = 0; i < out_columns_.size(); ++i) {
out << "\n " << out_columns_[i];
}
out << "\n\n";
}
}

35
mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc
View File

@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <iomanip>
#include <iostream>
#include <utility>
@ -51,22 +52,28 @@ RepeatOp::~RepeatOp() {}
// A print method typically used for debugging
void RepeatOp::Print(std::ostream &out, bool show_all) const {
// Call base class printer first
PipelineOp::Print(out, show_all);
// Then display our own stuff
out << "RepeatOp:"
<< "\nCurrent repeat count: " << repeat_count_ << "\nMax repeat count: " << max_repeats_
<< "\nLeaf Nodes in my execution path:";
if (!eoe_ops_.empty()) {
out << "\n";
for (size_t i = 0; i < eoe_ops_.size(); i++) {
out << " Operator: " << eoe_ops_[i]->id() << "\n";
}
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <RepeatOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << " [repeats: " << max_repeats_ << "]\n";
} else {
out << " kNone.";
// Call the super class for displaying any common detailed info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nCurrent repeat count: " << repeat_count_ << "\nMax repeat count: " << max_repeats_
<< "\nLeaf Nodes in execution path:";
if (!eoe_ops_.empty()) {
for (size_t i = 0; i < eoe_ops_.size(); i++) {
out << "\n Operator: " << eoe_ops_[i]->id();
}
} else {
out << " None.";
}
out << "\n\n";
}
out << "\n-------------------------\n\n"; // End the display with this line
}
// Base-class override for executing specific RepeatOp configurations. This code will be called

22
mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc
View File

@ -19,6 +19,7 @@
#include <securec.h>
#include <algorithm>
#include <chrono>
#include <iomanip>
#include <iostream>
#include <limits>
#include <random>
@ -108,13 +109,20 @@ Status ShuffleOp::SelfReset() {
// A print method typically used for debugging
void ShuffleOp::Print(std::ostream &out, bool show_all) const {
// Call base class printer first
PipelineOp::Print(out, show_all);
// Then display our own stuff
out << "ShuffleOp:\n Shuffle size: " << shuffle_size_ << "\n rows_per_buffer_: " << rows_per_buffer_
<< "\n shuffle_buffer_state_: " << shuffle_buffer_state_ << "\n shuffle_seed_: " << shuffle_seed_;
out << "\n-------------------------\n\n"; // End the display with this line
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <ShuffleOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << " [shuffle size: " << shuffle_size_ << "]\n";
} else {
// Call the super class for displaying any common detailed info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nShuffle size: " << shuffle_size_ << "\nRows per buffer: " << rows_per_buffer_
<< "\nShuffle buffer state: " << shuffle_buffer_state_ << "\nShuffle seed: " << shuffle_seed_ << "\n\n";
}
}
// Private function to add a new row to the shuffle buffer.

20
mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc
View File

@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <iomanip>
#include <iostream>
#include <utility>
@ -56,12 +57,19 @@ SkipOp::~SkipOp() {}
// A print method typically used for debugging
void SkipOp::Print(std::ostream &out, bool show_all) const {
// Call base class printer first
PipelineOp::Print(out, show_all);
// Then display our own stuff
out << "SkipOp:"
<< "\nCurrent skip count: " << skip_count_ << "\nMax skip count: " << max_skips_;
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <SkipOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << " [skips: " << max_skips_ << "]\n";
} else {
// Call the super class for displaying any common detailed info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nSkip count: " << skip_count_ << "\nMax skips: " << max_skips_ << "\n\n";
}
}
// Base-class override for handling cases when an eoe is received.

17
mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc
View File

@ -16,6 +16,7 @@
#include "dataset/engine/datasetops/source/celeba_op.h"
#include <fstream>
#include <iomanip>
#include "dataset/core/config_manager.h"
#include "dataset/util/path.h"
#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h"
@ -434,9 +435,19 @@ Status CelebAOp::LoadTensorRow(const std::pair<std::string, std::vector<int32_t>
}
void CelebAOp::Print(std::ostream &out, bool show_all) const {
DatasetOp::Print(out, show_all);
out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_exact_
<< "\nceleba dir: " << folder_path_ << "\n-------------------------\n";
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <CelebAOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nNumber of rows:" << num_rows_exact_ << "\nceleba dir: " << folder_path_ << "\n\n";
}
}
// Reset Sampler and wakeup Master thread (functor)

17
mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc
View File

@ -17,6 +17,7 @@
#include <algorithm>
#include <fstream>
#include <iomanip>
#include <utility>
#include "common/utils.h"
@ -225,9 +226,19 @@ Status CifarOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<Dat
}
void CifarOp::Print(std::ostream &out, bool show_all) const {
DatasetOp::Print(out, show_all);
out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_
<< "\nCifar Directory: " << folder_path_ << "\n-------------------------\n";
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <CifarOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nNumber of rows:" << num_rows_ << "\nCifar directory: " << folder_path_ << "\n\n";
}
}
// Reset Sampler and wakeup Master thread (functor)

23
mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc
View File

@ -13,8 +13,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "dataset/core/global_context.h"
#include "dataset/engine/datasetops/source/generator_op.h"
#include <iomanip>
#include "dataset/core/global_context.h"
#include "dataset/engine/db_connector.h"
#include "dataset/engine/data_buffer.h"
#include "dataset/engine/execution_tree.h"
@ -58,6 +59,26 @@ GeneratorOp::GeneratorOp(py::function generator_function, std::vector<std::strin
GeneratorOp::~GeneratorOp() { this->Dealloc(); }
void GeneratorOp::Print(std::ostream &out, bool show_all) const {
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <GeneratorOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nColumn names:\n";
for (int i = 0; i < column_names_.size(); ++i) {
out << "\n " << column_names_[i];
}
out << "\n\n";
}
}
void GeneratorOp::Dealloc() noexcept {
// Setup GIL state
PyGILState_STATE gstate;

5
mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h
View File

@ -95,6 +95,11 @@ class GeneratorOp : public PipelineOp {
~GeneratorOp();
// A print method typically used for debugging
// @param out - The output stream to write output to
// @param show_all - A bool to control if you want to show all info or just a summary
void Print(std::ostream &out, bool show_all) const override;
// << Stream output operator overload
// @notes This allows you to write the debug print info using stream operators
// @param out - reference to the output stream being overloaded

19
mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc
View File

@ -14,9 +14,8 @@
* limitations under the License.
*/
#include "dataset/engine/datasetops/source/image_folder_op.h"
#include <fstream>
#include <iomanip>
#include "common/utils.h"
#include "dataset/core/config_manager.h"
#include "dataset/core/tensor_shape.h"
@ -243,9 +242,19 @@ Status ImageFolderOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_p
}
void ImageFolderOp::Print(std::ostream &out, bool show_all) const {
DatasetOp::Print(out, show_all);
out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_
<< "\nImageFolder Directory: " << folder_path_ << "\n-------------------------\n";
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <ImageFolderOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nNumber of rows:" << num_rows_ << "\nImageFolder directory: " << folder_path_ << "\n\n";
}
}
// Reset Sampler and wakeup Master thread (functor)

17
mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc
View File

@ -17,6 +17,7 @@
#include <algorithm>
#include <fstream>
#include <iomanip>
#include <nlohmann/json.hpp>
#include "common/utils.h"
@ -239,9 +240,19 @@ Status ManifestOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<
}
void ManifestOp::Print(std::ostream &out, bool show_all) const {
DatasetOp::Print(out, show_all);
out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_
<< "\nManifest file: " << file_ << "\n-------------------------\n";
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <ManifestOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nNumber of rows:" << num_rows_ << "\nManifest file: " << file_ << "\n\n";
}
}
// Reset Sampler and wakeup Master thread (functor)

28
mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc
View File

@ -17,6 +17,7 @@
#include <algorithm>
#include <cstdint>
#include <iomanip>
#include <limits>
#include <utility>
@ -179,18 +180,21 @@ MindRecordOp::~MindRecordOp() {}
// A print method typically used for debugging
void MindRecordOp::Print(std::ostream &out, bool show_all) const {
// Call base class printer first
ParallelOp::Print(out, show_all);
// Then display our own stuff
out << "\nMindRecordOp:";
out << "\n 1 Dataset file : " << dataset_file_;
out << "\n Number of rows : " << num_rows_;
out << "\n Rows per buffer : " << rows_per_buffer_;
out << "\n Number of buffers : " << buffers_needed_;
out << "\n Number of ShardReader workers : " << num_mind_record_workers_;
out << "\n\n";
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <MindRecordOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\n1 Dataset file : " << dataset_file_ << "\nNumber of rows : " << num_rows_
<< "\nRows per buffer : " << rows_per_buffer_ << "\nNumber of buffers : " << buffers_needed_
<< "\nNumber of ShardReader workers : " << num_mind_record_workers_ << "\n\n";
}
}
template <typename T>

18
mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc
View File

@ -16,7 +16,7 @@
#include "dataset/engine/datasetops/source/mnist_op.h"
#include <fstream>
#include <iomanip>
#include "common/utils.h"
#include "dataset/core/config_manager.h"
#include "dataset/core/tensor_shape.h"
@ -190,9 +190,19 @@ Status MnistOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<Dat
}
void MnistOp::Print(std::ostream &out, bool show_all) const {
DatasetOp::Print(out, show_all);
out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_
<< "\nMNIST Directory: " << folder_path_ << "\n-------------------------\n";
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <MnistOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nNumber of rows:" << num_rows_ << "\nMNIST Directory: " << folder_path_ << "\n\n";
}
}
// Reset Sampler and wakeup Master thread (functor)

36
mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.cc
View File

@ -23,6 +23,7 @@
#include <chrono>
#include <cstdint>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <memory>
#include <mutex>
@ -319,31 +320,18 @@ StorageOp::~StorageOp() {}
// A print method typically used for debugging
void StorageOp::Print(std::ostream &out, bool show_all) const {
// Call base class printer first
ParallelOp::Print(out, show_all);
// Then display our own stuff
out << "\nStorageOp:";
out << "\n Dataset files dir : " << dataset_files_dir_ << "\n Dataset schema file : " << schema_file_;
if (!dataset_file_list_.empty()) {
out << "\n Dataset Files List:\n";
for (auto filename : dataset_file_list_) {
out << " " << filename << "\n";
}
}
out << "\n\n";
if (!data_buffers_.empty()) {
out << std::boolalpha << " Number of DataBuffers inside StorageOp: " << data_buffers_.size()
<< "\n Number of rows: " << num_rows_ << "\n Rows per buffer: " << rows_per_buffer_ << "\n\n DataBuffers:\n";
// Iterate over each DataBuffer and display the buffer id and the buffer
int32_t i = 0;
for (i = 0; i < data_buffers_.size(); i++) {
out << " " << i << ")\n";
data_buffers_[i]->Print(out, show_all);
}
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <StorageOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
out << "DataCache is empty!\n";
// Call the super class for displaying any common detailed info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nDetailed operator printing has not been implemented for this op.\n\n";
}
}

25
mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc
View File

@ -16,6 +16,7 @@
#include <algorithm>
#include <fstream>
#include <iomanip>
#include <memory>
#include <string>
#include <utility>
@ -90,6 +91,30 @@ TextFileOp::TextFileOp(int32_t num_workers, int64_t rows_per_buffer, int64_t num
worker_connector_size_ = worker_connector_size;
}
// A print method typically used for debugging
void TextFileOp::Print(std::ostream &out, bool show_all) const {
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <TextFileOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nRows per buffer: " << rows_per_buffer_ << "\nSample count: " << num_samples_
<< "\nDevice id: " << device_id_ << "\nNumber of devices: " << num_devices_
<< "\nShuffle files: " << ((shuffle_files_) ? "yes" : "no") << "\nText files list:\n";
for (int i = 0; i < text_files_list_.size(); ++i) {
out << " " << text_files_list_[i];
}
out << "\nData Schema:\n";
out << *data_schema_ << "\n\n";
}
}
Status TextFileOp::Init() {
RETURN_IF_NOT_OK(filename_index_->insert(text_files_list_));

5
mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h
View File

@ -144,6 +144,11 @@ class TextFileOp : public ParallelOp {
// Default destructor
~TextFileOp() = default;
// A print method typically used for debugging
// @param out - The output stream to write output to
// @param show_all - A bool to control if you want to show all info or just a summary
void Print(std::ostream &out, bool show_all) const override;
// Instantiates the internal queues and connectors
// @return Status - the error code returned
Status Init();

31
mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc
View File

@ -18,6 +18,7 @@
#include <cmath>
#include <condition_variable>
#include <future>
#include <iomanip>
#include <memory>
#include <mutex>
#include <utility>
@ -155,6 +156,36 @@ TFReaderOp::TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64
worker_connector_size_ = worker_connector_size;
}
// A print method typically used for debugging
void TFReaderOp::Print(std::ostream &out, bool show_all) const {
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <TFReaderOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nRows per buffer: " << rows_per_buffer_ << "\nTotal rows: " << total_rows_ << "\nDevice id: " << device_id_
<< "\nNumber of devices: " << num_devices_ << "\nShuffle files: " << ((shuffle_files_) ? "yes" : "no")
<< "\nDataset files list:\n";
for (int i = 0; i < dataset_files_list_.size(); ++i) {
out << " " << dataset_files_list_[i];
}
if (!columns_to_load_.empty()) {
out << "\nColumns to load:\n";
for (int i = 0; i < columns_to_load_.size(); ++i) {
out << " " << columns_to_load_[i];
}
}
out << "\nData Schema:\n";
out << *data_schema_ << "\n\n";
}
}
Status TFReaderOp::Init() {
if (data_schema_->Empty()) {
RETURN_IF_NOT_OK(CreateSchema(dataset_files_list_[0], columns_to_load_));

5
mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h
View File

@ -188,6 +188,11 @@ class TFReaderOp : public ParallelOp {
// Default destructor
~TFReaderOp() = default;
// A print method typically used for debugging
// @param out - The output stream to write output to
// @param show_all - A bool to control if you want to show all info or just a summary
void Print(std::ostream &out, bool show_all) const override;
// Instantiates the internal queues and connectors.
// @return Status - the error code returned.
Status Init();

18
mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc
View File

@ -16,7 +16,7 @@
#include "dataset/engine/datasetops/source/voc_op.h"
#include <fstream>
#include <iomanip>
#include "common/utils.h"
#include "dataset/core/config_manager.h"
#include "dataset/core/tensor_shape.h"
@ -133,9 +133,19 @@ Status VOCOp::operator()() {
}
void VOCOp::Print(std::ostream &out, bool show_all) const {
DatasetOp::Print(out, show_all);
out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_
<< "\nVOC Directory: " << folder_path_ << "\n-------------------\n";
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <VOCOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nNumber of rows: " << num_rows_ << "\nVOC Directory: " << folder_path_ << "\n\n";
}
}
Status VOCOp::Reset() {

21
mindspore/ccsrc/dataset/engine/datasetops/take_op.cc
View File

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <iomanip>
#include <utility>
#include "common/utils.h"
@ -47,12 +47,19 @@ TakeOp::TakeOp(int32_t count) : PipelineOp(0), max_takes_(count), take_count_(0)
// A print method typically used for debugging
void TakeOp::Print(std::ostream &out, bool show_all) const {
// Call base class printer first
PipelineOp::Print(out, show_all);
// Then display our own stuff
out << "TakeOp:"
<< "\nCurrent take count: " << take_count_ << "\nMax take count: " << max_takes_;
// Always show the id and name as first line regardless if this summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <TakeOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << " [takes: " << max_takes_ << "]\n";
} else {
// Call the super class for displaying any common detailed info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nTake count: " << take_count_ << "\nMax takes: " << max_takes_ << "\n\n";
}
}
// This function will be call muti times to returns the buffer, when meet required max take count or meet

18
mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc
View File

@ -15,6 +15,7 @@
*/
#include "dataset/engine/datasetops/zip_op.h"
#include <utility>
#include <iomanip>
#include "dataset/core/constants.h"
#include "dataset/engine/data_buffer.h"
#include "dataset/engine/db_connector.h"
@ -224,10 +225,19 @@ Status ZipOp::drainPipeline() {
// A function that prints info about the Operator
void ZipOp::Print(std::ostream &out, // In: The output stream to print to
bool show_all) const { // In: T/F if it should print everything
// Call base class printer first
PipelineOp::Print(out, show_all);
out << "\nZipOp:\n"
<< "\nDatasets: " << children_num_ << "\n\n";
// Always show the id and name as first line regardless if this is summary or detailed print
out << "(" << std::setw(2) << operator_id_ << ") <ZipOp>:";
if (!show_all) {
// Call the super class for displaying any common 1-liner info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op
out << "\n";
} else {
// Call the super class for displaying any common detailed info
PipelineOp::Print(out, show_all);
// Then show any custom derived-internal stuff
out << "\nDatasets: " << children_num_ << "\n\n";
}
}
// overwrite function and handle eof

33
mindspore/ccsrc/dataset/engine/execution_tree.cc
View File

@ -81,13 +81,29 @@ Status ExecutionTree::AssignRoot(const std::shared_ptr<DatasetOp> &op) {
}
// A print method typically used for debugging
void ExecutionTree::Print(std::ostream &out, bool show_all) const {
out << "Total number of nodes in the ExecutionTree (may or may not be connected nodes): " << id_count_
<< "\nTree state: " << static_cast<int>(tree_state_) << "\n";
if (root_ != nullptr) {
// Just call the printer on the root node. Each node descends to it's children to print them if
// showAll is true.
root_->Print(out, show_all);
void ExecutionTree::Print(std::ostream &out) const {
out << "Execution tree summary:\n"
<< "-----------------------\n";
this->PrintNode(out, root_, "", true, false);
out << "\nExecution tree operator details:\n"
<< "--------------------------------\n";
this->PrintNode(out, root_, "", true, true);
}
// A helper functions for doing the recursive printing
void ExecutionTree::PrintNode(std::ostream &out, const std::shared_ptr<DatasetOp> &dataset_op, std::string indent,
bool last, bool detailed) const {
// Decide which printer to use based on detailed arg.
if (!detailed) {
out << indent << "+- " << *dataset_op;
indent += (last ? " " : "| ");
} else {
dataset_op->Print(out, detailed);
}
// Descend to children
for (int32_t i = 0; i < dataset_op->child_.size(); ++i) {
this->PrintNode(out, dataset_op->child_[i], indent, (i == (dataset_op->child_.size() - 1)), detailed);
}
}
@ -100,6 +116,9 @@ Status ExecutionTree::Launch() {
" Expected state: " + std::to_string(static_cast<int>(kDeTStateReady));
RETURN_STATUS_UNEXPECTED(err_msg);
}
std::ostringstream ss;
ss << *this;
MS_LOG(INFO) << "Printing the tree before launch tasks:\n" << ss.str();
for (auto itr = this->begin(); itr != this->end(); ++itr) {
// An inlined operator is one that has an output connector size of 0, and it does not
// require a thread to execute. Instead, the work of this operator is executed inlined

14
mindspore/ccsrc/dataset/engine/execution_tree.h
View File

@ -19,6 +19,7 @@
#include <functional>
#include <memory>
#include <stack>
#include <string>
#include <vector>
#include "dataset/engine/datasetops/dataset_op.h"
#include "dataset/util/status.h"
@ -114,8 +115,7 @@ class ExecutionTree {
// A print method typically used for debugging
// @param out - The output stream to write output to
// @param show_all - A bool to control if you want to show all info or just a summary
void Print(std::ostream &out, bool show_all) const;
void Print(std::ostream &out) const;
// Returns an iterator positioned at the start
// @return Iterator - The iterator
@ -133,7 +133,7 @@ class ExecutionTree {
// @param exe_tree - reference to the execution tree to display
// @return - the output stream must be returned
friend std::ostream &operator<<(std::ostream &out, ExecutionTree &exe_tree) {
exe_tree.Print(out, false);
exe_tree.Print(out);
return out;
}
@ -178,6 +178,14 @@ class ExecutionTree {
TaskGroup *AllTasks() const { return tg_.get(); }
private:
// A helper functions for doing the recursive printing
// @param dataset_op - The dataset op to print
// @param indent - an indent string for aligning child levels in output
// @param last - an indicator if it's the last child or not
// @param detailed - should it display the detailed node output or the summary line
void PrintNode(std::ostream &out, const std::shared_ptr<DatasetOp> &dataset_op, std::string indent, bool last,
bool detailed) const;
std::unique_ptr<TaskGroup> tg_; // Class for worker management
std::shared_ptr<DatasetOp> root_; // The root node of the tree
int32_t id_count_; // Counter for generating operator id's