diff --git a/mindspore/ccsrc/minddata/dataset/api/datasets.cc b/mindspore/ccsrc/minddata/dataset/api/datasets.cc index 12bdcd5d85b..0d5a945af34 100644 --- a/mindspore/ccsrc/minddata/dataset/api/datasets.cc +++ b/mindspore/ccsrc/minddata/dataset/api/datasets.cc @@ -30,6 +30,7 @@ #include "minddata/dataset/engine/datasetops/skip_op.h" #include "minddata/dataset/engine/datasetops/project_op.h" #include "minddata/dataset/engine/datasetops/zip_op.h" +#include "minddata/dataset/engine/datasetops/rename_op.h" #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h" #include "minddata/dataset/engine/datasetops/source/sampler/random_sampler.h" @@ -40,12 +41,13 @@ namespace mindspore { namespace dataset { namespace api { -#define RETURN_NULL_IF_ERROR(_s) \ - do { \ - Status __rc = (_s); \ - if (__rc.IsError()) { \ - return nullptr; \ - } \ +#define RETURN_EMPTY_IF_ERROR(_s) \ + do { \ + Status __rc = (_s); \ + if (__rc.IsError()) { \ + MS_LOG(ERROR) << __rc; \ + return {}; \ + } \ } while (false) // Function to create the iterator, which will build and launch the execution tree. @@ -55,8 +57,7 @@ std::shared_ptr Dataset::CreateIterator() { iter = std::make_shared(); Status rc = iter->BuildAndLaunchTree(shared_from_this()); if (rc.IsError()) { - MS_LOG(ERROR) << rc; - MS_LOG(ERROR) << "CreateIterator failed."; + MS_LOG(ERROR) << "CreateIterator failed." << rc; return nullptr; } @@ -201,6 +202,20 @@ std::shared_ptr Dataset::Project(const std::vector return ds; } +// Function to create a RenameDataset. +std::shared_ptr Dataset::Rename(const std::vector &input_columns, + const std::vector &output_columns) { + auto ds = std::make_shared(input_columns, output_columns); + // Call derived class validation method. + if (!ds->ValidateParams()) { + return nullptr; + } + + ds->children.push_back(shared_from_this()); + + return ds; +} + // Function to create a Zip dataset std::shared_ptr Dataset::Zip(const std::vector> &datasets) { // Default values @@ -244,7 +259,7 @@ bool ImageFolderDataset::ValidateParams() { return true; } -std::shared_ptr>> ImageFolderDataset::Build() { +std::vector> ImageFolderDataset::Build() { // A vector containing shared pointer to the Dataset Ops that this object will create std::vector> node_ops; @@ -257,14 +272,14 @@ std::shared_ptr>> ImageFolderDataset::Bui // This arg is exist in ImageFolderOp, but not externalized (in Python API). std::unique_ptr schema = std::make_unique(); TensorShape scalar = TensorShape::CreateScalar(); - RETURN_NULL_IF_ERROR( + RETURN_EMPTY_IF_ERROR( schema->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); - RETURN_NULL_IF_ERROR( + RETURN_EMPTY_IF_ERROR( schema->AddColumn(ColDescriptor("label", DataType(DataType::DE_INT32), TensorImpl::kFlexible, 0, &scalar))); node_ops.push_back(std::make_shared(num_workers_, rows_per_buffer_, dataset_dir_, connector_que_size_, recursive_, decode_, exts_, class_indexing_, std::move(schema), std::move(sampler_->Build()))); - return std::make_shared>>(node_ops); + return node_ops; } MnistDataset::MnistDataset(std::string dataset_dir, std::shared_ptr sampler) @@ -279,7 +294,7 @@ bool MnistDataset::ValidateParams() { return true; } -std::shared_ptr>> MnistDataset::Build() { +std::vector> MnistDataset::Build() { // A vector containing shared pointer to the Dataset Ops that this object will create std::vector> node_ops; @@ -290,14 +305,14 @@ std::shared_ptr>> MnistDataset::Build() { // Do internal Schema generation. auto schema = std::make_unique(); - RETURN_NULL_IF_ERROR(schema->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kCv, 1))); + RETURN_EMPTY_IF_ERROR(schema->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kCv, 1))); TensorShape scalar = TensorShape::CreateScalar(); - RETURN_NULL_IF_ERROR( + RETURN_EMPTY_IF_ERROR( schema->AddColumn(ColDescriptor("label", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 0, &scalar))); node_ops.push_back(std::make_shared(num_workers_, rows_per_buffer_, dataset_dir_, connector_que_size_, std::move(schema), std::move(sampler_->Build()))); - return std::make_shared>>(node_ops); + return node_ops; } BatchDataset::BatchDataset(int32_t batch_size, bool drop_remainder, bool pad, std::vector cols_to_map, @@ -308,7 +323,7 @@ BatchDataset::BatchDataset(int32_t batch_size, bool drop_remainder, bool pad, st cols_to_map_(cols_to_map), pad_map_(pad_map) {} -std::shared_ptr>> BatchDataset::Build() { +std::vector> BatchDataset::Build() { // A vector containing shared pointer to the Dataset Ops that this object will create std::vector> node_ops; @@ -320,11 +335,12 @@ std::shared_ptr>> BatchDataset::Build() { node_ops.push_back(std::make_shared(batch_size_, drop_remainder_, pad_, connector_que_size_, num_workers_, cols_to_map_, pad_map_)); #endif - return std::make_shared>>(node_ops); + return node_ops; } bool BatchDataset::ValidateParams() { if (batch_size_ <= 0) { + MS_LOG(ERROR) << "Batch: Batch size cannot be negative"; return false; } @@ -333,16 +349,17 @@ bool BatchDataset::ValidateParams() { RepeatDataset::RepeatDataset(uint32_t count) : repeat_count_(count) {} -std::shared_ptr>> RepeatDataset::Build() { +std::vector> RepeatDataset::Build() { // A vector containing shared pointer to the Dataset Ops that this object will create std::vector> node_ops; node_ops.push_back(std::make_shared(repeat_count_)); - return std::make_shared>>(node_ops); + return node_ops; } bool RepeatDataset::ValidateParams() { if (repeat_count_ <= 0) { + MS_LOG(ERROR) << "Repeat: Repeat count cannot be negative"; return false; } @@ -355,7 +372,7 @@ MapDataset::MapDataset(std::vector> operations, output_columns_(output_columns), project_columns_(project_columns) {} -std::shared_ptr>> MapDataset::Build() { +std::vector> MapDataset::Build() { // A vector containing shared pointer to the Dataset Ops that this object will create std::vector> node_ops; @@ -380,11 +397,12 @@ std::shared_ptr>> MapDataset::Build() { } node_ops.push_back(map_op); - return std::make_shared>>(node_ops); + return node_ops; } bool MapDataset::ValidateParams() { if (operations_.empty()) { + MS_LOG(ERROR) << "Map: No operation is specified."; return false; } @@ -396,13 +414,13 @@ ShuffleDataset::ShuffleDataset(int32_t shuffle_size, bool reset_every_epoch) : shuffle_size_(shuffle_size), shuffle_seed_(GetSeed()), reset_every_epoch_(reset_every_epoch) {} // Function to build the ShuffleOp -std::shared_ptr>> ShuffleDataset::Build() { +std::vector> ShuffleDataset::Build() { // A vector containing shared pointer to the Dataset Ops that this object will create std::vector> node_ops; node_ops.push_back(std::make_shared(shuffle_size_, shuffle_seed_, connector_que_size_, reset_every_epoch_, rows_per_buffer_)); - return std::make_shared>>(node_ops); + return node_ops; } // Function to validate the parameters for ShuffleDataset @@ -419,12 +437,12 @@ bool ShuffleDataset::ValidateParams() { SkipDataset::SkipDataset(int32_t count) : skip_count_(count) {} // Function to build the SkipOp -std::shared_ptr>> SkipDataset::Build() { +std::vector> SkipDataset::Build() { // A vector containing shared pointer to the Dataset Ops that this object will create std::vector> node_ops; node_ops.push_back(std::make_shared(skip_count_, connector_que_size_)); - return std::make_shared>>(node_ops); + return node_ops; } // Function to validate the parameters for SkipDataset @@ -454,7 +472,7 @@ bool Cifar10Dataset::ValidateParams() { } // Function to build CifarOp -std::shared_ptr>> Cifar10Dataset::Build() { +std::vector> Cifar10Dataset::Build() { // A vector containing shared pointer to the Dataset Ops that this object will create std::vector> node_ops; @@ -465,15 +483,15 @@ std::shared_ptr>> Cifar10Dataset::Build() // Do internal Schema generation. auto schema = std::make_unique(); - RETURN_NULL_IF_ERROR(schema->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kCv, 1))); + RETURN_EMPTY_IF_ERROR(schema->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kCv, 1))); TensorShape scalar = TensorShape::CreateScalar(); - RETURN_NULL_IF_ERROR( + RETURN_EMPTY_IF_ERROR( schema->AddColumn(ColDescriptor("label", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 0, &scalar))); node_ops.push_back(std::make_shared(CifarOp::CifarType::kCifar10, num_workers_, rows_per_buffer_, dataset_dir_, connector_que_size_, std::move(schema), std::move(sampler_->Build()))); - return std::make_shared>>(node_ops); + return node_ops; } // Function to build ProjectOp @@ -487,12 +505,12 @@ bool ProjectDataset::ValidateParams() { return true; } -std::shared_ptr>> ProjectDataset::Build() { +std::vector> ProjectDataset::Build() { // A vector containing shared pointer to the Dataset Ops that this object will create std::vector> node_ops; node_ops.push_back(std::make_shared(columns_)); - return std::make_shared>>(node_ops); + return node_ops; } // Function to build ZipOp @@ -500,12 +518,37 @@ ZipDataset::ZipDataset() {} bool ZipDataset::ValidateParams() { return true; } -std::shared_ptr>> ZipDataset::Build() { +std::vector> ZipDataset::Build() { // A vector containing shared pointer to the Dataset Ops that this object will create std::vector> node_ops; node_ops.push_back(std::make_shared(rows_per_buffer_, connector_que_size_)); - return std::make_shared>>(node_ops); + return node_ops; +} + +// Function to build RenameOp +RenameDataset::RenameDataset(const std::vector &input_columns, + const std::vector &output_columns) + : input_columns_(input_columns), output_columns_(output_columns) {} + +bool RenameDataset::ValidateParams() { + if (input_columns_.empty() || output_columns_.empty()) { + MS_LOG(ERROR) << "input and output columns must be specified"; + return false; + } + if (input_columns_.size() != output_columns_.size()) { + MS_LOG(ERROR) << "input and output columns must be the same size"; + return false; + } + return true; +} + +std::vector> RenameDataset::Build() { + // A vector containing shared pointer to the Dataset Ops that this object will create + std::vector> node_ops; + + node_ops.push_back(std::make_shared(input_columns_, output_columns_, connector_que_size_)); + return node_ops; } } // namespace api diff --git a/mindspore/ccsrc/minddata/dataset/api/iterator.cc b/mindspore/ccsrc/minddata/dataset/api/iterator.cc index 8136601ed84..e60ddc76436 100644 --- a/mindspore/ccsrc/minddata/dataset/api/iterator.cc +++ b/mindspore/ccsrc/minddata/dataset/api/iterator.cc @@ -56,7 +56,13 @@ Status Iterator::BuildAndLaunchTree(std::shared_ptr ds) { RETURN_STATUS_UNEXPECTED("Input is null pointer"); } else { // Convert the current root node. - auto root_op = ds->Build()->front(); + auto root_ops = ds->Build(); + if (root_ops.empty()) { + RETURN_STATUS_UNEXPECTED("Node operation returned nothing"); + } + + auto root_op = root_ops.front(); + RETURN_UNEXPECTED_IF_NULL(root_op); RETURN_IF_NOT_OK(tree_->AssociateNode(root_op)); @@ -70,20 +76,22 @@ Status Iterator::BuildAndLaunchTree(std::shared_ptr ds) { // Iterate through all the direct children of the first element in our BFS queue for (auto child : node_pair.first->children) { auto child_ops = child->Build(); - RETURN_UNEXPECTED_IF_NULL(child_ops); + if (child_ops.empty()) { + RETURN_STATUS_UNEXPECTED("Node operation returned nothing"); + } auto node_op = node_pair.second; // Iterate through all the DatasetOps returned by calling Build on the last Dataset object, associate them // with the execution tree and add the child and parent relationship between the nodes // Note that some Dataset objects might return more than one DatasetOps // e.g. MapDataset will return MapOp and ProjectOp if project_columns is set for MapDataset - for (auto child_op : *child_ops) { + for (auto child_op : child_ops) { RETURN_IF_NOT_OK(tree_->AssociateNode(child_op)); RETURN_IF_NOT_OK(node_op->AddChild(child_op)); node_op = child_op; } // Add the child and the last element of the returned DatasetOps (which is now the leaf node in our current // execution tree) to the BFS queue - q.push(std::make_pair(child, child_ops->back())); + q.push(std::make_pair(child, child_ops.back())); } } RETURN_IF_NOT_OK(tree_->AssignRoot(root_op)); diff --git a/mindspore/ccsrc/minddata/dataset/include/datasets.h b/mindspore/ccsrc/minddata/dataset/include/datasets.h index 9cf9787841b..fb7bc44dad0 100644 --- a/mindspore/ccsrc/minddata/dataset/include/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/include/datasets.h @@ -50,6 +50,7 @@ class SkipDataset; class Cifar10Dataset; class ProjectDataset; class ZipDataset; +class RenameDataset; /// \brief Function to create an ImageFolderDataset /// \notes A source dataset that reads images from a tree of directories @@ -98,8 +99,8 @@ class Dataset : public std::enable_shared_from_this { ~Dataset() = default; /// \brief Pure virtual function to convert a Dataset class into a runtime dataset object - /// \return shared pointer to the list of newly created DatasetOps - virtual std::shared_ptr>> Build() = 0; + /// \return The list of shared pointers to the newly created DatasetOps + virtual std::vector> Build() = 0; /// \brief Pure virtual function for derived class to implement parameters validation /// \return bool True if all the params are valid @@ -179,6 +180,14 @@ class Dataset : public std::enable_shared_from_this { /// \return Shared pointer to the current Dataset std::shared_ptr Zip(const std::vector> &datasets); + /// \brief Function to create a Rename Dataset + /// \notes Renames the columns in the input dataset + /// \param[in] input_columns List of the input columns to rename + /// \param[in] output_columns List of the output columns + /// \return Shared pointer to the current Dataset + std::shared_ptr Rename(const std::vector &input_columns, + const std::vector &output_columns); + protected: std::vector> children; std::shared_ptr parent; @@ -202,8 +211,8 @@ class ImageFolderDataset : public Dataset { ~ImageFolderDataset() = default; /// \brief a base class override function to create the required runtime dataset op objects for this class - /// \return shared pointer to the list of newly created DatasetOps - std::shared_ptr>> Build() override; + /// \return The list of shared pointers to the newly created DatasetOps + std::vector> Build() override; /// \brief Parameters validation /// \return bool true if all the params are valid @@ -227,8 +236,8 @@ class MnistDataset : public Dataset { ~MnistDataset() = default; /// \brief a base class override function to create the required runtime dataset op objects for this class - /// \return shared pointer to the list of newly created DatasetOps - std::shared_ptr>> Build() override; + /// \return The list of shared pointers to the newly created DatasetOps + std::vector> Build() override; /// \brief Parameters validation /// \return bool true if all the params are valid @@ -249,8 +258,8 @@ class BatchDataset : public Dataset { ~BatchDataset() = default; /// \brief a base class override function to create the required runtime dataset op objects for this class - /// \return shared pointer to the list of newly created DatasetOps - std::shared_ptr>> Build() override; + /// \return The list of shared pointers to the newly created DatasetOps + std::vector> Build() override; /// \brief Parameters validation /// \return bool true if all the params are valid @@ -273,8 +282,8 @@ class RepeatDataset : public Dataset { ~RepeatDataset() = default; /// \brief a base class override function to create the required runtime dataset op objects for this class - /// \return shared pointer to the list of newly created DatasetOps - std::shared_ptr>> Build() override; + /// \return The list of shared pointers to the newly created DatasetOps + std::vector> Build() override; /// \brief Parameters validation /// \return bool true if all the params are valid @@ -290,7 +299,7 @@ class ShuffleDataset : public Dataset { ~ShuffleDataset() = default; - std::shared_ptr>> Build() override; + std::vector> Build() override; bool ValidateParams() override; @@ -309,8 +318,8 @@ class SkipDataset : public Dataset { ~SkipDataset() = default; /// \brief a base class override function to create the required runtime dataset op objects for this class - /// \return shared pointer to the list of newly created DatasetOps - std::shared_ptr>> Build() override; + /// \return The list of shared pointers to the newly created DatasetOps + std::vector> Build() override; /// \brief Parameters validation /// \return bool true if all the params are valid @@ -330,8 +339,8 @@ class MapDataset : public Dataset { ~MapDataset() = default; /// \brief a base class override function to create the required runtime dataset op objects for this class - /// \return shared pointer to the list of newly created DatasetOps - std::shared_ptr>> Build() override; + /// \return The list of shared pointers to the newly created DatasetOps + std::vector> Build() override; /// \brief Parameters validation /// \return bool true if all the params are valid @@ -353,8 +362,8 @@ class Cifar10Dataset : public Dataset { ~Cifar10Dataset() = default; /// \brief a base class override function to create the required runtime dataset op objects for this class - /// \return shared pointer to the list of newly created DatasetOps - std::shared_ptr>> Build() override; + /// \return The list of shared pointers to the newly created DatasetOps + std::vector> Build() override; /// \brief Parameters validation /// \return bool true if all the params are valid @@ -375,8 +384,8 @@ class ProjectDataset : public Dataset { ~ProjectDataset() = default; /// \brief a base class override function to create the required runtime dataset op objects for this class - /// \return shared pointer to the list of newly created DatasetOps - std::shared_ptr>> Build() override; + /// \return The list of shared pointers to the newly created DatasetOps + std::vector> Build() override; /// \brief Parameters validation /// \return bool true if all the params are valid @@ -395,14 +404,35 @@ class ZipDataset : public Dataset { ~ZipDataset() = default; /// \brief a base class override function to create the required runtime dataset op objects for this class - /// \return shared pointer to the list of newly created DatasetOps - std::shared_ptr>> Build() override; + /// \return The list of shared pointers to the newly created DatasetOps + std::vector> Build() override; /// \brief Parameters validation /// \return bool true if all the params are valid bool ValidateParams() override; }; +class RenameDataset : public Dataset { + public: + /// \brief Constructor + explicit RenameDataset(const std::vector &input_columns, const std::vector &output_columns); + + /// \brief Destructor + ~RenameDataset() = default; + + /// \brief a base class override function to create the required runtime dataset op objects for this class + /// \return The list of shared pointers to the newly created DatasetOps + std::vector> Build() override; + + /// \brief Parameters validation + /// \return bool true if all the params are valid + bool ValidateParams() override; + + private: + std::vector input_columns_; + std::vector output_columns_; +}; + } // namespace api } // namespace dataset } // namespace mindspore diff --git a/tests/ut/cpp/dataset/c_api_test.cc b/tests/ut/cpp/dataset/c_api_test.cc index 560958e2a81..5ff1862cfd6 100644 --- a/tests/ut/cpp/dataset/c_api_test.cc +++ b/tests/ut/cpp/dataset/c_api_test.cc @@ -49,22 +49,22 @@ TEST_F(MindDataTestPipeline, TestBatchAndRepeat) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::shared_ptr ds = Mnist(folder_path, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds int32_t repeat_num = 2; ds = ds->Repeat(repeat_num); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 2; ds = ds->Batch(batch_size); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -78,7 +78,7 @@ TEST_F(MindDataTestPipeline, TestBatchAndRepeat) { iter->GetNextRow(&row); } - EXPECT_TRUE(i == 10); + EXPECT_EQ(i, 10); // Manually terminate the pipeline iter->Stop(); @@ -88,33 +88,33 @@ TEST_F(MindDataTestPipeline, TestTensorOpsAndMap) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::shared_ptr ds = Mnist(folder_path, RandomSampler(false, 20)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds int32_t repeat_num = 2; ds = ds->Repeat(repeat_num); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create objects for the tensor ops std::shared_ptr resize_op = vision::Resize({30, 30}); - EXPECT_TRUE(resize_op != nullptr); + EXPECT_NE(resize_op, nullptr); std::shared_ptr center_crop_op = vision::CenterCrop({16, 16}); - EXPECT_TRUE(center_crop_op != nullptr); + EXPECT_NE(center_crop_op, nullptr); // Create a Map operation on ds ds = ds->Map({resize_op, center_crop_op}); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 1; ds = ds->Batch(batch_size); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -128,7 +128,7 @@ TEST_F(MindDataTestPipeline, TestTensorOpsAndMap) { iter->GetNextRow(&row); } - EXPECT_TRUE(i == 40); + EXPECT_EQ(i, 40); // Manually terminate the pipeline iter->Stop(); @@ -138,34 +138,34 @@ TEST_F(MindDataTestPipeline, TestUniformAugWithOps) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::shared_ptr ds = Mnist(folder_path, RandomSampler(false, 20)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds int32_t repeat_num = 1; ds = ds->Repeat(repeat_num); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create objects for the tensor ops std::shared_ptr resize_op = vision::Resize({30, 30}); - EXPECT_TRUE(resize_op != nullptr); + EXPECT_NE(resize_op, nullptr); std::shared_ptr random_crop_op = vision::RandomCrop({28, 28}); - EXPECT_TRUE(random_crop_op != nullptr); + EXPECT_NE(random_crop_op, nullptr); std::shared_ptr center_crop_op = vision::CenterCrop({16, 16}); - EXPECT_TRUE(center_crop_op != nullptr); + EXPECT_NE(center_crop_op, nullptr); std::shared_ptr uniform_aug_op = vision::UniformAugment({random_crop_op, center_crop_op}, 2); - EXPECT_TRUE(uniform_aug_op != nullptr); + EXPECT_NE(uniform_aug_op, nullptr); // Create a Map operation on ds ds = ds->Map({resize_op, uniform_aug_op}); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -179,7 +179,7 @@ TEST_F(MindDataTestPipeline, TestUniformAugWithOps) { iter->GetNextRow(&row); } - EXPECT_TRUE(i == 20); + EXPECT_EQ(i, 20); // Manually terminate the pipeline iter->Stop(); @@ -189,33 +189,33 @@ TEST_F(MindDataTestPipeline, TestRandomFlip) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds int32_t repeat_num = 2; ds = ds->Repeat(repeat_num); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create objects for the tensor ops std::shared_ptr random_vertical_flip_op = vision::RandomVerticalFlip(0.5); - EXPECT_TRUE(random_vertical_flip_op != nullptr); + EXPECT_NE(random_vertical_flip_op, nullptr); std::shared_ptr random_horizontal_flip_op = vision::RandomHorizontalFlip(0.5); - EXPECT_TRUE(random_horizontal_flip_op != nullptr); + EXPECT_NE(random_horizontal_flip_op, nullptr); // Create a Map operation on ds ds = ds->Map({random_vertical_flip_op, random_horizontal_flip_op}); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 1; ds = ds->Batch(batch_size); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -229,7 +229,7 @@ TEST_F(MindDataTestPipeline, TestRandomFlip) { iter->GetNextRow(&row); } - EXPECT_TRUE(i == 20); + EXPECT_EQ(i, 20); // Manually terminate the pipeline iter->Stop(); @@ -239,22 +239,22 @@ TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds int32_t repeat_num = 2; ds = ds->Repeat(repeat_num); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 2; ds = ds->Batch(batch_size); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -268,7 +268,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) { iter->GetNextRow(&row); } - EXPECT_TRUE(i == 10); + EXPECT_EQ(i, 10); // Manually terminate the pipeline iter->Stop(); @@ -327,7 +327,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderWithSamplers) { iter->GetNextRow(&row); } - EXPECT_TRUE(i == 12); + EXPECT_EQ(i, 12); // Manually terminate the pipeline iter->Stop(); @@ -337,36 +337,36 @@ TEST_F(MindDataTestPipeline, TestPad) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds int32_t repeat_num = 2; ds = ds->Repeat(repeat_num); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create objects for the tensor ops std::shared_ptr pad_op1 = vision::Pad({1, 2, 3, 4}, {0}, BorderType::kSymmetric); - EXPECT_TRUE(pad_op1 != nullptr); + EXPECT_NE(pad_op1, nullptr); std::shared_ptr pad_op2 = vision::Pad({1}, {1, 1, 1}, BorderType::kEdge); - EXPECT_TRUE(pad_op2 != nullptr); + EXPECT_NE(pad_op2, nullptr); std::shared_ptr pad_op3 = vision::Pad({1, 4}); - EXPECT_TRUE(pad_op3 != nullptr); + EXPECT_NE(pad_op3, nullptr); // Create a Map operation on ds ds = ds->Map({pad_op1, pad_op2, pad_op3}); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 1; ds = ds->Batch(batch_size); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -380,7 +380,7 @@ TEST_F(MindDataTestPipeline, TestPad) { iter->GetNextRow(&row); } - EXPECT_TRUE(i == 20); + EXPECT_EQ(i, 20); // Manually terminate the pipeline iter->Stop(); @@ -390,33 +390,33 @@ TEST_F(MindDataTestPipeline, TestCutOut) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds int32_t repeat_num = 2; ds = ds->Repeat(repeat_num); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create objects for the tensor ops std::shared_ptr cut_out1 = vision::CutOut(30, 5); - EXPECT_TRUE(cut_out1!= nullptr); + EXPECT_NE(cut_out1, nullptr); std::shared_ptr cut_out2 = vision::CutOut(30); - EXPECT_TRUE(cut_out2 != nullptr); + EXPECT_NE(cut_out2, nullptr); // Create a Map operation on ds ds = ds->Map({cut_out1, cut_out2}); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 1; ds = ds->Batch(batch_size); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -430,7 +430,7 @@ TEST_F(MindDataTestPipeline, TestCutOut) { iter->GetNextRow(&row); } - EXPECT_TRUE(i == 20); + EXPECT_EQ(i, 20); // Manually terminate the pipeline iter->Stop(); @@ -440,30 +440,30 @@ TEST_F(MindDataTestPipeline, TestNormalize) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds int32_t repeat_num = 2; ds = ds->Repeat(repeat_num); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create objects for the tensor ops std::shared_ptr normalize = vision::Normalize({121.0, 115.0, 100.0}, {70.0, 68.0, 71.0}); - EXPECT_TRUE(normalize != nullptr); + EXPECT_NE(normalize, nullptr); // Create a Map operation on ds ds = ds->Map({normalize}); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 1; ds = ds->Batch(batch_size); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -477,7 +477,7 @@ TEST_F(MindDataTestPipeline, TestNormalize) { iter->GetNextRow(&row); } - EXPECT_TRUE(i == 20); + EXPECT_EQ(i, 20); // Manually terminate the pipeline iter->Stop(); @@ -487,30 +487,30 @@ TEST_F(MindDataTestPipeline, TestDecode) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, false, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds int32_t repeat_num = 2; ds = ds->Repeat(repeat_num); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create objects for the tensor ops std::shared_ptr decode = vision::Decode(true); - EXPECT_TRUE(decode != nullptr); + EXPECT_NE(decode, nullptr); // Create a Map operation on ds ds = ds->Map({decode}); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 1; ds = ds->Batch(batch_size); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -533,27 +533,27 @@ TEST_F(MindDataTestPipeline, TestShuffleDataset) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Shuffle operation on ds int32_t shuffle_size = 10; ds = ds->Shuffle(shuffle_size); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds int32_t repeat_num = 2; ds = ds->Repeat(repeat_num); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 2; ds = ds->Batch(batch_size); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -567,7 +567,7 @@ TEST_F(MindDataTestPipeline, TestShuffleDataset) { iter->GetNextRow(&row); } - EXPECT_TRUE(i == 10); + EXPECT_EQ(i, 10); // Manually terminate the pipeline iter->Stop(); @@ -579,17 +579,17 @@ TEST_F(MindDataTestPipeline, TestSkipDataset) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Skip operation on ds int32_t count = 3; ds = ds->Skip(count); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -605,7 +605,7 @@ TEST_F(MindDataTestPipeline, TestSkipDataset) { MS_LOG(INFO) << "Number of rows: " << i; // Expect 10-3=7 rows - EXPECT_TRUE(i == 7); + EXPECT_EQ(i, 7); // Manually terminate the pipeline iter->Stop(); @@ -617,13 +617,13 @@ TEST_F(MindDataTestPipeline, TestSkipDatasetError1) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Skip operation on ds with invalid count input int32_t count = -1; ds = ds->Skip(count); // Expect nullptr for invalid input skip_count - EXPECT_TRUE(ds == nullptr); + EXPECT_EQ(ds, nullptr); } TEST_F(MindDataTestPipeline, TestCifar10Dataset) { @@ -631,22 +631,22 @@ TEST_F(MindDataTestPipeline, TestCifar10Dataset) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::shared_ptr ds = Cifar10(folder_path, 0, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds int32_t repeat_num = 2; ds = ds->Repeat(repeat_num); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 2; ds = ds->Batch(batch_size); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -660,7 +660,7 @@ TEST_F(MindDataTestPipeline, TestCifar10Dataset) { iter->GetNextRow(&row); } - EXPECT_TRUE(i == 10); + EXPECT_EQ(i, 10); // Manually terminate the pipeline iter->Stop(); @@ -670,41 +670,41 @@ TEST_F(MindDataTestPipeline, TestRandomColorAdjust) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds int32_t repeat_num = 2; ds = ds->Repeat(repeat_num); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create objects for the tensor ops std::shared_ptr random_color_adjust1 = vision::RandomColorAdjust({1.0}, {0.0}, {0.5}, {0.5}); - EXPECT_TRUE(random_color_adjust1 != nullptr); + EXPECT_NE(random_color_adjust1, nullptr); std::shared_ptr random_color_adjust2 = vision::RandomColorAdjust({1.0, 1.0}, {0.0, 0.0}, {0.5, 0.5}, {0.5, 0.5}); - EXPECT_TRUE(random_color_adjust2 != nullptr); + EXPECT_NE(random_color_adjust2, nullptr); std::shared_ptr random_color_adjust3 = vision::RandomColorAdjust({0.5, 1.0}, {0.0, 0.5}, {0.25, 0.5}, {0.25, 0.5}); - EXPECT_TRUE(random_color_adjust3 != nullptr); + EXPECT_NE(random_color_adjust3, nullptr); std::shared_ptr random_color_adjust4 = vision::RandomColorAdjust(); - EXPECT_TRUE(random_color_adjust4 != nullptr); + EXPECT_NE(random_color_adjust4, nullptr); // Create a Map operation on ds ds = ds->Map({random_color_adjust1, random_color_adjust2, random_color_adjust3, random_color_adjust4}); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 1; ds = ds->Batch(batch_size); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -718,7 +718,7 @@ TEST_F(MindDataTestPipeline, TestRandomColorAdjust) { iter->GetNextRow(&row); } - EXPECT_TRUE(i == 20); + EXPECT_EQ(i, 20); // Manually terminate the pipeline iter->Stop(); @@ -728,82 +728,30 @@ TEST_F(MindDataTestPipeline, TestRandomRotation) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds int32_t repeat_num = 2; ds = ds->Repeat(repeat_num); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create objects for the tensor ops std::shared_ptr random_rotation_op = vision::RandomRotation({-180, 180}); - EXPECT_TRUE(random_rotation_op != nullptr); + EXPECT_NE(random_rotation_op, nullptr); // Create a Map operation on ds ds = ds->Map({random_rotation_op}); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 1; ds = ds->Batch(batch_size); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); - - // Iterate the dataset and get each row - std::unordered_map> row; - iter->GetNextRow(&row); - - uint64_t i = 0; - while (row.size() != 0) { - i++; - auto image = row["image"]; - MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - iter->GetNextRow(&row); - } - - EXPECT_TRUE(i == 20); - - // Manually terminate the pipeline - iter->Stop(); -} - -TEST_F(MindDataTestPipeline, TestProjectMap) { - // Create an ImageFolder Dataset - std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); - - // Create a Repeat operation on ds - int32_t repeat_num = 2; - ds = ds->Repeat(repeat_num); - EXPECT_TRUE(ds != nullptr); - - // Create objects for the tensor ops - std::shared_ptr random_vertical_flip_op = vision::RandomVerticalFlip(0.5); - EXPECT_TRUE(random_vertical_flip_op != nullptr); - - // Create a Map operation on ds - ds = ds->Map({random_vertical_flip_op}, {}, {}, {"image", "label"}); - EXPECT_TRUE(ds != nullptr); - - // Create a Project operation on ds - std::vector column_project = {"image"}; - ds = ds->Project(column_project); - EXPECT_TRUE(ds != nullptr); - - // Create a Batch operation on ds - int32_t batch_size = 1; - ds = ds->Batch(batch_size); - EXPECT_TRUE(ds != nullptr); - - // Create an iterator over the result of the above dataset - // This will trigger the creation of the Execution Tree and launch it. - std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -823,39 +771,99 @@ TEST_F(MindDataTestPipeline, TestProjectMap) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestZip) { +TEST_F(MindDataTestPipeline, TestProjectMap) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); + + // Create a Repeat operation on ds + int32_t repeat_num = 2; + ds = ds->Repeat(repeat_num); + EXPECT_NE(ds, nullptr); + + // Create objects for the tensor ops + std::shared_ptr random_vertical_flip_op = vision::RandomVerticalFlip(0.5); + EXPECT_NE(random_vertical_flip_op, nullptr); + + // Create a Map operation on ds + ds = ds->Map({random_vertical_flip_op}, {}, {}, {"image", "label"}); + EXPECT_NE(ds, nullptr); // Create a Project operation on ds std::vector column_project = {"image"}; ds = ds->Project(column_project); - EXPECT_TRUE(ds != nullptr); - - folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds1 = Cifar10(folder_path, 0, RandomSampler(false, 10)); - EXPECT_TRUE(ds1 != nullptr); - - // Create a Project operation on ds - column_project = {"label"}; - ds1 = ds1->Project(column_project); - EXPECT_TRUE(ds1 != nullptr); - - // Create a Zip operation on the datasets - ds = ds->Zip({ds, ds1}); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 1; ds = ds->Batch(batch_size); - EXPECT_TRUE(ds != nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_TRUE(iter != nullptr); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + iter->GetNextRow(&row); + } + + EXPECT_EQ(i, 20); + + // Manually terminate the pipeline + iter->Stop(); +} + +TEST_F(MindDataTestPipeline, TestZipSuccess) { + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); + + // Create a Project operation on ds + std::vector column_project = {"image"}; + ds = ds->Project(column_project); + EXPECT_NE(ds, nullptr); + + // Create an ImageFolder Dataset + std::shared_ptr ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10)); + EXPECT_NE(ds1, nullptr); + + // Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names) + ds1 = ds1->Rename({"image", "label"}, {"col1", "col2"}); + EXPECT_NE(ds1, nullptr); + + folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds2 = Cifar10(folder_path, 0, RandomSampler(false, 10)); + EXPECT_NE(ds2, nullptr); + + // Create a Project operation on ds + column_project = {"label"}; + ds2 = ds2->Project(column_project); + EXPECT_NE(ds2, nullptr); + + // Create a Zip operation on the datasets + ds = ds->Zip({ds, ds1, ds2}); + EXPECT_NE(ds, nullptr); + + // Create a Batch operation on ds + int32_t batch_size = 1; + ds = ds->Batch(batch_size); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map> row; @@ -874,3 +882,97 @@ TEST_F(MindDataTestPipeline, TestZip) { // Manually terminate the pipeline iter->Stop(); } + +TEST_F(MindDataTestPipeline, TestZipFail) { + // We expect this test to fail because we are the both datasets we are zipping have "image" and "label" columns + // and zip doesn't accept datasets with same column names + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); + + // Create an ImageFolder Dataset + std::shared_ptr ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10)); + EXPECT_NE(ds1, nullptr); + + // Create a Zip operation on the datasets + ds = ds->Zip({ds, ds1}); + EXPECT_NE(ds, nullptr); + + // Create a Batch operation on ds + int32_t batch_size = 1; + ds = ds->Batch(batch_size); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_EQ(iter, nullptr); +} + +TEST_F(MindDataTestPipeline, TestRenameSuccess) { + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); + + // Create a Repeat operation on ds + int32_t repeat_num = 2; + ds = ds->Repeat(repeat_num); + EXPECT_NE(ds, nullptr); + + // Create a Rename operation on ds + ds = ds->Rename({"image", "label"}, {"col1", "col2"}); + EXPECT_NE(ds, nullptr); + + // Create a Batch operation on ds + int32_t batch_size = 1; + ds = ds->Batch(batch_size); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + uint64_t i = 0; + EXPECT_NE(row.find("col1"), row.end()); + EXPECT_NE(row.find("col2"), row.end()); + EXPECT_EQ(row.find("image"), row.end()); + EXPECT_EQ(row.find("label"), row.end()); + + while (row.size() != 0) { + i++; + auto image = row["col1"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + iter->GetNextRow(&row); + } + + EXPECT_EQ(i, 20); + + // Manually terminate the pipeline + iter->Stop(); +} + +TEST_F(MindDataTestPipeline, TestRenameFail) { + // We expect this test to fail because input and output in Rename are not the same size + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); + + // Create a Repeat operation on ds + int32_t repeat_num = 2; + ds = ds->Repeat(repeat_num); + EXPECT_NE(ds, nullptr); + + // Create a Rename operation on ds + ds = ds->Rename({"image", "label"}, {"col2"}); + EXPECT_EQ(ds, nullptr); +} \ No newline at end of file