p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add some of dataset ut cases about createIterator columns

This commit is contained in:
liucunwei 2021-12-02 19:20:55 +08:00
parent d2f3d6a793
commit ac27fa2cff
15 changed files with 670 additions and 437 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
tests/ut/cpp/dataset/c_api_dataset_album_test.cc
View File

@ -220,60 +220,60 @@ TEST_F(MindDataTestPipeline, TestAlbumBasicWithPipeline) {
/// Description: test iterator of AlbumDataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestAlbumIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumIteratorOneColumn.";
// Create a Album Dataset
std::string folder_path = datasets_root_path_ + "/testAlbum/images";
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
std::vector<std::string> column_names = {"image", "label", "id"};
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names);
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumIteratorOneColumn.";
// Create a Album Dataset
std::string folder_path = datasets_root_path_ + "/testAlbum/images";
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
std::vector <std::string> column_names = {"image", "label", "id"};
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 7);
EXPECT_EQ(i, 7);
// Manually terminate the pipeline
iter->Stop();
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: AlbumIteratorWrongColumn.
/// Description: test iterator of AlbumDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestAlbumIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumIteratorWrongColumn.";
// Create a Album Dataset
std::string folder_path = datasets_root_path_ + "/testAlbum/images";
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
std::vector<std::string> column_names = {"image", "label", "id"};
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names);
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumIteratorWrongColumn.";
// Create a Album Dataset
std::string folder_path = datasets_root_path_ + "/testAlbum/images";
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
std::vector<std::string> column_names = {"image", "label", "id"};
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names);
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: AlbumDatasetGetters.

60
tests/ut/cpp/dataset/c_api_dataset_dbpedia_test.cc
View File

@ -119,6 +119,66 @@ TEST_F(MindDataTestPipeline, TestDBpediaDatasetUsageAll) {
iter->Stop();
}
/// Feature: TestDBpediaDatasetIteratorOneColumn.
/// Description: test iterator of DBpediaDataset with only the "class" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestDBpediaDatasetIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDBpediaDatasetIteratorOneColumn.";
// Create a DBpedia Dataset
std::string folder_path = datasets_root_path_ + "/testDBpedia/";
std::vector<std::string> column_names = {"class", "title", "content"};
std::shared_ptr<Dataset> ds = DBpedia(folder_path, "test", 0, ShuffleMode::kFalse);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "class" column and drop others
std::vector<std::string> columns = {"class"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_class = {1};
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "class shape:" << v.Shape();
EXPECT_EQ(expect_class, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 3);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: DBpediaDatasetIteratorWrongColumn.
/// Description: test iterator of DBpediaDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestDBpediaDatasetIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDBpediaDatasetIteratorWrongColumn.";
// Create a DBpedia Dataset
std::string folder_path = datasets_root_path_ + "/testDBpedia/";
std::vector<std::string> column_names = {"class", "title", "content"};
std::shared_ptr<Dataset> ds = DBpedia(folder_path, "test", 0, ShuffleMode::kFalse);
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: DBpedia.
/// Description: includes tests for shape, type, size.
/// Expectation: the data is processed successfully.

86
tests/ut/cpp/dataset/c_api_dataset_div2k_test.cc
View File

@ -128,62 +128,62 @@ TEST_F(MindDataTestPipeline, TestDIV2KBasicWithPipeline) {
/// Description: test iterator of DIV2KDataset with only the "hr_image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestDIV2KIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KIteratorOneColumn.";
// Create a DIV2K Dataset
std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
std::string usage = "train"; // train valid, all
std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild
int32_t scale = 2; // 2, 3, 4, 8
std::shared_ptr<Dataset> ds = DIV2K(dataset_path, usage, downgrade, scale);
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KIteratorOneColumn.";
// Create a DIV2K Dataset
std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
std::string usage = "train"; // train valid, all
std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild
int32_t scale = 2; // 2, 3, 4, 8
std::shared_ptr<Dataset> ds = DIV2K(dataset_path, usage, downgrade, scale);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"hr_image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"hr_image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 5);
EXPECT_EQ(i, 5);
// Manually terminate the pipeline
iter->Stop();
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: DIV2KIteratorWrongColumn.
/// Description: test iterator of DIV2KDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestDIV2KIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KIteratorWrongColumn.";
// Create a DIV2K Dataset
std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
std::string usage = "train"; // train valid, all
std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild
int32_t scale = 2; // 2, 3, 4, 8
std::shared_ptr<Dataset> ds = DIV2K(dataset_path, usage, downgrade, scale);
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KIteratorWrongColumn.";
// Create a DIV2K Dataset
std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
std::string usage = "train"; // train valid, all
std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild
int32_t scale = 2; // 2, 3, 4, 8
std::shared_ptr<Dataset> ds = DIV2K(dataset_path, usage, downgrade, scale);
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: DIV2KDatasetGetters.

78
tests/ut/cpp/dataset/c_api_dataset_emnist_test.cc
View File

@ -223,58 +223,58 @@ TEST_F(MindDataTestPipeline, TestEMnistTestDatasetWithPipeline) {
/// Description: test iterator of EMnistDataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestEMnistIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestEMnistIteratorOneColumn.";
// Create a EMnist Dataset
std::string folder_path = datasets_root_path_ + "/testEMnistDataset";
std::shared_ptr<Dataset> ds = EMnist(folder_path, "mnist", "train", std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestEMnistIteratorOneColumn.";
// Create a EMnist Dataset
std::string folder_path = datasets_root_path_ + "/testEMnistDataset";
std::shared_ptr<Dataset> ds = EMnist(folder_path, "mnist", "train", std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_image = {1, 28, 28, 1};
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_image = {1, 28, 28, 1};
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 5);
EXPECT_EQ(i, 5);
// Manually terminate the pipeline
iter->Stop();
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: EMnistIteratorWrongColumn.
/// Description: test iterator of EMnistDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestEMnistIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestEMnistIteratorWrongColumn.";
// Create a EMnist Dataset
std::string folder_path = datasets_root_path_ + "/testEMnistDataset";
std::shared_ptr<Dataset> ds = EMnist(folder_path, "mnist", "train", std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestEMnistIteratorWrongColumn.";
// Create a EMnist Dataset
std::string folder_path = datasets_root_path_ + "/testEMnistDataset";
std::shared_ptr<Dataset> ds = EMnist(folder_path, "mnist", "train", std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestGetEMnistTrainDatasetSize) {

76
tests/ut/cpp/dataset/c_api_dataset_fake_image_test.cc
View File

@ -122,56 +122,56 @@ TEST_F(MindDataTestPipeline, TestFakeImageDatasetWithPipeline) {
/// Description: test iterator of FakeImageDataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestFakeImageIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFakeImageIteratorOneColumn.";
// Create a FakeImage Dataset
std::shared_ptr<Dataset> ds = FakeImage(50, {28, 28, 3}, 3, 0, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFakeImageIteratorOneColumn.";
// Create a FakeImage Dataset
std::shared_ptr<Dataset> ds = FakeImage(50, {28, 28, 3}, 3, 0, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 2;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 2;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_image = {2, 28, 28, 3};
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_image = {2, 28, 28, 3};
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 5);
EXPECT_EQ(i, 5);
// Manually terminate the pipeline
iter->Stop();
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: FakeImageIteratorWrongColumn.
/// Description: test iterator of FakeImageDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestFakeImageIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFakeImageIteratorWrongColumn.";
// Create a FakeImage Dataset
std::shared_ptr<Dataset> ds = FakeImage(50, {28, 28, 3}, 3, 0, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFakeImageIteratorWrongColumn.";
// Create a FakeImage Dataset
std::shared_ptr<Dataset> ds = FakeImage(50, {28, 28, 3}, 3, 0, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: GetFakeImageDatasetSize

78
tests/ut/cpp/dataset/c_api_dataset_flickr_test.cc
View File

@ -120,60 +120,60 @@ TEST_F(MindDataTestPipeline, TestFlickrBasicWithPipeline) {
/// Description: test iterator of FlickrDataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestFlickrIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrIteratorOneColumn.";
std::string dataset_path = datasets_root_path_ + "/testFlickrData/flickr30k/flickr30k-images";
std::string file_path = datasets_root_path_ + "/testFlickrData/flickr30k/test1.token";
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrIteratorOneColumn.";
std::string dataset_path = datasets_root_path_ + "/testFlickrData/flickr30k/flickr30k-images";
std::string file_path = datasets_root_path_ + "/testFlickrData/flickr30k/test1.token";
// Create a Flickr30k Dataset
std::shared_ptr<Dataset> ds = Flickr(dataset_path, file_path);
EXPECT_NE(ds, nullptr);
// Create a Flickr30k Dataset
std::shared_ptr<Dataset> ds = Flickr(dataset_path, file_path);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 2);
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: FlickrIteratorWrongColumn.
/// Description: test iterator of FlickrDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestFlickrIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrIteratorWrongColumn.";
std::string dataset_path = datasets_root_path_ + "/testFlickrData/flickr30k/flickr30k-images";
std::string file_path = datasets_root_path_ + "/testFlickrData/flickr30k/test1.token";
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrIteratorWrongColumn.";
std::string dataset_path = datasets_root_path_ + "/testFlickrData/flickr30k/flickr30k-images";
std::string file_path = datasets_root_path_ + "/testFlickrData/flickr30k/test1.token";
// Create a Flickr30k Dataset
std::shared_ptr<Dataset> ds = Flickr(dataset_path, file_path);
EXPECT_NE(ds, nullptr);
// Create a Flickr30k Dataset
std::shared_ptr<Dataset> ds = Flickr(dataset_path, file_path);
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestFlickrGetters) {

56
tests/ut/cpp/dataset/c_api_dataset_lj_speech_test.cc
View File

@ -123,6 +123,62 @@ TEST_F(MindDataTestPipeline, TestLJSpeechDatasetWithPipeline) {
iter->Stop();
}
/// Feature: TestLJSpeechDatasetIteratorOneColumn.
/// Description: test iterator of LJSpeechDataset with only the "waveform" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestLJSpeechDatasetIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestLJSpeechDatasetIteratorOneColumn.";
// Create a LJSpeech dataset
std::string folder_path = datasets_root_path_ + "/testLJSpeechData/";
std::shared_ptr<Dataset> ds = LJSpeech(folder_path, std::make_shared<RandomSampler>(false, 3));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "waveform" column and drop others
std::vector<std::string> columns = {"waveform"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "waveform shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 3);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: TestLJSpeechDatasetIteratorWrongColumn.
/// Description: test iterator of LJSpeechDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestLJSpeechDatasetIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestLJSpeechDatasetIteratorWrongColumn.";
// Create a LJSpeech Dataset
std::string folder_path = datasets_root_path_ + "/testLJSpeechData/";
std::shared_ptr<Dataset> ds = LJSpeech(folder_path, std::make_shared<RandomSampler>(false, 3));
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: LJSpeechDataset
/// Description: test getting size of LJSpeechDataset
/// Expectation: the size is correct

74
tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc
View File

@ -116,56 +116,56 @@ TEST_F(MindDataTestPipeline, TestManifestBasicWithPipeline) {
/// Description: test iterator of ManifestDataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestManifestIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestIteratorOneColumn.";
std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
// Create a Manifest Dataset
std::shared_ptr<Dataset> ds = Manifest(file_path);
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestIteratorOneColumn.";
std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
// Create a Manifest Dataset
std::shared_ptr<Dataset> ds = Manifest(file_path);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 2);
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: ManifestIteratorWrongColumn.
/// Description: test iterator of ManifestDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestManifestIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestIteratorWrongColumn.";
std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
// Create a Manifest Dataset
std::shared_ptr<Dataset> ds = Manifest(file_path);
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestIteratorWrongColumn.";
std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
// Create a Manifest Dataset
std::shared_ptr<Dataset> ds = Manifest(file_path);
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: ManifestGetters.

80
tests/ut/cpp/dataset/c_api_dataset_photo_tour_test.cc
View File

@ -160,58 +160,58 @@ TEST_F(MindDataTestPipeline, TestPhotoTourTrainDatasetWithPipeline) {
/// Description: test iterator of PhotoTourDataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestPhotoTourIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestPhotoTourIteratorOneColumn.";
// Create a PhotoTour Dataset
std::string folder_path = datasets_root_path_ + "/testPhotoTourData";
std::shared_ptr<Dataset> ds = PhotoTour(folder_path, "liberty", "train", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestPhotoTourIteratorOneColumn.";
// Create a PhotoTour Dataset
std::string folder_path = datasets_root_path_ + "/testPhotoTourData";
std::shared_ptr<Dataset> ds = PhotoTour(folder_path, "liberty", "train", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 2;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 2;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_image = {2, 64, 64, 1};
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_image = {2, 64, 64, 1};
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 5);
EXPECT_EQ(i, 5);
// Manually terminate the pipeline
iter->Stop();
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: PhotoTourIteratorWrongColumn.
/// Description: test iterator of PhotoTourDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestPhotoTourIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestPhotoTourIteratorWrongColumn.";
// Create a PhotoTour Dataset
std::string folder_path = datasets_root_path_ + "/testPhotoTourData";
std::shared_ptr<Dataset> ds = PhotoTour(folder_path, "liberty", "train", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestPhotoTourIteratorWrongColumn.";
// Create a PhotoTour Dataset
std::string folder_path = datasets_root_path_ + "/testPhotoTourData";
std::shared_ptr<Dataset> ds = PhotoTour(folder_path, "liberty", "train", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: PhotoTourTrainDatasetSize.

84
tests/ut/cpp/dataset/c_api_dataset_places365_test.cc
View File

@ -203,60 +203,60 @@ TEST_F(MindDataTestPipeline, TestPlaces365TrainDatasetWithPipeline) {
/// Description: test iterator of Places365Dataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestPlaces365IteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestPlaces365IteratorOneColumn.";
// Create a Places365 Dataset
std::string folder_path = datasets_root_path_ + "/testPlaces365Data";
std::shared_ptr<Dataset> ds =
Places365(folder_path, "train-standard", true, true, std::make_shared<RandomSampler>(false, 4));
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestPlaces365IteratorOneColumn.";
// Create a Places365 Dataset
std::string folder_path = datasets_root_path_ + "/testPlaces365Data";
std::shared_ptr<Dataset> ds =
Places365(folder_path, "train-standard", true, true, std::make_shared<RandomSampler>(false, 4));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 2;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 2;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_image = {2, 256, 256, 3};
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_image = {2, 256, 256, 3};
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 2);
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: Places365IteratorWrongColumn.
/// Description: test iterator of Places365Dataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestPlaces365IteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestPlaces365IteratorWrongColumn.";
// Create a Places365 Dataset
std::string folder_path = datasets_root_path_ + "/testPlaces365Data";
std::shared_ptr<Dataset> ds =
Places365(folder_path, "train-standard", true, true, std::make_shared<RandomSampler>(false, 4));
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestPlaces365IteratorWrongColumn.";
// Create a Places365 Dataset
std::string folder_path = datasets_root_path_ + "/testPlaces365Data";
std::shared_ptr<Dataset> ds =
Places365(folder_path, "train-standard", true, true, std::make_shared<RandomSampler>(false, 4));
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: Places365TrainDatasetSize.

80
tests/ut/cpp/dataset/c_api_dataset_qmnist_test.cc
View File

@ -274,58 +274,58 @@ TEST_F(MindDataTestPipeline, TestQMnistDatasetWithPipeline) {
/// Description: test iterator of QMnistDataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestQMnistIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestQMnistIteratorOneColumn.";
// Create a QMnist Dataset
std::string folder_path = datasets_root_path_ + "/testQMnistData/";
std::shared_ptr<Dataset> ds = QMnist(folder_path, "train", true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestQMnistIteratorOneColumn.";
// Create a QMnist Dataset
std::string folder_path = datasets_root_path_ + "/testQMnistData/";
std::shared_ptr<Dataset> ds = QMnist(folder_path, "train", true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_image = {1, 28, 28, 1};
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_image = {1, 28, 28, 1};
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 5);
EXPECT_EQ(i, 5);
// Manually terminate the pipeline
iter->Stop();
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: QMnistIteratorWrongColumn.
/// Description: test iterator of QMnistDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestQMnistIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestQMnistIteratorWrongColumn.";
// Create a QMnist Dataset
std::string folder_path = datasets_root_path_ + "/testQMnistData/";
std::shared_ptr<Dataset> ds = QMnist(folder_path, "train", true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestQMnistIteratorWrongColumn.";
// Create a QMnist Dataset
std::string folder_path = datasets_root_path_ + "/testQMnistData/";
std::shared_ptr<Dataset> ds = QMnist(folder_path, "train", true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: GetQMnistDatasetSize.

74
tests/ut/cpp/dataset/c_api_dataset_sbu_test.cc
View File

@ -125,56 +125,56 @@ TEST_F(MindDataTestPipeline, TestSBUDatasetWithPipeline) {
/// Description: test iterator of SBUDataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestSBUIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUIteratorOneColumn.";
// Create a SBU Dataset
std::string folder_path = datasets_root_path_ + "/testSBUDataset/";
std::shared_ptr<Dataset> ds = SBU(folder_path, true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUIteratorOneColumn.";
// Create a SBU Dataset
std::string folder_path = datasets_root_path_ + "/testSBUDataset/";
std::shared_ptr<Dataset> ds = SBU(folder_path, true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 5);
EXPECT_EQ(i, 5);
// Manually terminate the pipeline
iter->Stop();
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: SBUIteratorWrongColumn.
/// Description: test iterator of SBUtDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestSBUIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUIteratorWrongColumn.";
// Create a SBU Dataset
std::string folder_path = datasets_root_path_ + "/testSBUDataset/";
std::shared_ptr<Dataset> ds = SBU(folder_path, true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUIteratorWrongColumn.";
// Create a SBU Dataset
std::string folder_path = datasets_root_path_ + "/testSBUDataset/";
std::shared_ptr<Dataset> ds = SBU(folder_path, true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: SBUDatasetSize.

59
tests/ut/cpp/dataset/c_api_dataset_speech_commands_test.cc
View File

@ -125,6 +125,65 @@ TEST_F(MindDataTestPipeline, TestSpeechCommandsDatasetWithPipeline) {
iter->Stop();
}
/// Feature: TestSpeechCommandsDatasetIteratorOneColumn.
/// Description: test iterator of SpeechCommands dataset with only the "waveform" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestSpeechCommandsDatasetIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpeechCommandsDatasetIteratorOneColumn.";
// Create a SpeechCommands dataset
std::string folder_path = datasets_root_path_ + "/testSpeechCommandsData/";
std::shared_ptr<Dataset> ds = SpeechCommands(folder_path, "all", std::make_shared<RandomSampler>(false, 2));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "waveform" column and drop others
std::vector<std::string> columns = {"waveform"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_shape = {1, 1, 16000};
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "waveform shape:" << v.Shape();
EXPECT_EQ(expect_shape, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: TestSpeechCommandsDatasetIteratorWrongColumn.
/// Description: test iterator of SpeechCommandsDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestSpeechCommandsDatasetIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpeechCommandsDatasetIteratorWrongColumn.";
// Create a SpeechCommands dataset
std::string folder_path = datasets_root_path_ + "/testSpeechCommandsData/";
std::shared_ptr<Dataset> ds = SpeechCommands(folder_path, "all", std::make_shared<RandomSampler>(false, 2));
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: Test SpeechCommands dataset.
/// Description: get the size of SpeechCommands dataset.
/// Expectation: the data is processed successfully.

80
tests/ut/cpp/dataset/c_api_dataset_usps_test.cc
View File

@ -198,58 +198,58 @@ TEST_F(MindDataTestPipeline, TestUSPSDatasetWithPipeline) {
/// Description: test iterator of USPSDataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestUSPSIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSIteratorOneColumn.";
// Create a USPS Dataset
std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
std::shared_ptr<Dataset> ds = USPS(folder_path, "train");
EXPECT_NE(ds, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSIteratorOneColumn.";
// Create a USPS Dataset
std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
std::shared_ptr<Dataset> ds = USPS(folder_path, "train");
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_image = {1, 16, 16, 1};
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_image = {1, 16, 16, 1};
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 3);
EXPECT_EQ(i, 3);
// Manually terminate the pipeline
iter->Stop();
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: USPSIteratorWrongColumn.
/// Description: test iterator of USPSDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestUSPSIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSIteratorWrongColumn.";
// Create a USPS Dataset
std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
std::shared_ptr<Dataset> ds = USPS(folder_path, "train");
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSIteratorWrongColumn.";
// Create a USPS Dataset
std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
std::shared_ptr<Dataset> ds = USPS(folder_path, "train");
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: GetUSPSDatasetSize.

58
tests/ut/cpp/dataset/c_api_dataset_yes_no_test.cc
View File

@ -116,6 +116,64 @@ TEST_F(MindDataTestPipeline, YesNoDatasetWithPipeline) {
iter->Stop();
}
/// Feature: TestYesNoDatasetIteratorOneColumn.
/// Description: test iterator of YesNo dataset with only the "waveform" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestYesNoDatasetIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestYesNoDatasetIteratorOneColumn.";
// Create a YesNo dataset
std::string folder_path = datasets_root_path_ + "/testYesNoData/";
std::shared_ptr<Dataset> ds = YesNo(folder_path, std::make_shared<RandomSampler>(false, 2));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "waveform" column and drop others
std::vector<std::string> columns = {"waveform"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_image = {1, 1, 16000};
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "waveform shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: TestYesNoGetDatasetIteratorWrongColumn.
/// Description: test iterator of YesNoGetDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestYesNoGetDatasetIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestYesNoGetDatasetIteratorWrongColumn.";
// Create a YesNo dataset
std::string folder_path = datasets_root_path_ + "/testYesNoData/";
std::shared_ptr<Dataset> ds = YesNo(folder_path, std::make_shared<RandomSampler>(false, 2));
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: Test YesNo dataset.
/// Description: get the size of YesNo dataset.
/// Expectation: the data is processed successfully.