!4402 [Dataset] Fix c_api bugs
This commit is contained in:
parent
d3ef9f77c5
commit
373adaa553
mindspore/ccsrc/minddata/dataset
tests/ut/cpp/dataset
|
@ -397,6 +397,10 @@ bool ValidateCommonDatasetParams(std::string dataset_dir) {
|
|||
MS_LOG(ERROR) << "No dataset path is specified";
|
||||
return false;
|
||||
}
|
||||
if (access(dataset_dir.c_str(), R_OK) == -1) {
|
||||
MS_LOG(ERROR) << "No access to specified dataset path: " << dataset_dir;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
@ -1176,8 +1180,8 @@ std::vector<std::shared_ptr<DatasetOp>> RepeatDataset::Build() {
|
|||
}
|
||||
|
||||
bool RepeatDataset::ValidateParams() {
|
||||
if (repeat_count_ != -1 && repeat_count_ <= 0) {
|
||||
MS_LOG(ERROR) << "Repeat: Repeat count cannot be" << repeat_count_;
|
||||
if (repeat_count_ <= 0 && repeat_count_ != -1) {
|
||||
MS_LOG(ERROR) << "Repeat: repeat_count should be either -1 or positive integer, repeat_count_: " << repeat_count_;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
@ -1223,7 +1227,7 @@ std::vector<std::shared_ptr<DatasetOp>> SkipDataset::Build() {
|
|||
// Function to validate the parameters for SkipDataset
|
||||
bool SkipDataset::ValidateParams() {
|
||||
if (skip_count_ <= -1) {
|
||||
MS_LOG(ERROR) << "Skip: Invalid input, skip_count: " << skip_count_;
|
||||
MS_LOG(ERROR) << "Skip: skip_count should not be negative, skip_count: " << skip_count_;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
@ -1244,8 +1248,8 @@ std::vector<std::shared_ptr<DatasetOp>> TakeDataset::Build() {
|
|||
|
||||
// Function to validate the parameters for TakeDataset
|
||||
bool TakeDataset::ValidateParams() {
|
||||
if (take_count_ < -1) {
|
||||
MS_LOG(ERROR) << "Take: Invalid input, take_count: " << take_count_;
|
||||
if (take_count_ < 0 && take_count_ != -1) {
|
||||
MS_LOG(ERROR) << "Take: take_count should be either -1 or positive integer, take_count: " << take_count_;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
|
@ -25,7 +25,7 @@ namespace api {
|
|||
void Iterator::GetNextRow(TensorMap *row) {
|
||||
Status rc = iterator_->GetNextAsMap(row);
|
||||
if (rc.IsError()) {
|
||||
MS_LOG(ERROR) << "GetNextRow: Failed to get next row.";
|
||||
MS_LOG(ERROR) << "GetNextRow: Failed to get next row. Error status: " << rc;
|
||||
row->clear();
|
||||
}
|
||||
}
|
||||
|
@ -35,7 +35,7 @@ void Iterator::GetNextRow(TensorVec *row) {
|
|||
TensorRow tensor_row;
|
||||
Status rc = iterator_->FetchNextTensorRow(&tensor_row);
|
||||
if (rc.IsError()) {
|
||||
MS_LOG(ERROR) << "GetNextRow: Failed to get next row.";
|
||||
MS_LOG(ERROR) << "GetNextRow: Failed to get next row. Error status: " << rc;
|
||||
row->clear();
|
||||
}
|
||||
// Generate a vector as return
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASETS_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASETS_H_
|
||||
|
||||
#include <unistd.h>
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
|
|
|
@ -437,22 +437,22 @@ TEST_F(MindDataTestPipeline, TestRepeatDefault) {
|
|||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr <Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
EXPECT_NE(ds,nullptr);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
// Default value of repeat count is -1, expected to repeat infinitely
|
||||
ds = ds->Repeat();
|
||||
EXPECT_NE(ds,nullptr);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
int32_t batch_size = 1;
|
||||
ds = ds->Batch(batch_size);
|
||||
EXPECT_NE(ds,nullptr);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr <Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter,nullptr);
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map <std::string, std::shared_ptr<Tensor>> row;
|
||||
|
@ -460,14 +460,16 @@ TEST_F(MindDataTestPipeline, TestRepeatDefault) {
|
|||
uint64_t i = 0;
|
||||
while (row.size()!= 0) {
|
||||
// manually stop
|
||||
if(i==100){break;}
|
||||
if (i == 100) {
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
auto image = row["image"];
|
||||
MS_LOG(INFO)<< "Tensor image shape: " << image->shape();
|
||||
iter->GetNextRow(&row);
|
||||
}
|
||||
|
||||
EXPECT_EQ(i,100);
|
||||
EXPECT_EQ(i, 100);
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
@ -478,22 +480,22 @@ TEST_F(MindDataTestPipeline, TestRepeatOne) {
|
|||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr <Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
EXPECT_NE(ds,nullptr);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
int32_t repeat_num = 1;
|
||||
ds = ds->Repeat(repeat_num);
|
||||
EXPECT_NE(ds,nullptr);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
int32_t batch_size = 1;
|
||||
ds = ds->Batch(batch_size);
|
||||
EXPECT_NE(ds,nullptr);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr <Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter,nullptr);
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map <std::string, std::shared_ptr<Tensor>> row;
|
||||
|
@ -506,13 +508,27 @@ TEST_F(MindDataTestPipeline, TestRepeatOne) {
|
|||
iter->GetNextRow(&row);
|
||||
}
|
||||
|
||||
EXPECT_EQ(i,10);
|
||||
EXPECT_EQ(i, 10);
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestRepeatFail) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRepeatFail.";
|
||||
TEST_F(MindDataTestPipeline, TestRepeatFail1) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRepeatFail1.";
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
int32_t repeat_num = 0;
|
||||
ds = ds->Repeat(repeat_num);
|
||||
EXPECT_EQ(ds, nullptr);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestRepeatFail2) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRepeatFail2.";
|
||||
// This case is expected to fail because the repeat count is invalid (<-1 && !=0).
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
|
|
|
@ -133,3 +133,25 @@ TEST_F(MindDataTestPipeline, TestMnistFail1) {
|
|||
std::shared_ptr<Dataset> ds = Mnist("", RandomSampler(false, 10));
|
||||
EXPECT_EQ(ds, nullptr);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestImageFolderFail2) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestImageFolderFail2.";
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 2), {".JGP"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
EXPECT_EQ(row.size(), 0);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue