Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!8248 Add macro and fix album testcase 

Merge pull request !8248 from EricZ/helper_dep_removal
This commit is contained in:
mindspore-ci-bot 2020-11-06 20:33:20 +08:00 committed by Gitee
parent d2d5ffedec 394dbf50e1
commit 97e1f6f438
5 changed files with 92 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc
View File

@ -108,7 +108,7 @@ Status DatasetOp::InsertAsParent(std::shared_ptr<DatasetOp> to_add) {
} }
RETURN_IF_NOT_OK(to_add->AddChild(shared_from_this())); RETURN_IF_NOT_OK(to_add->AddChild(shared_from_this()));
if (tree_->root()->id() == this->id()) { if (tree_->root()->id() == this->id()) {
tree_->AssignRoot(to_add); RETURN_IF_NOT_OK(tree_->AssignRoot(to_add));
} }
return Status::OK(); return Status::OK();
} }
@ -156,7 +156,7 @@ Status DatasetOp::Remove() {
// We don't have a parent, so we are the root node being removed. // We don't have a parent, so we are the root node being removed.
// clear the parent list of our child so that it becomes the new root. // clear the parent list of our child so that it becomes the new root.
child_[0]->parent_.clear(); child_[0]->parent_.clear();
tree_->AssignRoot(child_[0]); RETURN_IF_NOT_OK(tree_->AssignRoot(child_[0]));
} }
} }

2
mindspore/ccsrc/minddata/dataset/engine/opt/pre/epoch_injection_pass.cc
View File

@ -68,7 +68,7 @@ Status EpochInjectionPass::RunOnTree(ExecutionTree *tree, bool *modified) {
std::shared_ptr<EpochCtrlOp> epoch_ctrl_op; std::shared_ptr<EpochCtrlOp> epoch_ctrl_op;
RETURN_IF_NOT_OK(EpochCtrlOp::Builder(num_epochs).Build(&epoch_ctrl_op)); RETURN_IF_NOT_OK(EpochCtrlOp::Builder(num_epochs).Build(&epoch_ctrl_op));
RETURN_IF_NOT_OK(tree->AssociateNode(epoch_ctrl_op)); RETURN_IF_NOT_OK(tree->AssociateNode(epoch_ctrl_op));
epoch_inject_node->InsertAsParent(epoch_ctrl_op); RETURN_IF_NOT_OK(epoch_inject_node->InsertAsParent(epoch_ctrl_op));
} }
MS_LOG(INFO) << "Pre pass: Injection pass complete."; MS_LOG(INFO) << "Pre pass: Injection pass complete.";

82
tests/ut/cpp/dataset/album_op_test.cc
View File

@ -90,57 +90,45 @@ TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchema) {
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
std::vector<std::string> column_names = {"image", "label", "id"}; std::vector<std::string> column_names = {"image", "label", "id"};
auto tree = Build({AlbumSchema(16, 2, 32, folder_path, schema_file, column_names, false), Repeat(2)}); auto tree = Build({AlbumSchema(16, 2, 32, folder_path, schema_file, column_names, false), Repeat(2)});
tree->Prepare(); ASSERT_OK(tree->Prepare());
Status rc = tree->Launch(); ASSERT_OK(tree->Launch());
if (rc.IsError()) {
MS_LOG(ERROR) << "Return code error detected during tree launch: " << ".";
EXPECT_TRUE(false);
} else {
DatasetIterator di(tree); DatasetIterator di(tree);
TensorMap tensor_map; TensorMap tensor_map;
di.GetNextAsMap(&tensor_map); ASSERT_OK(di.GetNextAsMap(&tensor_map));
EXPECT_TRUE(rc.IsOk());
uint64_t i = 0; uint64_t i = 0;
int32_t label = 0; std::string_view label = 0;
while (tensor_map.size() != 0) { while (tensor_map.size() != 0) {
tensor_map["label"]->GetItemAt<int32_t>(&label, {}); EXPECT_TRUE(tensor_map["label"]->GetItemAt(&label, {}));
MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape"
<< tensor_map["label"] << "\n"; << tensor_map["label"] << "\n";
i++; i++;
di.GetNextAsMap(&tensor_map); di.GetNextAsMap(&tensor_map);
} }
MS_LOG(INFO) << "got rows" << i << "\n"; MS_LOG(INFO) << "got rows: " << i << "\n";
EXPECT_TRUE(i == 14); EXPECT_TRUE(i == 14);
} }
}
TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchemaNoOrder) { TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchemaNoOrder) {
std::string folder_path = datasets_root_path_ + "/testAlbum/images"; std::string folder_path = datasets_root_path_ + "/testAlbum/images";
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
auto tree = Build({AlbumSchema(16, 2, 32, folder_path, schema_file), Repeat(2)}); auto tree = Build({AlbumSchema(16, 2, 32, folder_path, schema_file), Repeat(2)});
tree->Prepare(); ASSERT_OK(tree->Prepare());
Status rc = tree->Launch(); ASSERT_OK(tree->Launch());
if (rc.IsError()) {
MS_LOG(ERROR) << "Return code error detected during tree launch: " << ".";
EXPECT_TRUE(false);
} else {
DatasetIterator di(tree); DatasetIterator di(tree);
TensorMap tensor_map; TensorMap tensor_map;
di.GetNextAsMap(&tensor_map); ASSERT_TRUE(di.GetNextAsMap(&tensor_map));
EXPECT_TRUE(rc.IsOk());
uint64_t i = 0; uint64_t i = 0;
int32_t label = 0; std::string_view label;
while (tensor_map.size() != 0) { while (tensor_map.size() != 0) {
tensor_map["label"]->GetItemAt<int32_t>(&label, {}); EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {}));
MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape"
<< tensor_map["label"] << "\n"; << tensor_map["label"] << "\n";
i++; i++;
di.GetNextAsMap(&tensor_map); di.GetNextAsMap(&tensor_map);
} }
MS_LOG(INFO) << "got rows" << i << "\n"; MS_LOG(INFO) << "got rows: " << i << "\n";
EXPECT_TRUE(i == 14); EXPECT_TRUE(i == 14);
} }
}
TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchemaFloat) { TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchemaFloat) {
std::string folder_path = datasets_root_path_ + "/testAlbum/images"; std::string folder_path = datasets_root_path_ + "/testAlbum/images";
@ -148,62 +136,50 @@ TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchemaFloat) {
std::string schema_file = datasets_root_path_ + "/testAlbum/floatSchema.json"; std::string schema_file = datasets_root_path_ + "/testAlbum/floatSchema.json";
auto tree = Build({AlbumSchema(16, 2, 32, folder_path, schema_file), Repeat(2)}); auto tree = Build({AlbumSchema(16, 2, 32, folder_path, schema_file), Repeat(2)});
tree->Prepare(); tree->Prepare();
Status rc = tree->Launch(); ASSERT_OK(tree->Launch());
if (rc.IsError()) {
MS_LOG(ERROR) << "Return code error detected during tree launch: " << ".";
EXPECT_TRUE(false);
} else {
DatasetIterator di(tree); DatasetIterator di(tree);
TensorMap tensor_map; TensorMap tensor_map;
di.GetNextAsMap(&tensor_map); ASSERT_OK(di.GetNextAsMap(&tensor_map));
EXPECT_TRUE(rc.IsOk());
uint64_t i = 0; uint64_t i = 0;
int32_t label = 0; std::string_view label;
double priority = 0; double priority = 0;
while (tensor_map.size() != 0) { while (tensor_map.size() != 0) {
tensor_map["label"]->GetItemAt<int32_t>(&label, {}); EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {}));
tensor_map["_priority"]->GetItemAt<double>(&priority, {}); EXPECT_OK(tensor_map["_priority"]->GetItemAt<double>(&priority, {}));
MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape"
<< tensor_map["label"] << "priority: " << priority << "\n"; << tensor_map["label"] << "priority: " << priority << "\n";
i++; i++;
di.GetNextAsMap(&tensor_map); di.GetNextAsMap(&tensor_map);
} }
MS_LOG(INFO) << "got rows" << i << "\n"; MS_LOG(INFO) << "got rows: " << i << "\n";
EXPECT_TRUE(i == 14); EXPECT_TRUE(i == 14);
} }
}
TEST_F(MindDataTestAlbum, TestSequentialAlbumWithFullSchema) { TEST_F(MindDataTestAlbum, TestSequentialAlbumWithFullSchema) {
std::string folder_path = datasets_root_path_ + "/testAlbum/images"; std::string folder_path = datasets_root_path_ + "/testAlbum/images";
// add the priority column // add the priority column
std::string schema_file = datasets_root_path_ + "/testAlbum/fullSchema.json"; std::string schema_file = datasets_root_path_ + "/testAlbum/fullSchema.json";
auto tree = Build({AlbumSchema(16, 2, 32, folder_path, schema_file), Repeat(2)}); auto tree = Build({AlbumSchema(16, 2, 32, folder_path, schema_file), Repeat(2)});
tree->Prepare(); ASSERT_OK(tree->Prepare());
Status rc = tree->Launch(); ASSERT_OK(tree->Launch());
if (rc.IsError()) {
MS_LOG(ERROR) << "Return code error detected during tree launch: " << ".";
EXPECT_TRUE(false);
} else {
DatasetIterator di(tree); DatasetIterator di(tree);
TensorMap tensor_map; TensorMap tensor_map;
di.GetNextAsMap(&tensor_map); ASSERT_OK(di.GetNextAsMap(&tensor_map));
EXPECT_TRUE(rc.IsOk());
uint64_t i = 0; uint64_t i = 0;
int32_t label = 0; std::string_view label = 0;
double priority = 0; double priority = 0;
int64_t id = 0; int64_t id = 0;
while (tensor_map.size() != 0) { while (tensor_map.size() != 0) {
tensor_map["label"]->GetItemAt<int32_t>(&label, {}); EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {}));
tensor_map["_priority"]->GetItemAt<double>(&priority, {}); EXPECT_OK(tensor_map["_priority"]->GetItemAt<double>(&priority, {}));
tensor_map["id"]->GetItemAt<int64_t>(&id, {}); EXPECT_OK(tensor_map["id"]->GetItemAt<int64_t>(&id, {}));
MS_LOG(ERROR) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape"
<< tensor_map["label"] << "priority: " << priority << " embedding : " << << tensor_map["label"] << "priority: " << priority << " embedding : "
tensor_map["_embedding"]->shape() << " id: " << id << "\n"; << tensor_map["_embedding"]->shape() << " id: " << id << "\n";
i++; i++;
di.GetNextAsMap(&tensor_map); di.GetNextAsMap(&tensor_map);
} }
MS_LOG(INFO) << "got rows" << i << "\n"; MS_LOG(INFO) << "got rows: " << i << "\n";
EXPECT_TRUE(i == 14); EXPECT_TRUE(i == 14);
} }
}

19
tests/ut/cpp/dataset/common/common.h
View File

@ -17,8 +17,27 @@
#define TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_ #define TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_
#include "gtest/gtest.h" #include "gtest/gtest.h"
#include "minddata/dataset/util/status.h"
#include "utils/log_adapter.h" #include "utils/log_adapter.h"
#define ASSERT_OK(_s) \
do { \
Status __rc = (_s); \
if (__rc.IsError()) { \
MS_LOG(ERROR) << __rc.ToString() << "."; \
ASSERT_TRUE(false); \
} \
} while (false)
#define EXPECT_OK(_s) \
do { \
Status __rc = (_s); \
if (__rc.IsError()) { \
MS_LOG(ERROR) << __rc.ToString() << "."; \
EXPECT_TRUE(false); \
} \
} while (false)
namespace UT { namespace UT {
class Common : public testing::Test { class Common : public testing::Test {
public: public:

8
tests/ut/cpp/dataset/data_helper_test.cc
View File

@ -153,6 +153,10 @@ TEST_F(MindDataTestDataHelper, MindDataTestTensorWriteFloat) {
// create buffer using system mempool // create buffer using system mempool
DataHelper dh; DataHelper dh;
void *data = malloc(t->SizeInBytes()); void *data = malloc(t->SizeInBytes());
if (data == nullptr) {
MS_LOG(ERROR) << "malloc failed";
ASSERT_TRUE(false);
}
auto bytes_copied = dh.DumpData(t->GetBuffer(), t->SizeInBytes(), data, t->SizeInBytes()); auto bytes_copied = dh.DumpData(t->GetBuffer(), t->SizeInBytes(), data, t->SizeInBytes());
if (bytes_copied != t->SizeInBytes()) { if (bytes_copied != t->SizeInBytes()) {
EXPECT_TRUE(false); EXPECT_TRUE(false);
@ -177,6 +181,10 @@ TEST_F(MindDataTestDataHelper, MindDataTestTensorWriteUInt) {
// create buffer using system mempool // create buffer using system mempool
DataHelper dh; DataHelper dh;
void *data = malloc(t->SizeInBytes()); void *data = malloc(t->SizeInBytes());
if (data == nullptr) {
MS_LOG(ERROR) << "malloc failed";
ASSERT_TRUE(false);
}
auto bytes_copied = dh.DumpData(t->GetBuffer(), t->SizeInBytes(), data, t->SizeInBytes()); auto bytes_copied = dh.DumpData(t->GetBuffer(), t->SizeInBytes(), data, t->SizeInBytes());
if (bytes_copied != t->SizeInBytes()) { if (bytes_copied != t->SizeInBytes()) {
EXPECT_TRUE(false); EXPECT_TRUE(false);