p60975123
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Support 3 composed data ops and cleanup code

This commit is contained in:
TinaMengtingZhang 2021-02-19 18:18:33 -05:00
parent c8657a6723
commit 518828b639
14 changed files with 387 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mindspore/ccsrc/minddata/dataset/api/datasets.cc
View File

@ -483,7 +483,6 @@ FilterDataset::FilterDataset(std::shared_ptr<Dataset> input, std::function<Tenso
}
#endif
// FIXME - Should be removed once all Tensor op API class has been added
MapDataset::MapDataset(std::shared_ptr<Dataset> input, std::vector<std::shared_ptr<TensorOperation>> operations,
const std::vector<std::string> &input_columns, const std::vector<std::string> &output_columns,
const std::vector<std::string> &project_columns, const std::shared_ptr<DatasetCache> &cache,

1
mindspore/ccsrc/minddata/dataset/api/execute.cc
View File

@ -21,6 +21,7 @@
#include "minddata/dataset/core/tensor_row.h"
#include "minddata/dataset/include/tensor.h"
#include "minddata/dataset/include/type_id.h"
#include "minddata/dataset/kernels/ir/tensor_operation.h"
#include "minddata/dataset/kernels/tensor_op.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"

89
mindspore/ccsrc/minddata/dataset/api/transforms.cc
View File

@ -16,22 +16,40 @@
#include "minddata/dataset/include/transforms.h"
#include <algorithm>
#include "minddata/dataset/kernels/ir/data/transforms_ir.h"
namespace mindspore {
namespace dataset {
// Transform operations for data.
namespace transforms {
// FUNCTIONS TO CREATE DATA TRANSFORM OPERATIONS
// API CLASS FOR DATA TRANSFORM OPERATIONS
// (In alphabetical order)
// Function to create ComposeOperation.
std::shared_ptr<ComposeOperation> Compose(const std::vector<std::shared_ptr<TensorOperation>> &transforms) {
auto op = std::make_shared<ComposeOperation>(transforms);
// Input validation
return op->ValidateParams() ? op : nullptr;
// Constructor to Compose.
Compose::Compose(const std::vector<TensorTransform *> &transforms) {
(void)std::transform(
transforms.begin(), transforms.end(), std::back_inserter(transforms_),
[](TensorTransform *op) -> std::shared_ptr<TensorOperation> { return op != nullptr ? op->Parse() : nullptr; });
}
Compose::Compose(const std::vector<std::shared_ptr<TensorTransform>> &transforms) {
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
[](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> {
return op != nullptr ? op->Parse() : nullptr;
});
}
Compose::Compose(const std::vector<std::reference_wrapper<TensorTransform>> &transforms) {
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
[](TensorTransform &op) -> std::shared_ptr<TensorOperation> { return op.Parse(); });
}
std::shared_ptr<TensorOperation> Compose::Parse() { return std::make_shared<ComposeOperation>(transforms_); }
// Constructor to Duplicate
Duplicate::Duplicate() {}
@ -42,31 +60,66 @@ OneHot::OneHot(int32_t num_classes) : num_classes_(num_classes) {}
std::shared_ptr<TensorOperation> OneHot::Parse() { return std::make_shared<OneHotOperation>(num_classes_); }
// Function to create RandomApplyOperation.
std::shared_ptr<RandomApplyOperation> RandomApply(const std::vector<std::shared_ptr<TensorOperation>> &transforms,
double prob) {
auto op = std::make_shared<RandomApplyOperation>(transforms, prob);
// Input validation
return op->ValidateParams() ? op : nullptr;
// Constructor to RandomApply.
RandomApply::RandomApply(const std::vector<TensorTransform *> &transforms, double prob) : prob_(prob) {
(void)std::transform(
transforms.begin(), transforms.end(), std::back_inserter(transforms_),
[](TensorTransform *op) -> std::shared_ptr<TensorOperation> { return op != nullptr ? op->Parse() : nullptr; });
}
// Function to create RandomChoiceOperation.
std::shared_ptr<RandomChoiceOperation> RandomChoice(const std::vector<std::shared_ptr<TensorOperation>> &transforms) {
auto op = std::make_shared<RandomChoiceOperation>(transforms);
// Input validation
return op->ValidateParams() ? op : nullptr;
RandomApply::RandomApply(const std::vector<std::shared_ptr<TensorTransform>> &transforms, double prob) : prob_(prob) {
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
[](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> {
return op != nullptr ? op->Parse() : nullptr;
});
}
RandomApply::RandomApply(const std::vector<std::reference_wrapper<TensorTransform>> &transforms, double prob)
: prob_(prob) {
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
[](TensorTransform &op) -> std::shared_ptr<TensorOperation> { return op.Parse(); });
}
std::shared_ptr<TensorOperation> RandomApply::Parse() {
return std::make_shared<RandomApplyOperation>(transforms_, prob_);
}
// Constructor to RandomChoice.
RandomChoice::RandomChoice(const std::vector<TensorTransform *> &transforms) {
(void)std::transform(
transforms.begin(), transforms.end(), std::back_inserter(transforms_),
[](TensorTransform *op) -> std::shared_ptr<TensorOperation> { return op != nullptr ? op->Parse() : nullptr; });
}
RandomChoice::RandomChoice(const std::vector<std::shared_ptr<TensorTransform>> &transforms) {
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
[](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> {
return op != nullptr ? op->Parse() : nullptr;
});
}
RandomChoice::RandomChoice(const std::vector<std::reference_wrapper<TensorTransform>> &transforms) {
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
[](TensorTransform &op) -> std::shared_ptr<TensorOperation> { return op.Parse(); });
}
std::shared_ptr<TensorOperation> RandomChoice::Parse() { return std::make_shared<RandomChoiceOperation>(transforms_); }
// Constructor to TypeCast
TypeCast::TypeCast(std::string data_type) : data_type_(data_type) {}
std::shared_ptr<TensorOperation> TypeCast::Parse() { return std::make_shared<TypeCastOperation>(data_type_); }
#ifndef ENABLE_ANDROID
// Constructor to Unique
Unique::Unique() {}
#ifndef ENABLE_ANDROID
std::shared_ptr<TensorOperation> Unique::Parse() { return std::make_shared<UniqueOperation>(); }
#else
std::shared_ptr<TensorOperation> Unique::Parse() {
MS_LOG(ERROR) << "Unique op is not supported for Android.";
return nullptr;
}
#endif
} // namespace transforms
} // namespace dataset

2
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.cc
View File

@ -24,7 +24,7 @@
#include "minddata/dataset/engine/datasetops/map_op/map_op.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/kernels/ir/tensor_operation.h"
#include "minddata/dataset/util/status.h"
namespace mindspore {
namespace dataset {

1
mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.cc
View File

@ -26,6 +26,7 @@
#include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h"
#include "minddata/dataset/engine/ir/datasetops/take_node.h"
#include "minddata/dataset/engine/ir/datasetops/zip_node.h"
#include "minddata/dataset/kernels/ir/tensor_operation.h"
namespace mindspore {
namespace dataset {

17
mindspore/ccsrc/minddata/dataset/include/datasets.h
View File

@ -295,14 +295,18 @@ class Dataset : public std::enable_shared_from_this<Dataset> {
/// \param[in] project_columns A list of column names to project
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current MapDataset
std::shared_ptr<MapDataset> Map(std::vector<std::shared_ptr<TensorOperation>> operations,
std::shared_ptr<MapDataset> Map(std::vector<TensorTransform *> operations,
const std::vector<std::string> &input_columns = {},
const std::vector<std::string> &output_columns = {},
const std::vector<std::string> &project_columns = {},
const std::shared_ptr<DatasetCache> &cache = nullptr,
std::vector<std::shared_ptr<DSCallback>> callbacks = {}) {
return std::make_shared<MapDataset>(shared_from_this(), operations, input_columns, output_columns, project_columns,
cache, callbacks);
std::vector<std::shared_ptr<TensorOperation>> transform_ops;
(void)std::transform(
operations.begin(), operations.end(), std::back_inserter(transform_ops),
[](TensorTransform *op) -> std::shared_ptr<TensorOperation> { return op != nullptr ? op->Parse() : nullptr; });
return std::make_shared<MapDataset>(shared_from_this(), transform_ops, input_columns, output_columns,
project_columns, cache, callbacks);
}
std::shared_ptr<MapDataset> Map(std::vector<std::shared_ptr<TensorTransform>> operations,
@ -312,9 +316,10 @@ class Dataset : public std::enable_shared_from_this<Dataset> {
const std::shared_ptr<DatasetCache> &cache = nullptr,
std::vector<std::shared_ptr<DSCallback>> callbacks = {}) {
std::vector<std::shared_ptr<TensorOperation>> transform_ops;
(void)std::transform(
operations.begin(), operations.end(), std::back_inserter(transform_ops),
[](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> { return op->Parse(); });
(void)std::transform(operations.begin(), operations.end(), std::back_inserter(transform_ops),
[](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> {
return op != nullptr ? op->Parse() : nullptr;
});
return std::make_shared<MapDataset>(shared_from_this(), transform_ops, input_columns, output_columns,
project_columns, cache, callbacks);
}

32
mindspore/ccsrc/minddata/dataset/include/text.h
View File

@ -60,7 +60,7 @@ class BasicTokenizer : public TensorTransform {
~BasicTokenizer() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -100,7 +100,7 @@ class BertTokenizer : public TensorTransform {
~BertTokenizer() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -126,7 +126,7 @@ class CaseFold : public TensorTransform {
~CaseFold() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
//// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
#endif
@ -152,7 +152,7 @@ class JiebaTokenizer : public TensorTransform {
~JiebaTokenizer() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
Status AddWord(const std::string &word, int64_t freq = 0);
@ -181,7 +181,7 @@ class Lookup : public TensorTransform {
~Lookup() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -209,7 +209,7 @@ class Ngram : public TensorTransform {
~Ngram() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -239,7 +239,7 @@ class NormalizeUTF8 : public TensorTransform {
~NormalizeUTF8() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -260,7 +260,7 @@ class RegexReplace : public TensorTransform {
~RegexReplace() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -284,7 +284,7 @@ class RegexTokenizer : public TensorTransform {
~RegexTokenizer() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -312,7 +312,7 @@ class SentencePieceTokenizer : public TensorTransform {
~SentencePieceTokenizer() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -336,7 +336,7 @@ class SlidingWindow : public TensorTransform {
~SlidingWindow() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -359,7 +359,7 @@ class ToNumber : public TensorTransform {
~ToNumber() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -377,7 +377,7 @@ class TruncateSequencePair : public TensorTransform {
~TruncateSequencePair() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -395,7 +395,7 @@ class UnicodeCharTokenizer : public TensorTransform {
~UnicodeCharTokenizer() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -415,7 +415,7 @@ class UnicodeScriptTokenizer : public TensorTransform {
~UnicodeScriptTokenizer() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -434,7 +434,7 @@ class WhitespaceTokenizer : public TensorTransform {
~WhitespaceTokenizer() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:

94
mindspore/ccsrc/minddata/dataset/include/transforms.h
View File

@ -25,11 +25,11 @@
#include "include/api/status.h"
#include "minddata/dataset/include/constants.h"
// FIXME - This internal IR header will be removed when external API classes are provided
#include "minddata/dataset/kernels/ir/data/transforms_ir.h"
namespace mindspore {
namespace dataset {
class TensorOperation;
// Abstract class to represent a tensor transform operation in the data pipeline.
/// \class TensorTransform transforms.h
/// \brief A base class to represent a tensor transform operation in the data pipeline.
@ -54,16 +54,26 @@ class TensorTransform : public std::enable_shared_from_this<TensorTransform> {
// Transform operations for performing data transformation.
namespace transforms {
// Transform Op classes (in alphabetical order)
class ComposeOperation;
class RandomApplyOperation;
class RandomChoiceOperation;
/// \brief Function to create a Compose TensorOperation.
/// \brief Compose Op.
/// \notes Compose a list of transforms into a single transform.
/// \param[in] transforms A vector of transformations to be applied.
/// \return Shared pointer to the current TensorOperation.
std::shared_ptr<ComposeOperation> Compose(const std::vector<std::shared_ptr<TensorOperation>> &transforms);
class Compose : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] transforms A vector of transformations to be applied.
explicit Compose(const std::vector<TensorTransform *> &transforms);
explicit Compose(const std::vector<std::shared_ptr<TensorTransform>> &transforms);
explicit Compose(const std::vector<std::reference_wrapper<TensorTransform>> &transforms);
/// \brief Destructor
~Compose() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
std::vector<std::shared_ptr<TensorOperation>> transforms_;
};
/// \brief Duplicate Op.
/// \notes Duplicate the input tensor to a new output tensor.
@ -77,7 +87,7 @@ class Duplicate : public TensorTransform {
~Duplicate() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
@ -93,26 +103,56 @@ class OneHot : public TensorTransform {
~OneHot() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
float num_classes_;
};
/// \brief Function to create a RandomApply TensorOperation.
/// \brief RandomApply Op.
/// \notes Randomly perform a series of transforms with a given probability.
/// \param[in] transforms A vector of transformations to be applied.
/// \param[in] prob The probability to apply the transformation list (default=0.5)
/// \return Shared pointer to the current TensorOperation.
std::shared_ptr<RandomApplyOperation> RandomApply(const std::vector<std::shared_ptr<TensorOperation>> &transforms,
double prob = 0.5);
class RandomApply : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] transforms A vector of transformations to be applied.
/// \param[in] prob The probability to apply the transformation list (default=0.5)
explicit RandomApply(const std::vector<TensorTransform *> &transforms, double prob = 0.5);
explicit RandomApply(const std::vector<std::shared_ptr<TensorTransform>> &transforms, double prob = 0.5);
explicit RandomApply(const std::vector<std::reference_wrapper<TensorTransform>> &transforms, double prob = 0.5);
/// \brief Function to create a RandomChoice TensorOperation.
/// \brief Destructor
~RandomApply() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
std::vector<std::shared_ptr<TensorOperation>> transforms_;
double prob_;
};
/// \brief RandomChoice Op.
/// \notes Randomly selects one transform from a list of transforms to perform operation.
/// \param[in] transforms A vector of transformations to be chosen from to apply.
/// \return Shared pointer to the current TensorOperation.
std::shared_ptr<RandomChoiceOperation> RandomChoice(const std::vector<std::shared_ptr<TensorOperation>> &transforms);
class RandomChoice : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] transforms A vector of transformations to be chosen from to apply.
explicit RandomChoice(const std::vector<TensorTransform *> &transforms);
explicit RandomChoice(const std::vector<std::shared_ptr<TensorTransform>> &transforms);
explicit RandomChoice(const std::vector<std::reference_wrapper<TensorTransform>> &transforms);
/// \brief Destructor
~RandomChoice() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
std::vector<std::shared_ptr<TensorOperation>> transforms_;
};
/// \brief TypeCast Op.
/// \notes Tensor operation to cast to a given MindSpore data type.
@ -126,14 +166,13 @@ class TypeCast : public TensorTransform {
~TypeCast() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
std::string data_type_;
};
#ifndef ENABLE_ANDROID
/// \brief Unique Op.
/// \notes Return an output tensor containing all the unique elements of the input tensor in
/// the same order that they occur in the input tensor.
@ -146,10 +185,9 @@ class Unique : public TensorTransform {
~Unique() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return return code
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
#endif
} // namespace transforms
} // namespace dataset
} // namespace mindspore

2
mindspore/ccsrc/minddata/dataset/kernels/ir/validators.cc
View File

@ -184,6 +184,8 @@ Status ValidateVectorTransforms(const std::string &op_name,
op_name + ": transform ops must not be null, got transform[" + std::to_string(i) + "] == nullptr.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
} else {
RETURN_IF_NOT_OK(transforms[i]->ValidateParams());
}
}

3
mindspore/ccsrc/minddata/dataset/kernels/ir/validators.h
View File

@ -22,13 +22,12 @@
#include <vector>
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/kernels/ir/tensor_operation.h"
#include "minddata/dataset/util/status.h"
namespace mindspore {
namespace dataset {
class TensorOperation;
// Helper function to validate probability
Status ValidateProbability(const std::string &op_name, const float probability);

2
tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc
View File

@ -166,7 +166,7 @@ TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) {
// TensorShape({211653}), TensorShape({1, 4})};
uint64_t i = 0;
uint64_t j = 0;
// uint64_t j = 0;
while (row.size() != 0) {
// MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape();
// EXPECT_EQ(2, row.size());

2
tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc
View File

@ -238,7 +238,7 @@ TEST_F(MindDataTestPipeline, TestManifestClassIndex) {
iter->GetNextRow(&row);
uint64_t i = 0;
int32_t label_idx = 0;
// int32_t label_idx = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];

248
tests/ut/cpp/dataset/c_api_transforms_test.cc
View File

@ -35,10 +35,11 @@ TEST_F(MindDataTestPipeline, TestComposeSuccess) {
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 3));
EXPECT_NE(ds, nullptr);
/* FIXME - Disable until proper external API for Compose is provided
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> compose = transforms::Compose({vision::Decode(), vision::Resize({777, 777})});
EXPECT_NE(compose, nullptr);
auto decode_op(new vision::Decode());
auto resize_op(new vision::Resize({777, 777}));
transforms::Compose compose({decode_op, resize_op});
// Create a Map operation on ds
ds = ds->Map({compose}, {"image"});
@ -69,25 +70,71 @@ TEST_F(MindDataTestPipeline, TestComposeSuccess) {
// Manually terminate the pipeline
iter->Stop();
*/
}
TEST_F(MindDataTestPipeline, TestComposeFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeFail with invalid transform.";
/* FIXME - Disable until proper external API for Compose is provided
TEST_F(MindDataTestPipeline, TestComposeFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeFail1 with invalid transform.";
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Resize: Non-positive size value: -1 at element: 0
// Compose: transform ops must not be null
std::shared_ptr<TensorOperation> compose1 = transforms::Compose({vision::Decode(), vision::Resize({-1})});
EXPECT_EQ(compose1, nullptr);
auto decode_op = vision::Decode();
auto resize_op = vision::Resize({-1});
auto compose = transforms::Compose({decode_op, resize_op});
// Create a Map operation on ds
ds = ds->Map({compose}, {"image"});
EXPECT_NE(ds, nullptr);
std::shared_ptr<Iterator> iter = ds->CreateIterator();
// Expect failure: invalid Compose parameter(invalid transform op)
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestComposeFail2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeFail2 with invalid transform.";
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Compose: transform ops must not be null
std::shared_ptr<TensorOperation> compose2 = transforms::Compose({vision::Decode(), nullptr});
EXPECT_EQ(compose2, nullptr);
std::shared_ptr<TensorTransform> decode_op = std::make_shared<vision::Decode>();
std::shared_ptr<TensorTransform> compose(new transforms::Compose({decode_op, nullptr}));
// Create a Map operation on ds
ds = ds->Map({compose}, {"image"});
EXPECT_NE(ds, nullptr);
std::shared_ptr<Iterator> iter = ds->CreateIterator();
// Expect failure: invalid Compose parameter (transform ops must not be null)
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestComposeFail3) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeFail3 with invalid transform.";
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Compose: transform list must not be empty
std::shared_ptr<TensorOperation> compose3 = transforms::Compose({});
EXPECT_EQ(compose3, nullptr);
*/
std::vector<std::shared_ptr<TensorTransform>> list = {};
auto compose = transforms::Compose(list);
// Create a Map operation on ds
ds = ds->Map({compose}, {"image"});
EXPECT_NE(ds, nullptr);
std::shared_ptr<Iterator> iter = ds->CreateIterator();
// Expect failure: invalid Compose parameter (transform list must not be empty)
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestDuplicateSuccess) {
@ -296,10 +343,10 @@ TEST_F(MindDataTestPipeline, TestRandomApplySuccess) {
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5));
EXPECT_NE(ds, nullptr);
/* FIXME - Disable until proper external API for RandomApply is provided
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> random_apply = transforms::RandomApply({vision::Resize({777, 777})}, 0.8);
EXPECT_NE(random_apply, nullptr);
auto resize_op = vision::Resize({777, 777});
auto random_apply = transforms::RandomApply({resize_op}, 0.8);
// Create a Map operation on ds
ds = ds->Map({random_apply}, {"image"});
@ -328,29 +375,92 @@ TEST_F(MindDataTestPipeline, TestRandomApplySuccess) {
// Manually terminate the pipeline
iter->Stop();
*/
}
TEST_F(MindDataTestPipeline, TestRandomApplyFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail with invalid transform.";
/* FIXME - Disable until proper external API for RandomApply is provided
TEST_F(MindDataTestPipeline, TestRandomApplyFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail1 with invalid transform.";
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Resize: Non-positive size value: -1 at element: 0
// RandomApply: transform ops must not be null
std::shared_ptr<TensorOperation> random_apply1 = transforms::RandomApply({vision::Decode(), vision::Resize({-1})});
EXPECT_EQ(random_apply1, nullptr);
auto decode_op = vision::Decode();
auto resize_op = vision::Resize({-1});
auto random_apply = transforms::RandomApply({decode_op, resize_op});
// Create a Map operation on ds
ds = ds->Map({random_apply}, {"image"});
EXPECT_NE(ds, nullptr);
std::shared_ptr<Iterator> iter = ds->CreateIterator();
// Expect failure: invalid RandomApply parameter (transform ops must not be null)
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestRandomApplyFail2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail2 with invalid transform.";
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// RandomApply: transform ops must not be null
std::shared_ptr<TensorOperation> random_apply2 = transforms::RandomApply({vision::Decode(), nullptr});
EXPECT_EQ(random_apply2, nullptr);
std::shared_ptr<TensorTransform> decode_op = std::make_shared<vision::Decode>();
std::shared_ptr<TensorTransform> random_apply(new transforms::RandomApply({decode_op, nullptr}));
// RandomApply: transform list must not be empty
std::shared_ptr<TensorOperation> random_apply3 = transforms::RandomApply({});
EXPECT_EQ(random_apply3, nullptr);
// Create a Map operation on ds
ds = ds->Map({random_apply}, {"image"});
EXPECT_NE(ds, nullptr);
std::shared_ptr<Iterator> iter = ds->CreateIterator();
// Expect failure: invalid RandomApply parameter (transform ops must not be null)
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestRandomApplyFail3) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail3 with invalid transform.";
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// RandomApply: Probability has to be between 0 and 1
std::shared_ptr<TensorOperation> random_apply4 = transforms::RandomApply({vision::Resize({100})}, -1);
EXPECT_EQ(random_apply4, nullptr);
*/
auto resize_op = vision::Resize({100});
auto random_apply = transforms::RandomApply({resize_op}, -1);
// Create a Map operation on ds
ds = ds->Map({random_apply}, {"image"});
EXPECT_NE(ds, nullptr);
std::shared_ptr<Iterator> iter = ds->CreateIterator();
// Expect failure: invalid RandomApply parameter (Probability has to be between 0 and 1)
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestRandomApplyFail4) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail4 with invalid transform.";
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// RandomApply: transform list must not be empty
std::vector<std::shared_ptr<TensorTransform>> list = {};
auto random_apply = transforms::RandomApply(list);
// Create a Map operation on ds
ds = ds->Map({random_apply}, {"image"});
EXPECT_NE(ds, nullptr);
std::shared_ptr<Iterator> iter = ds->CreateIterator();
// Expect failure: invalid RandomApply parameter (transform list must not be empty)
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) {
@ -360,11 +470,11 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) {
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 3));
EXPECT_NE(ds, nullptr);
/* FIXME - Disable until proper external API for RandomChoice is provided
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> random_choice =
transforms::RandomChoice({vision::Resize({777, 777}), vision::Resize({888, 888})});
EXPECT_NE(random_choice, nullptr);
std::shared_ptr<TensorTransform> resize_op1(new vision::Resize({777, 777}));
std::shared_ptr<TensorTransform> resize_op2(new vision::Resize({888, 888}));
auto random_choice = transforms::RandomChoice({resize_op1, resize_op2});
// Create a Map operation on ds
ds = ds->Map({random_choice}, {"image"});
@ -393,25 +503,71 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) {
// Manually terminate the pipeline
iter->Stop();
*/
}
TEST_F(MindDataTestPipeline, TestRandomChoiceFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceFail with invalid transform.";
/* FIXME - Disable until proper external API for RandomChoice is provided
TEST_F(MindDataTestPipeline, TestRandomChoiceFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceFail1 with invalid transform.";
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Resize: Non-positive size value: -1 at element: 0
// RandomChoice: transform ops must not be null
std::shared_ptr<TensorOperation> random_choice1 = transforms::RandomChoice({vision::Decode(), vision::Resize({-1})});
EXPECT_EQ(random_choice1, nullptr);
auto decode_op = vision::Decode();
auto resize_op = vision::Resize({-1});
auto random_choice = transforms::RandomChoice({decode_op, resize_op});
// Create a Map operation on ds
ds = ds->Map({random_choice}, {"image"});
EXPECT_NE(ds, nullptr);
std::shared_ptr<Iterator> iter = ds->CreateIterator();
// Expect failure: invalid RandomApply parameter (transform ops must not be null)
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestRandomChoiceFail2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceFail2 with invalid transform.";
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// RandomChoice: transform ops must not be null
std::shared_ptr<TensorOperation> random_choice2 = transforms::RandomChoice({vision::Decode(), nullptr});
EXPECT_EQ(random_choice2, nullptr);
std::shared_ptr<TensorTransform> decode_op = std::make_shared<vision::Decode>();
std::shared_ptr<TensorTransform> random_choice(new transforms::RandomApply({decode_op, nullptr}));
// Create a Map operation on ds
ds = ds->Map({random_choice}, {"image"});
EXPECT_NE(ds, nullptr);
std::shared_ptr<Iterator> iter = ds->CreateIterator();
// Expect failure: invalid RandomApply parameter (transform ops must not be null)
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestRandomChoiceFail3) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceFail3 with invalid transform.";
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// RandomChoice: transform list must not be empty
std::shared_ptr<TensorOperation> random_choice3 = transforms::RandomChoice({});
EXPECT_EQ(random_choice3, nullptr);
*/
std::vector<std::shared_ptr<TensorTransform>> list = {};
auto random_choice = transforms::RandomChoice(list);
// Create a Map operation on ds
ds = ds->Map({random_choice}, {"image"});
EXPECT_NE(ds, nullptr);
std::shared_ptr<Iterator> iter = ds->CreateIterator();
// Expect failure: invalid RandomApply parameter (transform list must not be empty)
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestTypeCastSuccess) {

27
tests/ut/cpp/dataset/optimization_pass_test.cc
View File

@ -27,6 +27,8 @@
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/vision.h"
#include "minddata/dataset/include/vision_lite.h"
#include "minddata/dataset/kernels/ir/data/transforms_ir.h"
#include "minddata/dataset/kernels/ir/vision/vision_ir.h"
using namespace mindspore::dataset;
using mindspore::LogStream;
@ -42,8 +44,7 @@ TEST_F(MindDataTestOptimizationPass, MindDataTestAutoWorkerPass) {
std::shared_ptr<Dataset> map_leaf = ImageFolder("dir")->SetNumWorkers(0);
std::shared_ptr<Dataset> nonmap_leaf = RandomData(44, schema)->SetNumWorkers(0);
std::shared_ptr<Dataset> batch = Zip({map_leaf, nonmap_leaf})->Batch(1)->SetNumWorkers(0);
/* FIXME - Will uncomment out when full external API support is provided
std::shared_ptr<Dataset> map = batch->Map({})->SetNumWorkers(0);
std::shared_ptr<Dataset> map = batch->Map({std::shared_ptr<TensorTransform>()})->SetNumWorkers(0);
// {ImageFolder, RandomData} -> zip -> batch
EXPECT_EQ(map_leaf->IRNode()->num_workers(), 0);
EXPECT_EQ(nonmap_leaf->IRNode()->num_workers(), 0);
@ -65,15 +66,14 @@ TEST_F(MindDataTestOptimizationPass, MindDataTestAutoWorkerPass) {
MS_LOG(DEBUG) << nonmap_leaf->IRNode()->Name() << ": num_worker=" << nonmap_leaf->IRNode()->num_workers();
MS_LOG(DEBUG) << batch->IRNode()->Name() << ": num_worker=" << batch->IRNode()->num_workers();
MS_LOG(DEBUG) << map->IRNode()->Name() << ": num_worker=" << map->IRNode()->num_workers();
*/
}
TEST_F(MindDataTestOptimizationPass, MindDataTestTensorFusionPass) {
MS_LOG(INFO) << "Doing MindDataTestOptimizationPass-MindDataTestTensorFusionPass.";
std::string folder_path = datasets_root_path_ + "/testPK/data/";
/* FIXME - Will uncomment out when full external API support is provided
std::shared_ptr<Dataset> root =
ImageFolder(folder_path, false)->Map({vision::Decode(), vision::RandomResizedCrop({100})}, {"image"});
auto decode_op = vision::Decode();
auto random_resized_crop_op = vision::RandomResizedCrop({100});
std::shared_ptr<Dataset> root = ImageFolder(folder_path, false)->Map({decode_op, random_resized_crop_op}, {"image"});
TensorOpFusionPass fusion_pass;
bool modified = false;
@ -85,27 +85,26 @@ TEST_F(MindDataTestOptimizationPass, MindDataTestTensorFusionPass) {
auto fused_ops = map_node->operations();
ASSERT_EQ(fused_ops.size(), 1);
ASSERT_EQ(fused_ops[0]->Name(), vision::kRandomCropDecodeResizeOperation);
*/
}
TEST_F(MindDataTestOptimizationPass, MindDataTestTensorFusionPassPreBuiltTensorOperation) {
MS_LOG(INFO) << "Doing MindDataTestOptimizationPass-MindDataTestTensorFusionPassPreBuiltTensorOperation.";
std::string folder_path = datasets_root_path_ + "/testPK/data/";
/* FIXME - Will uncomment out when full external API support is provided
// make prebuilt tensor operation
auto decode = std::make_shared<transforms::PreBuiltOperation>(vision::Decode()->Build());
auto resize = std::make_shared<transforms::PreBuiltOperation>(vision::RandomResizedCrop({100})->Build());
std::shared_ptr<Dataset> root = ImageFolder(folder_path, false)->Map({decode, resize}, {"image"});
auto decode = std::make_shared<transforms::PreBuiltOperation>(vision::Decode().Parse()->Build());
auto resize = std::make_shared<transforms::PreBuiltOperation>(vision::RandomResizedCrop({100}).Parse()->Build());
std::vector<std::shared_ptr<TensorOperation>> op_list = {decode, resize};
std::vector<std::string> op_name = {"image"};
std::shared_ptr<DatasetNode> root = ImageFolder(folder_path, false)->IRNode();
std::shared_ptr<MapNode> map_node = std::make_shared<MapNode>(root, op_list, op_name);
TensorOpFusionPass fusion_pass;
bool modified = false;
std::shared_ptr<MapNode> map_node = std::dynamic_pointer_cast<MapNode>(root->IRNode());
// no deepcopy is performed because this doesn't go through tree_adapter
fusion_pass.Run(root->IRNode(), &modified);
fusion_pass.Run(map_node, &modified);
EXPECT_EQ(modified, true);
ASSERT_NE(map_node, nullptr);
auto fused_ops = map_node->operations();
ASSERT_EQ(fused_ops.size(), 1);
ASSERT_EQ(fused_ops[0]->Name(), kRandomCropDecodeResizeOp);
*/
}