From 6f2b4fa665ae013aa6e7e0282b31cf2bd720858e Mon Sep 17 00:00:00 2001 From: YangLuo Date: Tue, 30 Mar 2021 16:27:58 +0800 Subject: [PATCH] fix codex warning --- .../ccsrc/minddata/dataset/api/datasets.cc | 40 +++++------ .../ccsrc/minddata/dataset/api/samplers.cc | 14 ++-- mindspore/ccsrc/minddata/dataset/api/text.cc | 1 - .../ccsrc/minddata/dataset/api/vision.cc | 7 +- .../ccsrc/minddata/dataset/include/datasets.h | 66 ++++++++++--------- .../ccsrc/minddata/dataset/include/samplers.h | 16 ++--- .../ccsrc/minddata/dataset/include/vision.h | 7 +- .../dataset/liteapi/include/datasets.h | 10 +-- .../dataset/liteapi/include/samplers.h | 16 ++--- tests/ut/cpp/dataset/c_api_samplers_test.cc | 27 ++++++++ 10 files changed, 120 insertions(+), 84 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/api/datasets.cc b/mindspore/ccsrc/minddata/dataset/api/datasets.cc index f371c9262c9..85684c6e13b 100644 --- a/mindspore/ccsrc/minddata/dataset/api/datasets.cc +++ b/mindspore/ccsrc/minddata/dataset/api/datasets.cc @@ -800,7 +800,7 @@ AlbumDataset::AlbumDataset(const std::vector &dataset_dir, const std::vect } AlbumDataset::AlbumDataset(const std::vector &dataset_dir, const std::vector &data_schema, - const std::vector> &column_names, bool decode, Sampler *sampler, + const std::vector> &column_names, bool decode, const Sampler *sampler, const std::shared_ptr &cache) { auto sampler_obj = sampler ? sampler->Parse() : nullptr; auto ds = std::make_shared(CharToString(dataset_dir), CharToString(data_schema), @@ -826,8 +826,8 @@ CelebADataset::CelebADataset(const std::vector &dataset_dir, const std::ve SetCharToString(extensions), cache); ir_node_ = std::static_pointer_cast(ds); } -CelebADataset::CelebADataset(const std::vector &dataset_dir, const std::vector &usage, Sampler *sampler, - bool decode, const std::set> &extensions, +CelebADataset::CelebADataset(const std::vector &dataset_dir, const std::vector &usage, + const Sampler *sampler, bool decode, const std::set> &extensions, const std::shared_ptr &cache) { auto sampler_obj = sampler ? sampler->Parse() : nullptr; auto ds = std::make_shared(CharToString(dataset_dir), CharToString(usage), sampler_obj, decode, @@ -850,8 +850,8 @@ Cifar10Dataset::Cifar10Dataset(const std::vector &dataset_dir, const std:: auto ds = std::make_shared(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); ir_node_ = std::static_pointer_cast(ds); } -Cifar10Dataset::Cifar10Dataset(const std::vector &dataset_dir, const std::vector &usage, Sampler *sampler, - const std::shared_ptr &cache) { +Cifar10Dataset::Cifar10Dataset(const std::vector &dataset_dir, const std::vector &usage, + const Sampler *sampler, const std::shared_ptr &cache) { auto sampler_obj = sampler ? sampler->Parse() : nullptr; auto ds = std::make_shared(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); ir_node_ = std::static_pointer_cast(ds); @@ -870,8 +870,8 @@ Cifar100Dataset::Cifar100Dataset(const std::vector &dataset_dir, const std auto ds = std::make_shared(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); ir_node_ = std::static_pointer_cast(ds); } -Cifar100Dataset::Cifar100Dataset(const std::vector &dataset_dir, const std::vector &usage, Sampler *sampler, - const std::shared_ptr &cache) { +Cifar100Dataset::Cifar100Dataset(const std::vector &dataset_dir, const std::vector &usage, + const Sampler *sampler, const std::shared_ptr &cache) { auto sampler_obj = sampler ? sampler->Parse() : nullptr; auto ds = std::make_shared(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); ir_node_ = std::static_pointer_cast(ds); @@ -901,7 +901,7 @@ CocoDataset::CocoDataset(const std::vector &dataset_dir, const std::vector ir_node_ = std::static_pointer_cast(ds); } CocoDataset::CocoDataset(const std::vector &dataset_dir, const std::vector &annotation_file, - const std::vector &task, const bool &decode, Sampler *sampler, + const std::vector &task, const bool &decode, const Sampler *sampler, const std::shared_ptr &cache) { auto sampler_obj = sampler ? sampler->Parse() : nullptr; auto ds = std::make_shared(CharToString(dataset_dir), CharToString(annotation_file), CharToString(task), @@ -942,7 +942,7 @@ ImageFolderDataset::ImageFolderDataset(const std::vector &dataset_dir, boo ir_node_ = std::static_pointer_cast(ds); } -ImageFolderDataset::ImageFolderDataset(const std::vector &dataset_dir, bool decode, Sampler *sampler, +ImageFolderDataset::ImageFolderDataset(const std::vector &dataset_dir, bool decode, const Sampler *sampler, const std::set> &extensions, const std::map, int32_t> &class_indexing, const std::shared_ptr &cache) { @@ -981,7 +981,7 @@ ManifestDataset::ManifestDataset(const std::vector &dataset_file, const st ir_node_ = std::static_pointer_cast(ds); } ManifestDataset::ManifestDataset(const std::vector &dataset_file, const std::vector &usage, - Sampler *sampler, const std::map, int32_t> &class_indexing, + const Sampler *sampler, const std::map, int32_t> &class_indexing, bool decode, const std::shared_ptr &cache) { auto sampler_obj = sampler ? sampler->Parse() : nullptr; auto ds = std::make_shared(CharToString(dataset_file), CharToString(usage), sampler_obj, @@ -1000,7 +1000,7 @@ ManifestDataset::ManifestDataset(const std::vector &dataset_file, const st MindDataDataset::MindDataDataset(const std::vector &dataset_file, const std::vector> &columns_list, - const std::shared_ptr &sampler, nlohmann::json *padded_sample, + const std::shared_ptr &sampler, const nlohmann::json *padded_sample, int64_t num_padded) { auto sampler_obj = sampler ? sampler->Parse() : nullptr; nlohmann::json sample = nullptr; @@ -1012,8 +1012,8 @@ MindDataDataset::MindDataDataset(const std::vector &dataset_file, ir_node_ = std::static_pointer_cast(ds); } MindDataDataset::MindDataDataset(const std::vector &dataset_file, - const std::vector> &columns_list, Sampler *sampler, - nlohmann::json *padded_sample, int64_t num_padded) { + const std::vector> &columns_list, const Sampler *sampler, + const nlohmann::json *padded_sample, int64_t num_padded) { auto sampler_obj = sampler ? sampler->Parse() : nullptr; nlohmann::json sample = nullptr; if (padded_sample) { @@ -1025,7 +1025,7 @@ MindDataDataset::MindDataDataset(const std::vector &dataset_file, } MindDataDataset::MindDataDataset(const std::vector &dataset_file, const std::vector> &columns_list, - const std::reference_wrapper sampler, nlohmann::json *padded_sample, + const std::reference_wrapper sampler, const nlohmann::json *padded_sample, int64_t num_padded) { auto sampler_obj = sampler.get().Parse(); nlohmann::json sample = nullptr; @@ -1039,7 +1039,7 @@ MindDataDataset::MindDataDataset(const std::vector &dataset_file, } MindDataDataset::MindDataDataset(const std::vector> &dataset_files, const std::vector> &columns_list, - const std::shared_ptr &sampler, nlohmann::json *padded_sample, + const std::shared_ptr &sampler, const nlohmann::json *padded_sample, int64_t num_padded) { auto sampler_obj = sampler ? sampler->Parse() : nullptr; nlohmann::json sample = nullptr; @@ -1052,8 +1052,8 @@ MindDataDataset::MindDataDataset(const std::vector> &dataset_f ir_node_ = std::static_pointer_cast(ds); } MindDataDataset::MindDataDataset(const std::vector> &dataset_files, - const std::vector> &columns_list, Sampler *sampler, - nlohmann::json *padded_sample, int64_t num_padded) { + const std::vector> &columns_list, const Sampler *sampler, + const nlohmann::json *padded_sample, int64_t num_padded) { auto sampler_obj = sampler ? sampler->Parse() : nullptr; nlohmann::json sample = nullptr; if (padded_sample) { @@ -1066,7 +1066,7 @@ MindDataDataset::MindDataDataset(const std::vector> &dataset_f } MindDataDataset::MindDataDataset(const std::vector> &dataset_files, const std::vector> &columns_list, - const std::reference_wrapper sampler, nlohmann::json *padded_sample, + const std::reference_wrapper sampler, const nlohmann::json *padded_sample, int64_t num_padded) { auto sampler_obj = sampler.get().Parse(); nlohmann::json sample = nullptr; @@ -1085,7 +1085,7 @@ MnistDataset::MnistDataset(const std::vector &dataset_dir, const std::vect auto ds = std::make_shared(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); ir_node_ = std::static_pointer_cast(ds); } -MnistDataset::MnistDataset(const std::vector &dataset_dir, const std::vector &usage, Sampler *sampler, +MnistDataset::MnistDataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, const std::shared_ptr &cache) { auto sampler_obj = sampler ? sampler->Parse() : nullptr; auto ds = std::make_shared(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); @@ -1118,7 +1118,7 @@ VOCDataset::VOCDataset(const std::vector &dataset_dir, const std::vector &dataset_dir, const std::vector &task, const std::vector &usage, const std::map, int32_t> &class_indexing, - bool decode, Sampler *sampler, const std::shared_ptr &cache) { + bool decode, const Sampler *sampler, const std::shared_ptr &cache) { auto sampler_obj = sampler ? sampler->Parse() : nullptr; auto ds = std::make_shared(CharToString(dataset_dir), CharToString(task), CharToString(usage), MapCharToString(class_indexing), decode, sampler_obj, cache); diff --git a/mindspore/ccsrc/minddata/dataset/api/samplers.cc b/mindspore/ccsrc/minddata/dataset/api/samplers.cc index 50046d135e3..66959690288 100644 --- a/mindspore/ccsrc/minddata/dataset/api/samplers.cc +++ b/mindspore/ccsrc/minddata/dataset/api/samplers.cc @@ -31,7 +31,7 @@ DistributedSampler::DistributedSampler(int64_t num_shards, int64_t shard_id, boo offset_(offset), even_dist_(even_dist) {} -std::shared_ptr DistributedSampler::Parse() { +std::shared_ptr DistributedSampler::Parse() const { return std::make_shared(num_shards_, shard_id_, shuffle_, num_samples_, seed_, offset_, even_dist_); } @@ -40,7 +40,7 @@ std::shared_ptr DistributedSampler::Parse() { PKSampler::PKSampler(int64_t num_val, bool shuffle, int64_t num_samples) : num_val_(num_val), shuffle_(shuffle), num_samples_(num_samples) {} -std::shared_ptr PKSampler::Parse() { +std::shared_ptr PKSampler::Parse() const { return std::make_shared(num_val_, shuffle_, num_samples_); } @@ -48,7 +48,7 @@ std::shared_ptr PKSampler::Parse() { RandomSampler::RandomSampler(bool replacement, int64_t num_samples) : replacement_(replacement), num_samples_(num_samples) {} -std::shared_ptr RandomSampler::Parse() { +std::shared_ptr RandomSampler::Parse() const { return std::make_shared(replacement_, num_samples_); } @@ -56,7 +56,7 @@ std::shared_ptr RandomSampler::Parse() { SequentialSampler::SequentialSampler(int64_t start_index, int64_t num_samples) : start_index_(start_index), num_samples_(num_samples) {} -std::shared_ptr SequentialSampler::Parse() { +std::shared_ptr SequentialSampler::Parse() const { return std::make_shared(start_index_, num_samples_); } @@ -64,7 +64,7 @@ std::shared_ptr SequentialSampler::Parse() { SubsetSampler::SubsetSampler(std::vector indices, int64_t num_samples) : indices_(indices), num_samples_(num_samples) {} -std::shared_ptr SubsetSampler::Parse() { +std::shared_ptr SubsetSampler::Parse() const { return std::make_shared(indices_, num_samples_); } @@ -72,7 +72,7 @@ std::shared_ptr SubsetSampler::Parse() { SubsetRandomSampler::SubsetRandomSampler(std::vector indices, int64_t num_samples) : SubsetSampler(indices, num_samples) {} -std::shared_ptr SubsetRandomSampler::Parse() { +std::shared_ptr SubsetRandomSampler::Parse() const { return std::make_shared(indices_, num_samples_); } @@ -80,7 +80,7 @@ std::shared_ptr SubsetRandomSampler::Parse() { WeightedRandomSampler::WeightedRandomSampler(std::vector weights, int64_t num_samples, bool replacement) : weights_(weights), num_samples_(num_samples), replacement_(replacement) {} -std::shared_ptr WeightedRandomSampler::Parse() { +std::shared_ptr WeightedRandomSampler::Parse() const { return std::make_shared(weights_, num_samples_, replacement_); } diff --git a/mindspore/ccsrc/minddata/dataset/api/text.cc b/mindspore/ccsrc/minddata/dataset/api/text.cc index 1918b79513f..eae4811ff54 100644 --- a/mindspore/ccsrc/minddata/dataset/api/text.cc +++ b/mindspore/ccsrc/minddata/dataset/api/text.cc @@ -244,7 +244,6 @@ struct SentencePieceTokenizer::Data { : vocab_path_(CharToString(vocab_path)), out_type_(out_type) {} std::shared_ptr vocab_; std::string vocab_path_; - SPieceTokenizerLoadType load_type_; SPieceTokenizerOutType out_type_; }; diff --git a/mindspore/ccsrc/minddata/dataset/api/vision.cc b/mindspore/ccsrc/minddata/dataset/api/vision.cc index 6ce82a0365c..abcce030087 100644 --- a/mindspore/ccsrc/minddata/dataset/api/vision.cc +++ b/mindspore/ccsrc/minddata/dataset/api/vision.cc @@ -596,7 +596,8 @@ struct RandomSelectSubpolicy::Data { std::vector, double>>> policy_; }; -RandomSelectSubpolicy::RandomSelectSubpolicy(std::vector>> policy) +RandomSelectSubpolicy::RandomSelectSubpolicy( + const std::vector>> &policy) : data_(std::make_shared()) { for (int32_t i = 0; i < policy.size(); i++) { std::vector, double>> subpolicy; @@ -612,7 +613,7 @@ RandomSelectSubpolicy::RandomSelectSubpolicy(std::vector, double>>> policy) + const std::vector, double>>> &policy) : data_(std::make_shared()) { for (int32_t i = 0; i < policy.size(); i++) { std::vector, double>> subpolicy; @@ -628,7 +629,7 @@ RandomSelectSubpolicy::RandomSelectSubpolicy( } RandomSelectSubpolicy::RandomSelectSubpolicy( - std::vector, double>>> policy) + const std::vector, double>>> &policy) : data_(std::make_shared()) { for (int32_t i = 0; i < policy.size(); i++) { std::vector, double>> subpolicy; diff --git a/mindspore/ccsrc/minddata/dataset/include/datasets.h b/mindspore/ccsrc/minddata/dataset/include/datasets.h index a4d108fea78..9c6ae2455e3 100644 --- a/mindspore/ccsrc/minddata/dataset/include/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/include/datasets.h @@ -690,7 +690,7 @@ class AlbumDataset : public Dataset { const std::vector> &column_names, bool decode, const std::shared_ptr &sampler, const std::shared_ptr &cache); AlbumDataset(const std::vector &dataset_dir, const std::vector &data_schema, - const std::vector> &column_names, bool decode, Sampler *sampler, + const std::vector> &column_names, bool decode, const Sampler *sampler, const std::shared_ptr &cache); AlbumDataset(const std::vector &dataset_dir, const std::vector &data_schema, const std::vector> &column_names, bool decode, @@ -727,7 +727,8 @@ inline std::shared_ptr Album(const std::string &dataset_dir, const /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current Dataset inline std::shared_ptr Album(const std::string &dataset_dir, const std::string &data_schema, - const std::vector &column_names, bool decode, Sampler *sampler, + const std::vector &column_names, bool decode, + const Sampler *sampler, const std::shared_ptr &cache = nullptr) { return std::make_shared(StringToChar(dataset_dir), StringToChar(data_schema), VectorStringToChar(column_names), decode, sampler, cache); @@ -754,7 +755,7 @@ class CelebADataset : public Dataset { explicit CelebADataset(const std::vector &dataset_dir, const std::vector &usage, const std::shared_ptr &sampler, bool decode, const std::set> &extensions, const std::shared_ptr &cache); - explicit CelebADataset(const std::vector &dataset_dir, const std::vector &usage, Sampler *sampler, + explicit CelebADataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, bool decode, const std::set> &extensions, const std::shared_ptr &cache); explicit CelebADataset(const std::vector &dataset_dir, const std::vector &usage, @@ -793,8 +794,9 @@ inline std::shared_ptr CelebA( /// \param[in] extensions Set of file extensions to be included in the dataset (default={}). /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current Dataset -inline std::shared_ptr CelebA(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, - bool decode = false, const std::set &extensions = {}, +inline std::shared_ptr CelebA(const std::string &dataset_dir, const std::string &usage, + const Sampler *sampler, bool decode = false, + const std::set &extensions = {}, const std::shared_ptr &cache = nullptr) { return std::make_shared(StringToChar(dataset_dir), StringToChar(usage), sampler, decode, SetStringToChar(extensions), cache); @@ -822,7 +824,7 @@ class Cifar10Dataset : public Dataset { public: explicit Cifar10Dataset(const std::vector &dataset_dir, const std::vector &usage, const std::shared_ptr &sampler, const std::shared_ptr &cache); - explicit Cifar10Dataset(const std::vector &dataset_dir, const std::vector &usage, Sampler *sampler, + explicit Cifar10Dataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, const std::shared_ptr &cache); explicit Cifar10Dataset(const std::vector &dataset_dir, const std::vector &usage, const std::reference_wrapper sampler, const std::shared_ptr &cache); @@ -853,7 +855,8 @@ inline std::shared_ptr Cifar10( /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current Dataset inline std::shared_ptr Cifar10(const std::string &dataset_dir, const std::string &usage, - Sampler *sampler, const std::shared_ptr &cache = nullptr) { + const Sampler *sampler, + const std::shared_ptr &cache = nullptr) { return std::make_shared(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); } @@ -874,7 +877,7 @@ class Cifar100Dataset : public Dataset { public: explicit Cifar100Dataset(const std::vector &dataset_dir, const std::vector &usage, const std::shared_ptr &sampler, const std::shared_ptr &cache); - explicit Cifar100Dataset(const std::vector &dataset_dir, const std::vector &usage, Sampler *sampler, + explicit Cifar100Dataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, const std::shared_ptr &cache); explicit Cifar100Dataset(const std::vector &dataset_dir, const std::vector &usage, const std::reference_wrapper sampler, const std::shared_ptr &cache); @@ -905,7 +908,7 @@ inline std::shared_ptr Cifar100( /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current Dataset inline std::shared_ptr Cifar100(const std::string &dataset_dir, const std::string &usage, - Sampler *sampler, + const Sampler *sampler, const std::shared_ptr &cache = nullptr) { return std::make_shared(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); } @@ -964,7 +967,7 @@ class CocoDataset : public Dataset { const std::vector &task, const bool &decode, const std::shared_ptr &sampler, const std::shared_ptr &cache); CocoDataset(const std::vector &dataset_dir, const std::vector &annotation_file, - const std::vector &task, const bool &decode, Sampler *sampler, + const std::vector &task, const bool &decode, const Sampler *sampler, const std::shared_ptr &cache); CocoDataset(const std::vector &dataset_dir, const std::vector &annotation_file, const std::vector &task, const bool &decode, const std::reference_wrapper sampler, @@ -1015,7 +1018,7 @@ inline std::shared_ptr Coco(const std::string &dataset_dir, const s /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current Dataset inline std::shared_ptr Coco(const std::string &dataset_dir, const std::string &annotation_file, - const std::string &task, const bool &decode, Sampler *sampler, + const std::string &task, const bool &decode, const Sampler *sampler, const std::shared_ptr &cache = nullptr) { return std::make_shared(StringToChar(dataset_dir), StringToChar(annotation_file), StringToChar(task), decode, sampler, cache); @@ -1091,7 +1094,7 @@ class ImageFolderDataset : public Dataset { const std::shared_ptr &sampler, const std::set> &extensions, const std::map, int32_t> &class_indexing, const std::shared_ptr &cache); - explicit ImageFolderDataset(const std::vector &dataset_dir, bool decode, Sampler *sampler, + explicit ImageFolderDataset(const std::vector &dataset_dir, bool decode, const Sampler *sampler, const std::set> &extensions, const std::map, int32_t> &class_indexing, const std::shared_ptr &cache); @@ -1136,7 +1139,8 @@ inline std::shared_ptr ImageFolder( /// \param[in] class_indexing a class name to label map /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current ImageFolderDataset -inline std::shared_ptr ImageFolder(const std::string &dataset_dir, bool decode, Sampler *sampler, +inline std::shared_ptr ImageFolder(const std::string &dataset_dir, bool decode, + const Sampler *sampler, const std::set &extensions = {}, const std::map &class_indexing = {}, const std::shared_ptr &cache = nullptr) { @@ -1170,9 +1174,9 @@ class ManifestDataset : public Dataset { const std::shared_ptr &sampler, const std::map, int32_t> &class_indexing, bool decode, const std::shared_ptr &cache); - explicit ManifestDataset(const std::vector &dataset_file, const std::vector &usage, Sampler *sampler, - const std::map, int32_t> &class_indexing, bool decode, - const std::shared_ptr &cache); + explicit ManifestDataset(const std::vector &dataset_file, const std::vector &usage, + const Sampler *sampler, const std::map, int32_t> &class_indexing, + bool decode, const std::shared_ptr &cache); explicit ManifestDataset(const std::vector &dataset_file, const std::vector &usage, const std::reference_wrapper sampler, const std::map, int32_t> &class_indexing, bool decode, @@ -1212,7 +1216,7 @@ inline std::shared_ptr Manifest( /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current ManifestDataset inline std::shared_ptr Manifest(const std::string &dataset_file, const std::string &usage, - Sampler *sampler, + const Sampler *sampler, const std::map &class_indexing = {}, bool decode = false, const std::shared_ptr &cache = nullptr) { @@ -1242,21 +1246,22 @@ inline std::shared_ptr Manifest(const std::string &dataset_file class MindDataDataset : public Dataset { public: explicit MindDataDataset(const std::vector &dataset_file, const std::vector> &columns_list, - const std::shared_ptr &sampler, nlohmann::json *padded_sample, int64_t num_padded); + const std::shared_ptr &sampler, const nlohmann::json *padded_sample, + int64_t num_padded); explicit MindDataDataset(const std::vector &dataset_file, const std::vector> &columns_list, - Sampler *sampler, nlohmann::json *padded_sample, int64_t num_padded); + const Sampler *sampler, const nlohmann::json *padded_sample, int64_t num_padded); explicit MindDataDataset(const std::vector &dataset_file, const std::vector> &columns_list, - const std::reference_wrapper sampler, nlohmann::json *padded_sample, + const std::reference_wrapper sampler, const nlohmann::json *padded_sample, int64_t num_padded); explicit MindDataDataset(const std::vector> &dataset_files, const std::vector> &columns_list, const std::shared_ptr &sampler, - nlohmann::json *padded_sample, int64_t num_padded); + const nlohmann::json *padded_sample, int64_t num_padded); explicit MindDataDataset(const std::vector> &dataset_files, - const std::vector> &columns_list, Sampler *sampler, - nlohmann::json *padded_sample, int64_t num_padded); + const std::vector> &columns_list, const Sampler *sampler, + const nlohmann::json *padded_sample, int64_t num_padded); explicit MindDataDataset(const std::vector> &dataset_files, const std::vector> &columns_list, - const std::reference_wrapper sampler, nlohmann::json *padded_sample, + const std::reference_wrapper sampler, const nlohmann::json *padded_sample, int64_t num_padded); ~MindDataDataset() = default; }; @@ -1290,7 +1295,7 @@ inline std::shared_ptr MindData( /// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards. /// \return Shared pointer to the current MindDataDataset inline std::shared_ptr MindData(const std::string &dataset_file, - const std::vector &columns_list, Sampler *sampler, + const std::vector &columns_list, const Sampler *sampler, nlohmann::json *padded_sample = nullptr, int64_t num_padded = 0) { return std::make_shared(StringToChar(dataset_file), VectorStringToChar(columns_list), sampler, padded_sample, num_padded); @@ -1340,7 +1345,7 @@ inline std::shared_ptr MindData( /// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards. /// \return Shared pointer to the current MindDataDataset inline std::shared_ptr MindData(const std::vector &dataset_files, - const std::vector &columns_list, Sampler *sampler, + const std::vector &columns_list, const Sampler *sampler, nlohmann::json *padded_sample = nullptr, int64_t num_padded = 0) { return std::make_shared(VectorStringToChar(dataset_files), VectorStringToChar(columns_list), sampler, padded_sample, num_padded); @@ -1366,7 +1371,7 @@ class MnistDataset : public Dataset { public: explicit MnistDataset(const std::vector &dataset_dir, const std::vector &usage, const std::shared_ptr &sampler, const std::shared_ptr &cache); - explicit MnistDataset(const std::vector &dataset_dir, const std::vector &usage, Sampler *sampler, + explicit MnistDataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, const std::shared_ptr &cache); explicit MnistDataset(const std::vector &dataset_dir, const std::vector &usage, const std::reference_wrapper sampler, const std::shared_ptr &cache); @@ -1395,7 +1400,8 @@ inline std::shared_ptr Mnist(const std::string &dataset_dir, const /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current MnistDataset -inline std::shared_ptr Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, +inline std::shared_ptr Mnist(const std::string &dataset_dir, const std::string &usage, + const Sampler *sampler, const std::shared_ptr &cache = nullptr) { return std::make_shared(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); } @@ -1560,7 +1566,7 @@ class VOCDataset : public Dataset { bool decode, const std::shared_ptr &sampler, const std::shared_ptr &cache); explicit VOCDataset(const std::vector &dataset_dir, const std::vector &task, const std::vector &usage, const std::map, int32_t> &class_indexing, - bool decode, Sampler *sampler, const std::shared_ptr &cache); + bool decode, const Sampler *sampler, const std::shared_ptr &cache); explicit VOCDataset(const std::vector &dataset_dir, const std::vector &task, const std::vector &usage, const std::map, int32_t> &class_indexing, bool decode, const std::reference_wrapper sampler, @@ -1607,7 +1613,7 @@ inline std::shared_ptr VOC(const std::string &dataset_dir, const std /// \return Shared pointer to the current Dataset inline std::shared_ptr VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage, const std::map &class_indexing, - bool decode, Sampler *sampler, + bool decode, const Sampler *sampler, const std::shared_ptr &cache = nullptr) { return std::make_shared(StringToChar(dataset_dir), StringToChar(task), StringToChar(usage), MapStringToChar(class_indexing), decode, sampler, cache); diff --git a/mindspore/ccsrc/minddata/dataset/include/samplers.h b/mindspore/ccsrc/minddata/dataset/include/samplers.h index 13d39de1581..8fe6d99a7f3 100644 --- a/mindspore/ccsrc/minddata/dataset/include/samplers.h +++ b/mindspore/ccsrc/minddata/dataset/include/samplers.h @@ -61,7 +61,7 @@ class Sampler : std::enable_shared_from_this { protected: /// \brief Pure virtual function to convert a Sampler class into an IR Sampler object. /// \return shared pointer to the newly created TensorOperation. - virtual std::shared_ptr Parse() = 0; + virtual std::shared_ptr Parse() const = 0; std::vector> children_; }; @@ -89,7 +89,7 @@ class DistributedSampler final : public Sampler { protected: /// \brief Function to convert a Sampler into an IR SamplerObj. /// \return shared pointer to the newly created SamplerObj. - std::shared_ptr Parse() override; + std::shared_ptr Parse() const override; private: int64_t num_shards_; @@ -120,7 +120,7 @@ class PKSampler final : public Sampler { protected: /// \brief Function to convert a Sampler into an IR SamplerObj. /// \return shared pointer to the newly created SamplerObj. - std::shared_ptr Parse() override; + std::shared_ptr Parse() const override; private: int64_t num_val_; @@ -145,7 +145,7 @@ class RandomSampler final : public Sampler { protected: /// \brief Function to convert a Sampler into an IR SamplerObj. /// \return shared pointer to the newly created SamplerObj. - std::shared_ptr Parse() override; + std::shared_ptr Parse() const override; private: bool replacement_; @@ -169,7 +169,7 @@ class SequentialSampler final : public Sampler { protected: /// \brief Function to convert a Sampler into an IR SamplerObj. /// \return shared pointer to the newly created SamplerObj. - std::shared_ptr Parse() override; + std::shared_ptr Parse() const override; private: int64_t start_index_; @@ -193,7 +193,7 @@ class SubsetSampler : public Sampler { protected: /// \brief Function to convert a Sampler into an IR SamplerObj. /// \return shared pointer to the newly created SamplerObj. - std::shared_ptr Parse() override; + std::shared_ptr Parse() const override; std::vector indices_; int64_t num_samples_; @@ -216,7 +216,7 @@ class SubsetRandomSampler final : public SubsetSampler { protected: /// \brief Function to convert a Sampler into an IR SamplerObj. /// \return shared pointer to the newly created SamplerObj. - std::shared_ptr Parse() override; + std::shared_ptr Parse() const override; }; /// \brief A class to represent a Weighted Random Sampler in the data pipeline. @@ -238,7 +238,7 @@ class WeightedRandomSampler final : public Sampler { protected: /// \brief Function to convert a Sampler into an IR SamplerObj. /// \return shared pointer to the newly created SamplerObj. - std::shared_ptr Parse() override; + std::shared_ptr Parse() const override; private: std::vector weights_; diff --git a/mindspore/ccsrc/minddata/dataset/include/vision.h b/mindspore/ccsrc/minddata/dataset/include/vision.h index 6b84f4216ed..d3050b32453 100644 --- a/mindspore/ccsrc/minddata/dataset/include/vision.h +++ b/mindspore/ccsrc/minddata/dataset/include/vision.h @@ -648,16 +648,17 @@ class RandomSelectSubpolicy final : public TensorTransform { public: /// \brief Constructor. /// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are raw pointers - explicit RandomSelectSubpolicy(std::vector>> policy); + explicit RandomSelectSubpolicy(const std::vector>> &policy); /// \brief Constructor. /// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are shared pointers - explicit RandomSelectSubpolicy(std::vector, double>>> policy); + explicit RandomSelectSubpolicy( + const std::vector, double>>> &policy); /// \brief Constructor. /// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are object pointers explicit RandomSelectSubpolicy( - std::vector, double>>> policy); + const std::vector, double>>> &policy); /// \brief Destructor. ~RandomSelectSubpolicy() = default; diff --git a/mindspore/ccsrc/minddata/dataset/liteapi/include/datasets.h b/mindspore/ccsrc/minddata/dataset/liteapi/include/datasets.h index 0e9a8f72832..eadb609d8f3 100644 --- a/mindspore/ccsrc/minddata/dataset/liteapi/include/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/liteapi/include/datasets.h @@ -456,7 +456,7 @@ class AlbumDataset : public Dataset { const std::vector> &column_names, bool decode, const std::shared_ptr &sampler, const std::shared_ptr &cache); AlbumDataset(const std::vector &dataset_dir, const std::vector &data_schema, - const std::vector> &column_names, bool decode, Sampler *sampler, + const std::vector> &column_names, bool decode, const Sampler *sampler, const std::shared_ptr &cache); AlbumDataset(const std::vector &dataset_dir, const std::vector &data_schema, const std::vector> &column_names, bool decode, @@ -493,7 +493,8 @@ inline std::shared_ptr Album(const std::string &dataset_dir, const /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current Dataset inline std::shared_ptr Album(const std::string &dataset_dir, const std::string &data_schema, - const std::vector &column_names, bool decode, Sampler *sampler, + const std::vector &column_names, bool decode, + const Sampler *sampler, const std::shared_ptr &cache = nullptr) { return std::make_shared(StringToChar(dataset_dir), StringToChar(data_schema), VectorStringToChar(column_names), decode, sampler, cache); @@ -519,7 +520,7 @@ class MnistDataset : public Dataset { public: explicit MnistDataset(const std::vector &dataset_dir, const std::vector &usage, const std::shared_ptr &sampler, const std::shared_ptr &cache); - explicit MnistDataset(const std::vector &dataset_dir, const std::vector &usage, Sampler *sampler, + explicit MnistDataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, const std::shared_ptr &cache); explicit MnistDataset(const std::vector &dataset_dir, const std::vector &usage, const std::reference_wrapper sampler, const std::shared_ptr &cache); @@ -548,7 +549,8 @@ inline std::shared_ptr Mnist(const std::string &dataset_dir, const /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current MnistDataset -inline std::shared_ptr Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, +inline std::shared_ptr Mnist(const std::string &dataset_dir, const std::string &usage, + const Sampler *sampler, const std::shared_ptr &cache = nullptr) { return std::make_shared(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); } diff --git a/mindspore/ccsrc/minddata/dataset/liteapi/include/samplers.h b/mindspore/ccsrc/minddata/dataset/liteapi/include/samplers.h index 13d39de1581..8fe6d99a7f3 100644 --- a/mindspore/ccsrc/minddata/dataset/liteapi/include/samplers.h +++ b/mindspore/ccsrc/minddata/dataset/liteapi/include/samplers.h @@ -61,7 +61,7 @@ class Sampler : std::enable_shared_from_this { protected: /// \brief Pure virtual function to convert a Sampler class into an IR Sampler object. /// \return shared pointer to the newly created TensorOperation. - virtual std::shared_ptr Parse() = 0; + virtual std::shared_ptr Parse() const = 0; std::vector> children_; }; @@ -89,7 +89,7 @@ class DistributedSampler final : public Sampler { protected: /// \brief Function to convert a Sampler into an IR SamplerObj. /// \return shared pointer to the newly created SamplerObj. - std::shared_ptr Parse() override; + std::shared_ptr Parse() const override; private: int64_t num_shards_; @@ -120,7 +120,7 @@ class PKSampler final : public Sampler { protected: /// \brief Function to convert a Sampler into an IR SamplerObj. /// \return shared pointer to the newly created SamplerObj. - std::shared_ptr Parse() override; + std::shared_ptr Parse() const override; private: int64_t num_val_; @@ -145,7 +145,7 @@ class RandomSampler final : public Sampler { protected: /// \brief Function to convert a Sampler into an IR SamplerObj. /// \return shared pointer to the newly created SamplerObj. - std::shared_ptr Parse() override; + std::shared_ptr Parse() const override; private: bool replacement_; @@ -169,7 +169,7 @@ class SequentialSampler final : public Sampler { protected: /// \brief Function to convert a Sampler into an IR SamplerObj. /// \return shared pointer to the newly created SamplerObj. - std::shared_ptr Parse() override; + std::shared_ptr Parse() const override; private: int64_t start_index_; @@ -193,7 +193,7 @@ class SubsetSampler : public Sampler { protected: /// \brief Function to convert a Sampler into an IR SamplerObj. /// \return shared pointer to the newly created SamplerObj. - std::shared_ptr Parse() override; + std::shared_ptr Parse() const override; std::vector indices_; int64_t num_samples_; @@ -216,7 +216,7 @@ class SubsetRandomSampler final : public SubsetSampler { protected: /// \brief Function to convert a Sampler into an IR SamplerObj. /// \return shared pointer to the newly created SamplerObj. - std::shared_ptr Parse() override; + std::shared_ptr Parse() const override; }; /// \brief A class to represent a Weighted Random Sampler in the data pipeline. @@ -238,7 +238,7 @@ class WeightedRandomSampler final : public Sampler { protected: /// \brief Function to convert a Sampler into an IR SamplerObj. /// \return shared pointer to the newly created SamplerObj. - std::shared_ptr Parse() override; + std::shared_ptr Parse() const override; private: std::vector weights_; diff --git a/tests/ut/cpp/dataset/c_api_samplers_test.cc b/tests/ut/cpp/dataset/c_api_samplers_test.cc index dd0604a73ab..f079a68a0d3 100644 --- a/tests/ut/cpp/dataset/c_api_samplers_test.cc +++ b/tests/ut/cpp/dataset/c_api_samplers_test.cc @@ -207,6 +207,33 @@ TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess3) { iter->Stop(); } +TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess4) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess4."; + // Test pointer of distributed_sampler + SequentialSampler sampler = SequentialSampler(0, 4); + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, false, &sampler); + EXPECT_NE(ds, nullptr); + + // Iterate the dataset and get each row + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + std::unordered_map row; + iter->GetNextRow(&row); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto label = row["label"]; + iter->GetNextRow(&row); + } + + EXPECT_EQ(i, 4); + iter->Stop(); +} + TEST_F(MindDataTestPipeline, TestDistributedSamplerFail1) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail1."; // Test basic setting of distributed_sampler