mindspore-Ecosystem
/
mindspore
21
0
Fork 27
Code Issues Pull Requests 6 Projects Releases Wiki Activity

!14754 r1.2 fix codex warning 

From: @luoyang42
Reviewed-by: @heleiwang,@liucunwei
Signed-off-by: @liucunwei
This commit is contained in:
mindspore-ci-bot 2021-04-08 14:23:04 +08:00 committed by Gitee
parent 740b2abf3e 6f2b4fa665
commit 8eb795fa48
10 changed files with 120 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
mindspore/ccsrc/minddata/dataset/api/datasets.cc
View File

@ -800,7 +800,7 @@ AlbumDataset::AlbumDataset(const std::vector<char> &dataset_dir, const std::vect
}
AlbumDataset::AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema,
const std::vector<std::vector<char>> &column_names, bool decode, Sampler *sampler,
const std::vector<std::vector<char>> &column_names, bool decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<AlbumNode>(CharToString(dataset_dir), CharToString(data_schema),
@ -826,8 +826,8 @@ CelebADataset::CelebADataset(const std::vector<char> &dataset_dir, const std::ve
SetCharToString(extensions), cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
CelebADataset::CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
bool decode, const std::set<std::vector<char>> &extensions,
CelebADataset::CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const Sampler *sampler, bool decode, const std::set<std::vector<char>> &extensions,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<CelebANode>(CharToString(dataset_dir), CharToString(usage), sampler_obj, decode,
@ -850,8 +850,8 @@ Cifar10Dataset::Cifar10Dataset(const std::vector<char> &dataset_dir, const std::
auto ds = std::make_shared<Cifar10Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
Cifar10Dataset::Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
Cifar10Dataset::Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<Cifar10Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
@ -870,8 +870,8 @@ Cifar100Dataset::Cifar100Dataset(const std::vector<char> &dataset_dir, const std
auto ds = std::make_shared<Cifar100Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
Cifar100Dataset::Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
Cifar100Dataset::Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<Cifar100Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
@ -901,7 +901,7 @@ CocoDataset::CocoDataset(const std::vector<char> &dataset_dir, const std::vector
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
CocoDataset::CocoDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file,
const std::vector<char> &task, const bool &decode, Sampler *sampler,
const std::vector<char> &task, const bool &decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<CocoNode>(CharToString(dataset_dir), CharToString(annotation_file), CharToString(task),
@ -942,7 +942,7 @@ ImageFolderDataset::ImageFolderDataset(const std::vector<char> &dataset_dir, boo
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
ImageFolderDataset::ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, Sampler *sampler,
ImageFolderDataset::ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, const Sampler *sampler,
const std::set<std::vector<char>> &extensions,
const std::map<std::vector<char>, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache) {
@ -981,7 +981,7 @@ ManifestDataset::ManifestDataset(const std::vector<char> &dataset_file, const st
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
ManifestDataset::ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage,
Sampler *sampler, const std::map<std::vector<char>, int32_t> &class_indexing,
const Sampler *sampler, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<ManifestNode>(CharToString(dataset_file), CharToString(usage), sampler_obj,
@ -1000,7 +1000,7 @@ ManifestDataset::ManifestDataset(const std::vector<char> &dataset_file, const st
MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
const std::vector<std::vector<char>> &columns_list,
const std::shared_ptr<Sampler> &sampler, nlohmann::json *padded_sample,
const std::shared_ptr<Sampler> &sampler, const nlohmann::json *padded_sample,
int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
nlohmann::json sample = nullptr;
@ -1012,8 +1012,8 @@ MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
const std::vector<std::vector<char>> &columns_list, Sampler *sampler,
nlohmann::json *padded_sample, int64_t num_padded) {
const std::vector<std::vector<char>> &columns_list, const Sampler *sampler,
const nlohmann::json *padded_sample, int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
nlohmann::json sample = nullptr;
if (padded_sample) {
@ -1025,7 +1025,7 @@ MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
}
MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample,
const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample,
int64_t num_padded) {
auto sampler_obj = sampler.get().Parse();
nlohmann::json sample = nullptr;
@ -1039,7 +1039,7 @@ MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
}
MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list,
const std::shared_ptr<Sampler> &sampler, nlohmann::json *padded_sample,
const std::shared_ptr<Sampler> &sampler, const nlohmann::json *padded_sample,
int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
nlohmann::json sample = nullptr;
@ -1052,8 +1052,8 @@ MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_f
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list, Sampler *sampler,
nlohmann::json *padded_sample, int64_t num_padded) {
const std::vector<std::vector<char>> &columns_list, const Sampler *sampler,
const nlohmann::json *padded_sample, int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
nlohmann::json sample = nullptr;
if (padded_sample) {
@ -1066,7 +1066,7 @@ MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_f
}
MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample,
const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample,
int64_t num_padded) {
auto sampler_obj = sampler.get().Parse();
nlohmann::json sample = nullptr;
@ -1085,7 +1085,7 @@ MnistDataset::MnistDataset(const std::vector<char> &dataset_dir, const std::vect
auto ds = std::make_shared<MnistNode>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MnistDataset::MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
MnistDataset::MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<MnistNode>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
@ -1118,7 +1118,7 @@ VOCDataset::VOCDataset(const std::vector<char> &dataset_dir, const std::vector<c
}
VOCDataset::VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task,
const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) {
bool decode, const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<VOCNode>(CharToString(dataset_dir), CharToString(task), CharToString(usage),
MapCharToString(class_indexing), decode, sampler_obj, cache);

14
mindspore/ccsrc/minddata/dataset/api/samplers.cc
View File

@ -31,7 +31,7 @@ DistributedSampler::DistributedSampler(int64_t num_shards, int64_t shard_id, boo
offset_(offset),
even_dist_(even_dist) {}
std::shared_ptr<SamplerObj> DistributedSampler::Parse() {
std::shared_ptr<SamplerObj> DistributedSampler::Parse() const {
return std::make_shared<DistributedSamplerObj>(num_shards_, shard_id_, shuffle_, num_samples_, seed_, offset_,
even_dist_);
}
@ -40,7 +40,7 @@ std::shared_ptr<SamplerObj> DistributedSampler::Parse() {
PKSampler::PKSampler(int64_t num_val, bool shuffle, int64_t num_samples)
: num_val_(num_val), shuffle_(shuffle), num_samples_(num_samples) {}
std::shared_ptr<SamplerObj> PKSampler::Parse() {
std::shared_ptr<SamplerObj> PKSampler::Parse() const {
return std::make_shared<PKSamplerObj>(num_val_, shuffle_, num_samples_);
}
@ -48,7 +48,7 @@ std::shared_ptr<SamplerObj> PKSampler::Parse() {
RandomSampler::RandomSampler(bool replacement, int64_t num_samples)
: replacement_(replacement), num_samples_(num_samples) {}
std::shared_ptr<SamplerObj> RandomSampler::Parse() {
std::shared_ptr<SamplerObj> RandomSampler::Parse() const {
return std::make_shared<RandomSamplerObj>(replacement_, num_samples_);
}
@ -56,7 +56,7 @@ std::shared_ptr<SamplerObj> RandomSampler::Parse() {
SequentialSampler::SequentialSampler(int64_t start_index, int64_t num_samples)
: start_index_(start_index), num_samples_(num_samples) {}
std::shared_ptr<SamplerObj> SequentialSampler::Parse() {
std::shared_ptr<SamplerObj> SequentialSampler::Parse() const {
return std::make_shared<SequentialSamplerObj>(start_index_, num_samples_);
}
@ -64,7 +64,7 @@ std::shared_ptr<SamplerObj> SequentialSampler::Parse() {
SubsetSampler::SubsetSampler(std::vector<int64_t> indices, int64_t num_samples)
: indices_(indices), num_samples_(num_samples) {}
std::shared_ptr<SamplerObj> SubsetSampler::Parse() {
std::shared_ptr<SamplerObj> SubsetSampler::Parse() const {
return std::make_shared<SubsetSamplerObj>(indices_, num_samples_);
}
@ -72,7 +72,7 @@ std::shared_ptr<SamplerObj> SubsetSampler::Parse() {
SubsetRandomSampler::SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples)
: SubsetSampler(indices, num_samples) {}
std::shared_ptr<SamplerObj> SubsetRandomSampler::Parse() {
std::shared_ptr<SamplerObj> SubsetRandomSampler::Parse() const {
return std::make_shared<SubsetRandomSamplerObj>(indices_, num_samples_);
}
@ -80,7 +80,7 @@ std::shared_ptr<SamplerObj> SubsetRandomSampler::Parse() {
WeightedRandomSampler::WeightedRandomSampler(std::vector<double> weights, int64_t num_samples, bool replacement)
: weights_(weights), num_samples_(num_samples), replacement_(replacement) {}
std::shared_ptr<SamplerObj> WeightedRandomSampler::Parse() {
std::shared_ptr<SamplerObj> WeightedRandomSampler::Parse() const {
return std::make_shared<WeightedRandomSamplerObj>(weights_, num_samples_, replacement_);
}

1
mindspore/ccsrc/minddata/dataset/api/text.cc
View File

@ -244,7 +244,6 @@ struct SentencePieceTokenizer::Data {
: vocab_path_(CharToString(vocab_path)), out_type_(out_type) {}
std::shared_ptr<SentencePieceVocab> vocab_;
std::string vocab_path_;
SPieceTokenizerLoadType load_type_;
SPieceTokenizerOutType out_type_;
};

7
mindspore/ccsrc/minddata/dataset/api/vision.cc
View File

@ -596,7 +596,8 @@ struct RandomSelectSubpolicy::Data {
std::vector<std::vector<std::pair<std::shared_ptr<TensorOperation>, double>>> policy_;
};
RandomSelectSubpolicy::RandomSelectSubpolicy(std::vector<std::vector<std::pair<TensorTransform *, double>>> policy)
RandomSelectSubpolicy::RandomSelectSubpolicy(
const std::vector<std::vector<std::pair<TensorTransform *, double>>> &policy)
: data_(std::make_shared<Data>()) {
for (int32_t i = 0; i < policy.size(); i++) {
std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy;
@ -612,7 +613,7 @@ RandomSelectSubpolicy::RandomSelectSubpolicy(std::vector<std::vector<std::pair<T
}
RandomSelectSubpolicy::RandomSelectSubpolicy(
std::vector<std::vector<std::pair<std::shared_ptr<TensorTransform>, double>>> policy)
const std::vector<std::vector<std::pair<std::shared_ptr<TensorTransform>, double>>> &policy)
: data_(std::make_shared<Data>()) {
for (int32_t i = 0; i < policy.size(); i++) {
std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy;
@ -628,7 +629,7 @@ RandomSelectSubpolicy::RandomSelectSubpolicy(
}
RandomSelectSubpolicy::RandomSelectSubpolicy(
std::vector<std::vector<std::pair<std::reference_wrapper<TensorTransform>, double>>> policy)
const std::vector<std::vector<std::pair<std::reference_wrapper<TensorTransform>, double>>> &policy)
: data_(std::make_shared<Data>()) {
for (int32_t i = 0; i < policy.size(); i++) {
std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy;

66
mindspore/ccsrc/minddata/dataset/include/datasets.h
View File

@ -690,7 +690,7 @@ class AlbumDataset : public Dataset {
const std::vector<std::vector<char>> &column_names, bool decode, const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache);
AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema,
const std::vector<std::vector<char>> &column_names, bool decode, Sampler *sampler,
const std::vector<std::vector<char>> &column_names, bool decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema,
const std::vector<std::vector<char>> &column_names, bool decode,
@ -727,7 +727,8 @@ inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names, bool decode, Sampler *sampler,
const std::vector<std::string> &column_names, bool decode,
const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<AlbumDataset>(StringToChar(dataset_dir), StringToChar(data_schema),
VectorStringToChar(column_names), decode, sampler, cache);
@ -754,7 +755,7 @@ class CelebADataset : public Dataset {
explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, bool decode,
const std::set<std::vector<char>> &extensions, const std::shared_ptr<DatasetCache> &cache);
explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
bool decode, const std::set<std::vector<char>> &extensions,
const std::shared_ptr<DatasetCache> &cache);
explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
@ -793,8 +794,9 @@ inline std::shared_ptr<CelebADataset> CelebA(
/// \param[in] extensions Set of file extensions to be included in the dataset (default={}).
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
inline std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
bool decode = false, const std::set<std::string> &extensions = {},
inline std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage,
const Sampler *sampler, bool decode = false,
const std::set<std::string> &extensions = {},
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<CelebADataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, decode,
SetStringToChar(extensions), cache);
@ -822,7 +824,7 @@ class Cifar10Dataset : public Dataset {
public:
explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
@ -853,7 +855,8 @@ inline std::shared_ptr<Cifar10Dataset> Cifar10(
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
inline std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage,
Sampler *sampler, const std::shared_ptr<DatasetCache> &cache = nullptr) {
const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<Cifar10Dataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
}
@ -874,7 +877,7 @@ class Cifar100Dataset : public Dataset {
public:
explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
@ -905,7 +908,7 @@ inline std::shared_ptr<Cifar100Dataset> Cifar100(
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
inline std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage,
Sampler *sampler,
const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<Cifar100Dataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
}
@ -964,7 +967,7 @@ class CocoDataset : public Dataset {
const std::vector<char> &task, const bool &decode, const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache);
CocoDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file,
const std::vector<char> &task, const bool &decode, Sampler *sampler,
const std::vector<char> &task, const bool &decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
CocoDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file,
const std::vector<char> &task, const bool &decode, const std::reference_wrapper<Sampler> sampler,
@ -1015,7 +1018,7 @@ inline std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const s
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
inline std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file,
const std::string &task, const bool &decode, Sampler *sampler,
const std::string &task, const bool &decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<CocoDataset>(StringToChar(dataset_dir), StringToChar(annotation_file), StringToChar(task),
decode, sampler, cache);
@ -1091,7 +1094,7 @@ class ImageFolderDataset : public Dataset {
const std::shared_ptr<Sampler> &sampler, const std::set<std::vector<char>> &extensions,
const std::map<std::vector<char>, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache);
explicit ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, Sampler *sampler,
explicit ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, const Sampler *sampler,
const std::set<std::vector<char>> &extensions,
const std::map<std::vector<char>, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache);
@ -1136,7 +1139,8 @@ inline std::shared_ptr<ImageFolderDataset> ImageFolder(
/// \param[in] class_indexing a class name to label map
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current ImageFolderDataset
inline std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, Sampler *sampler,
inline std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode,
const Sampler *sampler,
const std::set<std::string> &extensions = {},
const std::map<std::string, int32_t> &class_indexing = {},
const std::shared_ptr<DatasetCache> &cache = nullptr) {
@ -1170,9 +1174,9 @@ class ManifestDataset : public Dataset {
const std::shared_ptr<Sampler> &sampler,
const std::map<std::vector<char>, int32_t> &class_indexing, bool decode,
const std::shared_ptr<DatasetCache> &cache);
explicit ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage, Sampler *sampler,
const std::map<std::vector<char>, int32_t> &class_indexing, bool decode,
const std::shared_ptr<DatasetCache> &cache);
explicit ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage,
const Sampler *sampler, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, const std::shared_ptr<DatasetCache> &cache);
explicit ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler,
const std::map<std::vector<char>, int32_t> &class_indexing, bool decode,
@ -1212,7 +1216,7 @@ inline std::shared_ptr<ManifestDataset> Manifest(
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current ManifestDataset
inline std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage,
Sampler *sampler,
const Sampler *sampler,
const std::map<std::string, int32_t> &class_indexing = {},
bool decode = false,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
@ -1242,21 +1246,22 @@ inline std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file
class MindDataDataset : public Dataset {
public:
explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
const std::shared_ptr<Sampler> &sampler, nlohmann::json *padded_sample, int64_t num_padded);
const std::shared_ptr<Sampler> &sampler, const nlohmann::json *padded_sample,
int64_t num_padded);
explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
Sampler *sampler, nlohmann::json *padded_sample, int64_t num_padded);
const Sampler *sampler, const nlohmann::json *padded_sample, int64_t num_padded);
explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample,
const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample,
int64_t num_padded);
explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list, const std::shared_ptr<Sampler> &sampler,
nlohmann::json *padded_sample, int64_t num_padded);
const nlohmann::json *padded_sample, int64_t num_padded);
explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list, Sampler *sampler,
nlohmann::json *padded_sample, int64_t num_padded);
const std::vector<std::vector<char>> &columns_list, const Sampler *sampler,
const nlohmann::json *padded_sample, int64_t num_padded);
explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample,
const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample,
int64_t num_padded);
~MindDataDataset() = default;
};
@ -1290,7 +1295,7 @@ inline std::shared_ptr<MindDataDataset> MindData(
/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards.
/// \return Shared pointer to the current MindDataDataset
inline std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file,
const std::vector<std::string> &columns_list, Sampler *sampler,
const std::vector<std::string> &columns_list, const Sampler *sampler,
nlohmann::json *padded_sample = nullptr, int64_t num_padded = 0) {
return std::make_shared<MindDataDataset>(StringToChar(dataset_file), VectorStringToChar(columns_list), sampler,
padded_sample, num_padded);
@ -1340,7 +1345,7 @@ inline std::shared_ptr<MindDataDataset> MindData(
/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards.
/// \return Shared pointer to the current MindDataDataset
inline std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
const std::vector<std::string> &columns_list, Sampler *sampler,
const std::vector<std::string> &columns_list, const Sampler *sampler,
nlohmann::json *padded_sample = nullptr, int64_t num_padded = 0) {
return std::make_shared<MindDataDataset>(VectorStringToChar(dataset_files), VectorStringToChar(columns_list), sampler,
padded_sample, num_padded);
@ -1366,7 +1371,7 @@ class MnistDataset : public Dataset {
public:
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
@ -1395,7 +1400,8 @@ inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current MnistDataset
inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage,
const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<MnistDataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
}
@ -1560,7 +1566,7 @@ class VOCDataset : public Dataset {
bool decode, const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
explicit VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task,
const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, Sampler *sampler, const std::shared_ptr<DatasetCache> &cache);
bool decode, const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache);
explicit VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task,
const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, const std::reference_wrapper<Sampler> sampler,
@ -1607,7 +1613,7 @@ inline std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std
/// \return Shared pointer to the current Dataset
inline std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task,
const std::string &usage, const std::map<std::string, int32_t> &class_indexing,
bool decode, Sampler *sampler,
bool decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<VOCDataset>(StringToChar(dataset_dir), StringToChar(task), StringToChar(usage),
MapStringToChar(class_indexing), decode, sampler, cache);

16
mindspore/ccsrc/minddata/dataset/include/samplers.h
View File

@ -61,7 +61,7 @@ class Sampler : std::enable_shared_from_this<Sampler> {
protected:
/// \brief Pure virtual function to convert a Sampler class into an IR Sampler object.
/// \return shared pointer to the newly created TensorOperation.
virtual std::shared_ptr<SamplerObj> Parse() = 0;
virtual std::shared_ptr<SamplerObj> Parse() const = 0;
std::vector<std::shared_ptr<Sampler>> children_;
};
@ -89,7 +89,7 @@ class DistributedSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t num_shards_;
@ -120,7 +120,7 @@ class PKSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t num_val_;
@ -145,7 +145,7 @@ class RandomSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
bool replacement_;
@ -169,7 +169,7 @@ class SequentialSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t start_index_;
@ -193,7 +193,7 @@ class SubsetSampler : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
std::vector<int64_t> indices_;
int64_t num_samples_;
@ -216,7 +216,7 @@ class SubsetRandomSampler final : public SubsetSampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
};
/// \brief A class to represent a Weighted Random Sampler in the data pipeline.
@ -238,7 +238,7 @@ class WeightedRandomSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
std::vector<double> weights_;

7
mindspore/ccsrc/minddata/dataset/include/vision.h
View File

@ -648,16 +648,17 @@ class RandomSelectSubpolicy final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are raw pointers
explicit RandomSelectSubpolicy(std::vector<std::vector<std::pair<TensorTransform *, double>>> policy);
explicit RandomSelectSubpolicy(const std::vector<std::vector<std::pair<TensorTransform *, double>>> &policy);
/// \brief Constructor.
/// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are shared pointers
explicit RandomSelectSubpolicy(std::vector<std::vector<std::pair<std::shared_ptr<TensorTransform>, double>>> policy);
explicit RandomSelectSubpolicy(
const std::vector<std::vector<std::pair<std::shared_ptr<TensorTransform>, double>>> &policy);
/// \brief Constructor.
/// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are object pointers
explicit RandomSelectSubpolicy(
std::vector<std::vector<std::pair<std::reference_wrapper<TensorTransform>, double>>> policy);
const std::vector<std::vector<std::pair<std::reference_wrapper<TensorTransform>, double>>> &policy);
/// \brief Destructor.
~RandomSelectSubpolicy() = default;

10
mindspore/ccsrc/minddata/dataset/liteapi/include/datasets.h
View File

@ -456,7 +456,7 @@ class AlbumDataset : public Dataset {
const std::vector<std::vector<char>> &column_names, bool decode, const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache);
AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema,
const std::vector<std::vector<char>> &column_names, bool decode, Sampler *sampler,
const std::vector<std::vector<char>> &column_names, bool decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema,
const std::vector<std::vector<char>> &column_names, bool decode,
@ -493,7 +493,8 @@ inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names, bool decode, Sampler *sampler,
const std::vector<std::string> &column_names, bool decode,
const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<AlbumDataset>(StringToChar(dataset_dir), StringToChar(data_schema),
VectorStringToChar(column_names), decode, sampler, cache);
@ -519,7 +520,7 @@ class MnistDataset : public Dataset {
public:
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
@ -548,7 +549,8 @@ inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current MnistDataset
inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage,
const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<MnistDataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
}

16
mindspore/ccsrc/minddata/dataset/liteapi/include/samplers.h
View File

@ -61,7 +61,7 @@ class Sampler : std::enable_shared_from_this<Sampler> {
protected:
/// \brief Pure virtual function to convert a Sampler class into an IR Sampler object.
/// \return shared pointer to the newly created TensorOperation.
virtual std::shared_ptr<SamplerObj> Parse() = 0;
virtual std::shared_ptr<SamplerObj> Parse() const = 0;
std::vector<std::shared_ptr<Sampler>> children_;
};
@ -89,7 +89,7 @@ class DistributedSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t num_shards_;
@ -120,7 +120,7 @@ class PKSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t num_val_;
@ -145,7 +145,7 @@ class RandomSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
bool replacement_;
@ -169,7 +169,7 @@ class SequentialSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t start_index_;
@ -193,7 +193,7 @@ class SubsetSampler : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
std::vector<int64_t> indices_;
int64_t num_samples_;
@ -216,7 +216,7 @@ class SubsetRandomSampler final : public SubsetSampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
};
/// \brief A class to represent a Weighted Random Sampler in the data pipeline.
@ -238,7 +238,7 @@ class WeightedRandomSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
std::vector<double> weights_;

27
tests/ut/cpp/dataset/c_api_samplers_test.cc
View File

@ -207,6 +207,33 @@ TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess3) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess4) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess4.";
// Test pointer of distributed_sampler
SequentialSampler sampler = SequentialSampler(0, 4);
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, &sampler);
EXPECT_NE(ds, nullptr);
// Iterate the dataset and get each row
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto label = row["label"];
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 4);
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestDistributedSamplerFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail1.";
// Test basic setting of distributed_sampler