forked from mindspore-Ecosystem/mindspore
!12484 MD C++ api decouple ABI compile macro
From: @luoyang42 Reviewed-by: Signed-off-by:
This commit is contained in:
Gitee
commit
3f3621e429
|
|
@ -16,11 +16,147 @@
|
|||
#ifndef MINDSPORE_INCLUDE_API_DUAL_ABI_HELPER_H_
|
||||
#define MINDSPORE_INCLUDE_API_DUAL_ABI_HELPER_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <set>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace mindspore {
|
||||
inline std::vector<char> StringToChar(const std::string &s) { return std::vector<char>(s.begin(), s.end()); }
|
||||
|
||||
inline std::string CharToString(const std::vector<char> &c) { return std::string(c.begin(), c.end()); }
|
||||
|
||||
inline std::optional<std::vector<char>> OptionalStringToChar(const std::optional<std::string> &s) {
|
||||
std::optional<std::vector<char>> ret = std::vector<char>(s->begin(), s->end());
|
||||
return (s == std::nullopt) ? std::nullopt : ret;
|
||||
}
|
||||
|
||||
inline std::optional<std::string> OptionalCharToString(const std::optional<std::vector<char>> &c) {
|
||||
std::optional<std::string> ret = std::string(c->begin(), c->end());
|
||||
return (c == std::nullopt) ? std::nullopt : ret;
|
||||
}
|
||||
|
||||
inline std::pair<std::vector<char>, int32_t> PairStringToChar(const std::pair<std::string, int32_t> &s) {
|
||||
return std::pair<std::vector<char>, int32_t>(std::vector<char>(s.first.begin(), s.first.end()), s.second);
|
||||
}
|
||||
|
||||
inline std::pair<std::string, int32_t> PairCharToString(const std::pair<std::vector<char>, int32_t> &c) {
|
||||
return std::pair<std::string, int32_t>(std::string(c.first.begin(), c.first.end()), c.second);
|
||||
}
|
||||
|
||||
inline std::vector<std::vector<char>> VectorStringToChar(const std::vector<std::string> &s) {
|
||||
std::vector<std::vector<char>> ret;
|
||||
std::transform(s.begin(), s.end(), std::back_inserter(ret),
|
||||
[](auto str) { return std::vector<char>(str.begin(), str.end()); });
|
||||
return ret;
|
||||
}
|
||||
|
||||
inline std::vector<std::string> VectorCharToString(const std::vector<std::vector<char>> &c) {
|
||||
std::vector<std::string> ret;
|
||||
std::transform(c.begin(), c.end(), std::back_inserter(ret),
|
||||
[](auto ch) { return std::string(ch.begin(), ch.end()); });
|
||||
return ret;
|
||||
}
|
||||
|
||||
inline std::set<std::vector<char>> SetStringToChar(const std::set<std::string> &s) {
|
||||
std::set<std::vector<char>> ret;
|
||||
std::transform(s.begin(), s.end(), std::inserter(ret, ret.begin()),
|
||||
[](auto str) { return std::vector<char>(str.begin(), str.end()); });
|
||||
return ret;
|
||||
}
|
||||
|
||||
inline std::set<std::string> SetCharToString(const std::set<std::vector<char>> &c) {
|
||||
std::set<std::string> ret;
|
||||
std::transform(c.begin(), c.end(), std::inserter(ret, ret.begin()),
|
||||
[](auto ch) { return std::string(ch.begin(), ch.end()); });
|
||||
return ret;
|
||||
}
|
||||
|
||||
inline std::map<std::vector<char>, int32_t> MapStringToChar(const std::map<std::string, int32_t> &s) {
|
||||
std::map<std::vector<char>, int32_t> ret;
|
||||
std::transform(s.begin(), s.end(), std::inserter(ret, ret.begin()), [](auto str) {
|
||||
return std::pair<std::vector<char>, int32_t>(std::vector<char>(str.first.begin(), str.first.end()), str.second);
|
||||
});
|
||||
return ret;
|
||||
}
|
||||
|
||||
inline std::map<std::string, int32_t> MapCharToString(const std::map<std::vector<char>, int32_t> &c) {
|
||||
std::map<std::string, int32_t> ret;
|
||||
std::transform(c.begin(), c.end(), std::inserter(ret, ret.begin()), [](auto ch) {
|
||||
return std::pair<std::string, int32_t>(std::string(ch.first.begin(), ch.first.end()), ch.second);
|
||||
});
|
||||
return ret;
|
||||
}
|
||||
|
||||
inline std::map<std::vector<char>, std::vector<char>> UnorderedMapStringToChar(
|
||||
const std::unordered_map<std::string, std::string> &s) {
|
||||
std::map<std::vector<char>, std::vector<char>> ret;
|
||||
std::transform(s.begin(), s.end(), std::inserter(ret, ret.begin()), [](auto str) {
|
||||
return std::pair<std::vector<char>, std::vector<char>>(std::vector<char>(str.first.begin(), str.first.end()),
|
||||
std::vector<char>(str.second.begin(), str.second.end()));
|
||||
});
|
||||
return ret;
|
||||
}
|
||||
|
||||
inline std::unordered_map<std::string, std::string> UnorderedMapCharToString(
|
||||
const std::map<std::vector<char>, std::vector<char>> &c) {
|
||||
std::unordered_map<std::string, std::string> ret;
|
||||
std::transform(c.begin(), c.end(), std::inserter(ret, ret.begin()), [](auto ch) {
|
||||
return std::pair<std::string, std::string>(std::string(ch.first.begin(), ch.first.end()),
|
||||
std::string(ch.second.begin(), ch.second.end()));
|
||||
});
|
||||
return ret;
|
||||
}
|
||||
|
||||
inline std::vector<std::pair<std::vector<char>, std::vector<int32_t>>> ClassIndexStringToChar(
|
||||
const std::vector<std::pair<std::string, std::vector<int32_t>>> &s) {
|
||||
std::vector<std::pair<std::vector<char>, std::vector<int32_t>>> ret;
|
||||
std::transform(s.begin(), s.end(), std::back_inserter(ret), [](auto str) {
|
||||
return std::pair<std::vector<char>, std::vector<int32_t>>(std::vector<char>(str.first.begin(), str.first.end()),
|
||||
str.second);
|
||||
});
|
||||
return ret;
|
||||
}
|
||||
|
||||
inline std::vector<std::pair<std::string, std::vector<int32_t>>> ClassIndexCharToString(
|
||||
const std::vector<std::pair<std::vector<char>, std::vector<int32_t>>> &c) {
|
||||
std::vector<std::pair<std::string, std::vector<int32_t>>> ret;
|
||||
std::transform(c.begin(), c.end(), std::back_inserter(ret), [](auto ch) {
|
||||
return std::pair<std::string, std::vector<int32_t>>(std::string(ch.first.begin(), ch.first.end()), ch.second);
|
||||
});
|
||||
return ret;
|
||||
}
|
||||
|
||||
template <class T>
|
||||
inline std::map<std::vector<char>, T> PadInfoStringToChar(const std::map<std::string, T> &s_pad_info) {
|
||||
std::map<std::vector<char>, T> ret;
|
||||
std::transform(s_pad_info.begin(), s_pad_info.end(), std::inserter(ret, ret.begin()), [](auto str) {
|
||||
return std::pair<std::vector<char>, T>(std::vector<char>(str.first.begin(), str.first.end()), str.second);
|
||||
});
|
||||
return ret;
|
||||
}
|
||||
|
||||
template <class T>
|
||||
inline std::map<std::string, T> PadInfoCharToString(const std::map<std::vector<char>, T> &c_pad_info) {
|
||||
std::map<std::string, T> ret;
|
||||
std::transform(c_pad_info.begin(), c_pad_info.end(), std::inserter(ret, ret.begin()), [](auto ch) {
|
||||
return std::pair<std::string, T>(std::string(ch.first.begin(), ch.first.end()), ch.second);
|
||||
});
|
||||
return ret;
|
||||
}
|
||||
|
||||
template <class T>
|
||||
inline void TensorMapCharToString(const std::map<std::vector<char>, T> *c, std::unordered_map<std::string, T> *s) {
|
||||
for (auto ch : *c) {
|
||||
auto key = std::string(ch.first.begin(), ch.first.end());
|
||||
auto val = ch.second;
|
||||
s->insert(std::pair<std::string, T>(key, val));
|
||||
}
|
||||
}
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_INCLUDE_API_DUAL_ABI_HELPER_H_
|
||||
|
|
|
|||
|
|
@ -94,8 +94,8 @@ bool set_callback_timeback(int32_t timeout) {
|
|||
int32_t get_callback_timeout() { return _config->callback_timeout(); }
|
||||
|
||||
// Function to load configurations from a file
|
||||
bool load(std::string file) {
|
||||
Status rc = _config->LoadFile(file);
|
||||
bool load(const std::vector<char> &file) {
|
||||
Status rc = _config->LoadFile(CharToString(file));
|
||||
if (rc.IsError()) {
|
||||
MS_LOG(ERROR) << rc << file;
|
||||
return false;
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -26,7 +26,7 @@ Iterator::Iterator() : consumer_(nullptr) {}
|
|||
Iterator::~Iterator() { Stop(); }
|
||||
|
||||
// Get the next row from the data pipeline.
|
||||
Status Iterator::GetNextRow(MSTensorMap *row) {
|
||||
Status Iterator::GetNextRowCharIF(MSTensorMapChar *row) {
|
||||
// Clean data buffer
|
||||
row->clear();
|
||||
std::unordered_map<std::string, std::shared_ptr<dataset::Tensor>> md_map;
|
||||
|
|
@ -38,7 +38,8 @@ Status Iterator::GetNextRow(MSTensorMap *row) {
|
|||
}
|
||||
for (auto de_tensor : md_map) {
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(de_tensor.second->HasData(), "Apply transform failed, output tensor has no data");
|
||||
row->insert(std::make_pair(de_tensor.first, mindspore::MSTensor(std::make_shared<DETensor>(de_tensor.second))));
|
||||
std::vector<char> col_name(de_tensor.first.begin(), de_tensor.first.end());
|
||||
row->insert(std::make_pair(col_name, mindspore::MSTensor(std::make_shared<DETensor>(de_tensor.second))));
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
|
|
|
|||
|
|
@ -31,38 +31,66 @@ namespace text {
|
|||
|
||||
#ifndef _WIN32
|
||||
// BasicTokenizer
|
||||
struct BasicTokenizer::Data {
|
||||
Data(bool lower_case, bool keep_whitespace, const NormalizeForm normalize_form, bool preserve_unused_token,
|
||||
bool with_offsets)
|
||||
: lower_case_(lower_case),
|
||||
keep_whitespace_(keep_whitespace),
|
||||
normalize_form_(normalize_form),
|
||||
preserve_unused_token_(preserve_unused_token),
|
||||
with_offsets_(with_offsets) {}
|
||||
bool lower_case_;
|
||||
bool keep_whitespace_;
|
||||
NormalizeForm normalize_form_;
|
||||
bool preserve_unused_token_;
|
||||
bool with_offsets_;
|
||||
};
|
||||
|
||||
BasicTokenizer::BasicTokenizer(bool lower_case, bool keep_whitespace, const NormalizeForm normalize_form,
|
||||
bool preserve_unused_token, bool with_offsets)
|
||||
: lower_case_(lower_case),
|
||||
keep_whitespace_(keep_whitespace),
|
||||
normalize_form_(normalize_form),
|
||||
preserve_unused_token_(preserve_unused_token),
|
||||
with_offsets_(with_offsets) {}
|
||||
: data_(std::make_shared<Data>(lower_case, keep_whitespace, normalize_form, preserve_unused_token, with_offsets)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> BasicTokenizer::Parse() {
|
||||
return std::make_shared<BasicTokenizerOperation>(lower_case_, keep_whitespace_, normalize_form_,
|
||||
preserve_unused_token_, with_offsets_);
|
||||
return std::make_shared<BasicTokenizerOperation>(data_->lower_case_, data_->keep_whitespace_, data_->normalize_form_,
|
||||
data_->preserve_unused_token_, data_->with_offsets_);
|
||||
}
|
||||
|
||||
// BertTokenizer
|
||||
BertTokenizer::BertTokenizer(const std::shared_ptr<Vocab> &vocab, const std::string &suffix_indicator,
|
||||
int32_t max_bytes_per_token, const std::string &unknown_token, bool lower_case,
|
||||
struct BertTokenizer::Data {
|
||||
Data(const std::shared_ptr<Vocab> &vocab, const std::vector<char> &suffix_indicator, int32_t max_bytes_per_token,
|
||||
const std::vector<char> &unknown_token, bool lower_case, bool keep_whitespace,
|
||||
const NormalizeForm normalize_form, bool preserve_unused_token, bool with_offsets)
|
||||
: vocab_(vocab),
|
||||
suffix_indicator_(CharToString(suffix_indicator)),
|
||||
max_bytes_per_token_(max_bytes_per_token),
|
||||
unknown_token_(CharToString(unknown_token)),
|
||||
lower_case_(lower_case),
|
||||
keep_whitespace_(keep_whitespace),
|
||||
normalize_form_(normalize_form),
|
||||
preserve_unused_token_(preserve_unused_token),
|
||||
with_offsets_(with_offsets) {}
|
||||
std::shared_ptr<Vocab> vocab_;
|
||||
std::string suffix_indicator_;
|
||||
int32_t max_bytes_per_token_;
|
||||
std::string unknown_token_;
|
||||
bool lower_case_;
|
||||
bool keep_whitespace_;
|
||||
NormalizeForm normalize_form_;
|
||||
bool preserve_unused_token_;
|
||||
bool with_offsets_;
|
||||
};
|
||||
|
||||
BertTokenizer::BertTokenizer(const std::shared_ptr<Vocab> &vocab, const std::vector<char> &suffix_indicator,
|
||||
int32_t max_bytes_per_token, const std::vector<char> &unknown_token, bool lower_case,
|
||||
bool keep_whitespace, const NormalizeForm normalize_form, bool preserve_unused_token,
|
||||
bool with_offsets)
|
||||
: vocab_(vocab),
|
||||
suffix_indicator_(suffix_indicator),
|
||||
max_bytes_per_token_(max_bytes_per_token),
|
||||
unknown_token_(unknown_token),
|
||||
lower_case_(lower_case),
|
||||
keep_whitespace_(keep_whitespace),
|
||||
normalize_form_(normalize_form),
|
||||
preserve_unused_token_(preserve_unused_token),
|
||||
with_offsets_(with_offsets) {}
|
||||
: data_(std::make_shared<Data>(vocab, suffix_indicator, max_bytes_per_token, unknown_token, lower_case,
|
||||
keep_whitespace, normalize_form, preserve_unused_token, with_offsets)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> BertTokenizer::Parse() {
|
||||
return std::make_shared<BertTokenizerOperation>(vocab_, suffix_indicator_, max_bytes_per_token_, unknown_token_,
|
||||
lower_case_, keep_whitespace_, normalize_form_,
|
||||
preserve_unused_token_, with_offsets_);
|
||||
return std::make_shared<BertTokenizerOperation>(
|
||||
data_->vocab_, data_->suffix_indicator_, data_->max_bytes_per_token_, data_->unknown_token_, data_->lower_case_,
|
||||
data_->keep_whitespace_, data_->normalize_form_, data_->preserve_unused_token_, data_->with_offsets_);
|
||||
}
|
||||
|
||||
// CaseFold
|
||||
|
|
@ -72,14 +100,28 @@ std::shared_ptr<TensorOperation> CaseFold::Parse() { return std::make_shared<Cas
|
|||
#endif
|
||||
|
||||
// JiebaTokenizer
|
||||
JiebaTokenizer::JiebaTokenizer(const std::string &hmm_path, const std::string &mp_path, const JiebaMode &mode,
|
||||
bool with_offsets)
|
||||
: hmm_path_(hmm_path), mp_path_(mp_path), mode_(mode), with_offsets_(with_offsets) {}
|
||||
struct JiebaTokenizer::Data {
|
||||
Data(const std::vector<char> &hmm_path, const std::vector<char> &mp_path, const JiebaMode &mode, bool with_offsets)
|
||||
: hmm_path_(CharToString(hmm_path)),
|
||||
mp_path_(CharToString(mp_path)),
|
||||
mode_(mode),
|
||||
with_offsets_(with_offsets),
|
||||
words_list_({}) {}
|
||||
std::string hmm_path_;
|
||||
std::string mp_path_;
|
||||
JiebaMode mode_;
|
||||
bool with_offsets_;
|
||||
std::vector<std::pair<std::string, int64_t>> words_list_;
|
||||
};
|
||||
|
||||
JiebaTokenizer::JiebaTokenizer(const std::vector<char> &hmm_path, const std::vector<char> &mp_path,
|
||||
const JiebaMode &mode, bool with_offsets)
|
||||
: data_(std::make_shared<Data>(hmm_path, mp_path, mode, with_offsets)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> JiebaTokenizer::Parse() {
|
||||
std::shared_ptr<JiebaTokenizerOperation> jieba_tokenizer =
|
||||
std::make_shared<JiebaTokenizerOperation>(hmm_path_, mp_path_, mode_, with_offsets_);
|
||||
for (auto &word : words_list_) {
|
||||
std::make_shared<JiebaTokenizerOperation>(data_->hmm_path_, data_->mp_path_, data_->mode_, data_->with_offsets_);
|
||||
for (auto &word : data_->words_list_) {
|
||||
Status rc = jieba_tokenizer->AddWord(word.first, word.second);
|
||||
if (rc.IsError()) {
|
||||
MS_LOG(ERROR) << rc;
|
||||
|
|
@ -100,109 +142,199 @@ Status JiebaTokenizer::AddWord(const std::string &word, int64_t freq) {
|
|||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
words_list_.emplace_back(word, freq);
|
||||
data_->words_list_.emplace_back(word, freq);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
// Lookup
|
||||
Lookup::Lookup(const std::shared_ptr<Vocab> &vocab, const std::optional<std::string> &unknown_token,
|
||||
const std::string &data_type)
|
||||
: vocab_(vocab), unknown_token_(unknown_token), data_type_(data_type) {}
|
||||
struct Lookup::Data {
|
||||
Data(const std::shared_ptr<Vocab> &vocab, const std::optional<std::vector<char>> &unknown_token,
|
||||
const std::vector<char> &data_type)
|
||||
: vocab_(vocab), unknown_token_(OptionalCharToString(unknown_token)), data_type_(CharToString(data_type)) {}
|
||||
std::shared_ptr<Vocab> vocab_;
|
||||
std::optional<std::string> unknown_token_;
|
||||
std::string data_type_;
|
||||
};
|
||||
|
||||
Lookup::Lookup(const std::shared_ptr<Vocab> &vocab, const std::optional<std::vector<char>> &unknown_token,
|
||||
const std::vector<char> &data_type)
|
||||
: data_(std::make_shared<Data>(vocab, unknown_token, data_type)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> Lookup::Parse() {
|
||||
return std::make_shared<LookupOperation>(vocab_, unknown_token_, data_type_);
|
||||
return std::make_shared<LookupOperation>(data_->vocab_, data_->unknown_token_, data_->data_type_);
|
||||
}
|
||||
|
||||
// Ngram
|
||||
Ngram::Ngram(const std::vector<int32_t> &ngrams, const std::pair<std::string, int32_t> &left_pad,
|
||||
const std::pair<std::string, int32_t> &right_pad, const std::string &separator)
|
||||
: ngrams_(ngrams), left_pad_(left_pad), right_pad_(right_pad), separator_(separator) {}
|
||||
struct Ngram::Data {
|
||||
Data(const std::vector<int32_t> &ngrams, const std::pair<std::vector<char>, int32_t> &left_pad,
|
||||
const std::pair<std::vector<char>, int32_t> &right_pad, const std::vector<char> &separator)
|
||||
: ngrams_(ngrams),
|
||||
left_pad_(PairCharToString(left_pad)),
|
||||
right_pad_(PairCharToString(right_pad)),
|
||||
separator_(CharToString(separator)) {}
|
||||
std::vector<int32_t> ngrams_;
|
||||
std::pair<std::string, int32_t> left_pad_;
|
||||
std::pair<std::string, int32_t> right_pad_;
|
||||
std::string separator_;
|
||||
};
|
||||
|
||||
Ngram::Ngram(const std::vector<int32_t> &ngrams, const std::pair<std::vector<char>, int32_t> &left_pad,
|
||||
const std::pair<std::vector<char>, int32_t> &right_pad, const std::vector<char> &separator)
|
||||
: data_(std::make_shared<Data>(ngrams, left_pad, right_pad, separator)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> Ngram::Parse() {
|
||||
return std::make_shared<NgramOperation>(ngrams_, left_pad_, right_pad_, separator_);
|
||||
return std::make_shared<NgramOperation>(data_->ngrams_, data_->left_pad_, data_->right_pad_, data_->separator_);
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
// NormalizeUTF8
|
||||
NormalizeUTF8::NormalizeUTF8(NormalizeForm normalize_form) : normalize_form_(normalize_form) {}
|
||||
struct NormalizeUTF8::Data {
|
||||
explicit Data(NormalizeForm normalize_form) : normalize_form_(normalize_form) {}
|
||||
NormalizeForm normalize_form_;
|
||||
};
|
||||
|
||||
NormalizeUTF8::NormalizeUTF8(NormalizeForm normalize_form) : data_(std::make_shared<Data>(normalize_form)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> NormalizeUTF8::Parse() {
|
||||
return std::make_shared<NormalizeUTF8Operation>(normalize_form_);
|
||||
return std::make_shared<NormalizeUTF8Operation>(data_->normalize_form_);
|
||||
}
|
||||
|
||||
// RegexReplace
|
||||
RegexReplace::RegexReplace(std::string pattern, std::string replace, bool replace_all)
|
||||
: pattern_(pattern), replace_(replace), replace_all_(replace_all) {}
|
||||
struct RegexReplace::Data {
|
||||
Data(const std::vector<char> &pattern, const std::vector<char> &replace, bool replace_all)
|
||||
: pattern_(CharToString(pattern)), replace_(CharToString(replace)), replace_all_(replace_all) {}
|
||||
std::string pattern_;
|
||||
std::string replace_;
|
||||
bool replace_all_;
|
||||
};
|
||||
|
||||
RegexReplace::RegexReplace(const std::vector<char> &pattern, const std::vector<char> &replace, bool replace_all)
|
||||
: data_(std::make_shared<Data>(pattern, replace, replace_all)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RegexReplace::Parse() {
|
||||
return std::make_shared<RegexReplaceOperation>(pattern_, replace_, replace_all_);
|
||||
return std::make_shared<RegexReplaceOperation>(data_->pattern_, data_->replace_, data_->replace_all_);
|
||||
}
|
||||
|
||||
// RegexTokenizer
|
||||
RegexTokenizer::RegexTokenizer(std::string delim_pattern, std::string keep_delim_pattern, bool with_offsets)
|
||||
: delim_pattern_(delim_pattern), keep_delim_pattern_(keep_delim_pattern), with_offsets_(with_offsets) {}
|
||||
struct RegexTokenizer::Data {
|
||||
Data(const std::vector<char> &delim_pattern, const std::vector<char> &keep_delim_pattern, bool with_offsets)
|
||||
: delim_pattern_(CharToString(delim_pattern)),
|
||||
keep_delim_pattern_(CharToString(keep_delim_pattern)),
|
||||
with_offsets_(with_offsets) {}
|
||||
std::string delim_pattern_;
|
||||
std::string keep_delim_pattern_;
|
||||
bool with_offsets_;
|
||||
};
|
||||
|
||||
RegexTokenizer::RegexTokenizer(const std::vector<char> &delim_pattern, const std::vector<char> &keep_delim_pattern,
|
||||
bool with_offsets)
|
||||
: data_(std::make_shared<Data>(delim_pattern, keep_delim_pattern, with_offsets)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RegexTokenizer::Parse() {
|
||||
return std::make_shared<RegexTokenizerOperation>(delim_pattern_, keep_delim_pattern_, with_offsets_);
|
||||
return std::make_shared<RegexTokenizerOperation>(data_->delim_pattern_, data_->keep_delim_pattern_,
|
||||
data_->with_offsets_);
|
||||
}
|
||||
#endif
|
||||
|
||||
// SentencePieceTokenizer
|
||||
struct SentencePieceTokenizer::Data {
|
||||
Data(const std::shared_ptr<SentencePieceVocab> &vocab, SPieceTokenizerOutType out_type)
|
||||
: vocab_(vocab), out_type_(out_type) {}
|
||||
Data(const std::vector<char> &vocab_path, SPieceTokenizerOutType out_type)
|
||||
: vocab_path_(CharToString(vocab_path)), out_type_(out_type) {}
|
||||
std::shared_ptr<SentencePieceVocab> vocab_;
|
||||
std::string vocab_path_;
|
||||
SPieceTokenizerLoadType load_type_;
|
||||
SPieceTokenizerOutType out_type_;
|
||||
};
|
||||
|
||||
SentencePieceTokenizer::SentencePieceTokenizer(const std::shared_ptr<SentencePieceVocab> &vocab,
|
||||
SPieceTokenizerOutType out_type)
|
||||
: vocab_(vocab), out_type_(out_type) {}
|
||||
: data_(std::make_shared<Data>(vocab, out_type)) {}
|
||||
|
||||
SentencePieceTokenizer::SentencePieceTokenizer(const std::string &vocab_path, SPieceTokenizerOutType out_type)
|
||||
: vocab_path_(vocab_path), out_type_(out_type) {}
|
||||
SentencePieceTokenizer::SentencePieceTokenizer(const std::vector<char> &vocab_path, SPieceTokenizerOutType out_type)
|
||||
: data_(std::make_shared<Data>(vocab_path, out_type)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> SentencePieceTokenizer::Parse() {
|
||||
if (vocab_ != nullptr) {
|
||||
return std::make_shared<SentencePieceTokenizerOperation>(vocab_, out_type_);
|
||||
if (data_->vocab_ != nullptr) {
|
||||
return std::make_shared<SentencePieceTokenizerOperation>(data_->vocab_, data_->out_type_);
|
||||
} else {
|
||||
return std::make_shared<SentencePieceTokenizerOperation>(vocab_path_, out_type_);
|
||||
return std::make_shared<SentencePieceTokenizerOperation>(data_->vocab_path_, data_->out_type_);
|
||||
}
|
||||
}
|
||||
|
||||
// SlidingWindow
|
||||
SlidingWindow::SlidingWindow(const int32_t width, const int32_t axis) : width_(width), axis_(axis) {}
|
||||
struct SlidingWindow::Data {
|
||||
Data(const int32_t width, const int32_t axis) : width_(width), axis_(axis) {}
|
||||
int32_t width_;
|
||||
int32_t axis_;
|
||||
};
|
||||
|
||||
SlidingWindow::SlidingWindow(const int32_t width, const int32_t axis) : data_(std::make_shared<Data>(width, axis)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> SlidingWindow::Parse() {
|
||||
return std::make_shared<SlidingWindowOperation>(width_, axis_);
|
||||
return std::make_shared<SlidingWindowOperation>(data_->width_, data_->axis_);
|
||||
}
|
||||
|
||||
// ToNumber
|
||||
ToNumber::ToNumber(const std::string &data_type) : data_type_(data_type) {}
|
||||
struct ToNumber::Data {
|
||||
explicit Data(const std::vector<char> &data_type) : data_type_(CharToString(data_type)) {}
|
||||
std::string data_type_;
|
||||
};
|
||||
|
||||
std::shared_ptr<TensorOperation> ToNumber::Parse() { return std::make_shared<ToNumberOperation>(data_type_); }
|
||||
ToNumber::ToNumber(const std::vector<char> &data_type) : data_(std::make_shared<Data>(data_type)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> ToNumber::Parse() { return std::make_shared<ToNumberOperation>(data_->data_type_); }
|
||||
|
||||
// TruncateSequencePair
|
||||
TruncateSequencePair::TruncateSequencePair(int32_t max_length) : max_length_(max_length) {}
|
||||
struct TruncateSequencePair::Data {
|
||||
explicit Data(int32_t max_length) : max_length_(max_length) {}
|
||||
int32_t max_length_;
|
||||
};
|
||||
|
||||
TruncateSequencePair::TruncateSequencePair(int32_t max_length) : data_(std::make_shared<Data>(max_length)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> TruncateSequencePair::Parse() {
|
||||
return std::make_shared<TruncateSequencePairOperation>(max_length_);
|
||||
return std::make_shared<TruncateSequencePairOperation>(data_->max_length_);
|
||||
}
|
||||
|
||||
// UnicodeCharTokenizer
|
||||
UnicodeCharTokenizer::UnicodeCharTokenizer(bool with_offsets) : with_offsets_(with_offsets) {}
|
||||
struct UnicodeCharTokenizer::Data {
|
||||
explicit Data(bool with_offsets) : with_offsets_(with_offsets) {}
|
||||
bool with_offsets_;
|
||||
};
|
||||
|
||||
UnicodeCharTokenizer::UnicodeCharTokenizer(bool with_offsets) : data_(std::make_shared<Data>(with_offsets)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> UnicodeCharTokenizer::Parse() {
|
||||
return std::make_shared<UnicodeCharTokenizerOperation>(with_offsets_);
|
||||
return std::make_shared<UnicodeCharTokenizerOperation>(data_->with_offsets_);
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
// UnicodeScriptTokenizer
|
||||
struct UnicodeScriptTokenizer::Data {
|
||||
Data(bool keep_whitespace, bool with_offsets) : keep_whitespace_(keep_whitespace), with_offsets_(with_offsets) {}
|
||||
bool keep_whitespace_;
|
||||
bool with_offsets_;
|
||||
};
|
||||
|
||||
UnicodeScriptTokenizer::UnicodeScriptTokenizer(bool keep_whitespace, bool with_offsets)
|
||||
: keep_whitespace_(keep_whitespace), with_offsets_(with_offsets) {}
|
||||
: data_(std::make_shared<Data>(keep_whitespace, with_offsets)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> UnicodeScriptTokenizer::Parse() {
|
||||
return std::make_shared<UnicodeScriptTokenizerOperation>(keep_whitespace_, with_offsets_);
|
||||
return std::make_shared<UnicodeScriptTokenizerOperation>(data_->keep_whitespace_, data_->with_offsets_);
|
||||
}
|
||||
|
||||
// WhitespaceTokenizer
|
||||
WhitespaceTokenizer::WhitespaceTokenizer(bool with_offsets) : with_offsets_(with_offsets) {}
|
||||
struct WhitespaceTokenizer::Data {
|
||||
explicit Data(bool with_offsets) : with_offsets_(with_offsets) {}
|
||||
bool with_offsets_;
|
||||
};
|
||||
|
||||
WhitespaceTokenizer::WhitespaceTokenizer(bool with_offsets) : data_(std::make_shared<Data>(with_offsets)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> WhitespaceTokenizer::Parse() {
|
||||
return std::make_shared<WhitespaceTokenizerOperation>(with_offsets_);
|
||||
return std::make_shared<WhitespaceTokenizerOperation>(data_->with_offsets_);
|
||||
}
|
||||
#endif
|
||||
} // namespace text
|
||||
|
|
|
|||
|
|
@ -30,25 +30,30 @@ namespace transforms {
|
|||
// (In alphabetical order)
|
||||
|
||||
// Constructor to Compose.
|
||||
Compose::Compose(const std::vector<TensorTransform *> &transforms) {
|
||||
struct Compose::Data {
|
||||
std::vector<std::shared_ptr<TensorOperation>> transforms_;
|
||||
};
|
||||
|
||||
Compose::Compose(const std::vector<TensorTransform *> &transforms) : data_(std::make_shared<Data>()) {
|
||||
(void)std::transform(
|
||||
transforms.begin(), transforms.end(), std::back_inserter(transforms_),
|
||||
transforms.begin(), transforms.end(), std::back_inserter(data_->transforms_),
|
||||
[](TensorTransform *op) -> std::shared_ptr<TensorOperation> { return op != nullptr ? op->Parse() : nullptr; });
|
||||
}
|
||||
|
||||
Compose::Compose(const std::vector<std::shared_ptr<TensorTransform>> &transforms) {
|
||||
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
|
||||
Compose::Compose(const std::vector<std::shared_ptr<TensorTransform>> &transforms) : data_(std::make_shared<Data>()) {
|
||||
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(data_->transforms_),
|
||||
[](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> {
|
||||
return op != nullptr ? op->Parse() : nullptr;
|
||||
});
|
||||
}
|
||||
|
||||
Compose::Compose(const std::vector<std::reference_wrapper<TensorTransform>> &transforms) {
|
||||
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
|
||||
Compose::Compose(const std::vector<std::reference_wrapper<TensorTransform>> &transforms)
|
||||
: data_(std::make_shared<Data>()) {
|
||||
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(data_->transforms_),
|
||||
[](TensorTransform &op) -> std::shared_ptr<TensorOperation> { return op.Parse(); });
|
||||
}
|
||||
|
||||
std::shared_ptr<TensorOperation> Compose::Parse() { return std::make_shared<ComposeOperation>(transforms_); }
|
||||
std::shared_ptr<TensorOperation> Compose::Parse() { return std::make_shared<ComposeOperation>(data_->transforms_); }
|
||||
|
||||
// Constructor to Duplicate
|
||||
Duplicate::Duplicate() {}
|
||||
|
|
@ -56,59 +61,87 @@ Duplicate::Duplicate() {}
|
|||
std::shared_ptr<TensorOperation> Duplicate::Parse() { return std::make_shared<DuplicateOperation>(); }
|
||||
|
||||
// Constructor to OneHot
|
||||
OneHot::OneHot(int32_t num_classes) : num_classes_(num_classes) {}
|
||||
struct OneHot::Data {
|
||||
explicit Data(int32_t num_classes) : num_classes_(num_classes) {}
|
||||
float num_classes_;
|
||||
};
|
||||
|
||||
std::shared_ptr<TensorOperation> OneHot::Parse() { return std::make_shared<OneHotOperation>(num_classes_); }
|
||||
OneHot::OneHot(int32_t num_classes) : data_(std::make_shared<Data>(num_classes)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> OneHot::Parse() { return std::make_shared<OneHotOperation>(data_->num_classes_); }
|
||||
|
||||
// Constructor to RandomApply.
|
||||
RandomApply::RandomApply(const std::vector<TensorTransform *> &transforms, double prob) : prob_(prob) {
|
||||
struct RandomApply::Data {
|
||||
std::vector<std::shared_ptr<TensorOperation>> transforms_;
|
||||
double prob_;
|
||||
};
|
||||
|
||||
RandomApply::RandomApply(const std::vector<TensorTransform *> &transforms, double prob)
|
||||
: data_(std::make_shared<Data>()) {
|
||||
(void)std::transform(
|
||||
transforms.begin(), transforms.end(), std::back_inserter(transforms_),
|
||||
transforms.begin(), transforms.end(), std::back_inserter(data_->transforms_),
|
||||
[](TensorTransform *op) -> std::shared_ptr<TensorOperation> { return op != nullptr ? op->Parse() : nullptr; });
|
||||
data_->prob_ = prob;
|
||||
}
|
||||
|
||||
RandomApply::RandomApply(const std::vector<std::shared_ptr<TensorTransform>> &transforms, double prob) : prob_(prob) {
|
||||
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
|
||||
RandomApply::RandomApply(const std::vector<std::shared_ptr<TensorTransform>> &transforms, double prob)
|
||||
: data_(std::make_shared<Data>()) {
|
||||
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(data_->transforms_),
|
||||
[](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> {
|
||||
return op != nullptr ? op->Parse() : nullptr;
|
||||
});
|
||||
data_->prob_ = prob;
|
||||
}
|
||||
|
||||
RandomApply::RandomApply(const std::vector<std::reference_wrapper<TensorTransform>> &transforms, double prob)
|
||||
: prob_(prob) {
|
||||
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
|
||||
: data_(std::make_shared<Data>()) {
|
||||
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(data_->transforms_),
|
||||
[](TensorTransform &op) -> std::shared_ptr<TensorOperation> { return op.Parse(); });
|
||||
data_->prob_ = prob;
|
||||
}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomApply::Parse() {
|
||||
return std::make_shared<RandomApplyOperation>(transforms_, prob_);
|
||||
return std::make_shared<RandomApplyOperation>(data_->transforms_, data_->prob_);
|
||||
}
|
||||
|
||||
// Constructor to RandomChoice.
|
||||
RandomChoice::RandomChoice(const std::vector<TensorTransform *> &transforms) {
|
||||
struct RandomChoice::Data {
|
||||
std::vector<std::shared_ptr<TensorOperation>> transforms_;
|
||||
};
|
||||
|
||||
RandomChoice::RandomChoice(const std::vector<TensorTransform *> &transforms) : data_(std::make_shared<Data>()) {
|
||||
(void)std::transform(
|
||||
transforms.begin(), transforms.end(), std::back_inserter(transforms_),
|
||||
transforms.begin(), transforms.end(), std::back_inserter(data_->transforms_),
|
||||
[](TensorTransform *op) -> std::shared_ptr<TensorOperation> { return op != nullptr ? op->Parse() : nullptr; });
|
||||
}
|
||||
|
||||
RandomChoice::RandomChoice(const std::vector<std::shared_ptr<TensorTransform>> &transforms) {
|
||||
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
|
||||
RandomChoice::RandomChoice(const std::vector<std::shared_ptr<TensorTransform>> &transforms)
|
||||
: data_(std::make_shared<Data>()) {
|
||||
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(data_->transforms_),
|
||||
[](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> {
|
||||
return op != nullptr ? op->Parse() : nullptr;
|
||||
});
|
||||
}
|
||||
|
||||
RandomChoice::RandomChoice(const std::vector<std::reference_wrapper<TensorTransform>> &transforms) {
|
||||
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
|
||||
RandomChoice::RandomChoice(const std::vector<std::reference_wrapper<TensorTransform>> &transforms)
|
||||
: data_(std::make_shared<Data>()) {
|
||||
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(data_->transforms_),
|
||||
[](TensorTransform &op) -> std::shared_ptr<TensorOperation> { return op.Parse(); });
|
||||
}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomChoice::Parse() { return std::make_shared<RandomChoiceOperation>(transforms_); }
|
||||
std::shared_ptr<TensorOperation> RandomChoice::Parse() {
|
||||
return std::make_shared<RandomChoiceOperation>(data_->transforms_);
|
||||
}
|
||||
|
||||
// Constructor to TypeCast
|
||||
TypeCast::TypeCast(std::string data_type) : data_type_(data_type) {}
|
||||
struct TypeCast::Data {
|
||||
explicit Data(const std::vector<char> &data_type) : data_type_(CharToString(data_type)) {}
|
||||
std::string data_type_;
|
||||
};
|
||||
|
||||
std::shared_ptr<TensorOperation> TypeCast::Parse() { return std::make_shared<TypeCastOperation>(data_type_); }
|
||||
TypeCast::TypeCast(const std::vector<char> &data_type) : data_(std::make_shared<Data>(data_type)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> TypeCast::Parse() { return std::make_shared<TypeCastOperation>(data_->data_type_); }
|
||||
|
||||
// Constructor to Unique
|
||||
Unique::Unique() {}
|
||||
|
|
|
|||
|
|
@ -42,85 +42,153 @@ namespace vision {
|
|||
// CONSTRUCTORS FOR API CLASSES TO CREATE VISION TENSOR TRANSFORM OPERATIONS
|
||||
// (In alphabetical order)
|
||||
|
||||
// Affine Transform Operation.
|
||||
struct Affine::Data {
|
||||
Data(float_t degrees, const std::vector<float> &translation, float scale, const std::vector<float> &shear,
|
||||
InterpolationMode interpolation, const std::vector<uint8_t> &fill_value)
|
||||
: degrees_(degrees),
|
||||
translation_(translation),
|
||||
scale_(scale),
|
||||
shear_(shear),
|
||||
interpolation_(interpolation),
|
||||
fill_value_(fill_value) {}
|
||||
float degrees_;
|
||||
std::vector<float> translation_;
|
||||
float scale_;
|
||||
std::vector<float> shear_;
|
||||
InterpolationMode interpolation_;
|
||||
std::vector<uint8_t> fill_value_;
|
||||
};
|
||||
|
||||
Affine::Affine(float_t degrees, const std::vector<float> &translation, float scale, const std::vector<float> &shear,
|
||||
InterpolationMode interpolation, const std::vector<uint8_t> &fill_value)
|
||||
: degrees_(degrees),
|
||||
translation_(translation),
|
||||
scale_(scale),
|
||||
shear_(shear),
|
||||
interpolation_(interpolation),
|
||||
fill_value_(fill_value) {}
|
||||
: data_(std::make_shared<Data>(degrees, translation, scale, shear, interpolation, fill_value)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> Affine::Parse() {
|
||||
return std::make_shared<AffineOperation>(degrees_, translation_, scale_, shear_, interpolation_, fill_value_);
|
||||
return std::make_shared<AffineOperation>(data_->degrees_, data_->translation_, data_->scale_, data_->shear_,
|
||||
data_->interpolation_, data_->fill_value_);
|
||||
}
|
||||
|
||||
// AutoContrast Transform Operation.
|
||||
AutoContrast::AutoContrast(float cutoff, std::vector<uint32_t> ignore) : cutoff_(cutoff), ignore_(ignore) {}
|
||||
struct AutoContrast::Data {
|
||||
Data(float cutoff, const std::vector<uint32_t> &ignore) : cutoff_(cutoff), ignore_(ignore) {}
|
||||
float cutoff_;
|
||||
std::vector<uint32_t> ignore_;
|
||||
};
|
||||
|
||||
AutoContrast::AutoContrast(float cutoff, std::vector<uint32_t> ignore)
|
||||
: data_(std::make_shared<Data>(cutoff, ignore)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> AutoContrast::Parse() {
|
||||
return std::make_shared<AutoContrastOperation>(cutoff_, ignore_);
|
||||
return std::make_shared<AutoContrastOperation>(data_->cutoff_, data_->ignore_);
|
||||
}
|
||||
|
||||
// BoundingBoxAugment Transform Operation.
|
||||
BoundingBoxAugment::BoundingBoxAugment(TensorTransform *transform, float ratio) : ratio_(ratio) {
|
||||
transform_ = transform ? transform->Parse() : nullptr;
|
||||
struct BoundingBoxAugment::Data {
|
||||
std::shared_ptr<TensorOperation> transform_;
|
||||
float ratio_;
|
||||
};
|
||||
|
||||
BoundingBoxAugment::BoundingBoxAugment(TensorTransform *transform, float ratio) : data_(std::make_shared<Data>()) {
|
||||
data_->transform_ = transform ? transform->Parse() : nullptr;
|
||||
data_->ratio_ = ratio;
|
||||
}
|
||||
|
||||
BoundingBoxAugment::BoundingBoxAugment(const std::shared_ptr<TensorTransform> &transform, float ratio) : ratio_(ratio) {
|
||||
transform_ = transform ? transform->Parse() : nullptr;
|
||||
BoundingBoxAugment::BoundingBoxAugment(const std::shared_ptr<TensorTransform> &transform, float ratio)
|
||||
: data_(std::make_shared<Data>()) {
|
||||
data_->transform_ = transform ? transform->Parse() : nullptr;
|
||||
data_->ratio_ = ratio;
|
||||
}
|
||||
|
||||
BoundingBoxAugment::BoundingBoxAugment(const std::reference_wrapper<TensorTransform> transform, float ratio)
|
||||
: ratio_(ratio) {
|
||||
transform_ = transform.get().Parse();
|
||||
: data_(std::make_shared<Data>()) {
|
||||
data_->transform_ = transform.get().Parse();
|
||||
data_->ratio_ = ratio;
|
||||
}
|
||||
|
||||
std::shared_ptr<TensorOperation> BoundingBoxAugment::Parse() {
|
||||
return std::make_shared<BoundingBoxAugmentOperation>(transform_, ratio_);
|
||||
return std::make_shared<BoundingBoxAugmentOperation>(data_->transform_, data_->ratio_);
|
||||
}
|
||||
#endif // not ENABLE_ANDROID
|
||||
|
||||
// CenterCrop Transform Operation.
|
||||
CenterCrop::CenterCrop(std::vector<int32_t> size) : size_(size) {}
|
||||
struct CenterCrop::Data {
|
||||
explicit Data(const std::vector<int32_t> &size) : size_(size) {}
|
||||
std::vector<int32_t> size_;
|
||||
};
|
||||
|
||||
std::shared_ptr<TensorOperation> CenterCrop::Parse() { return std::make_shared<CenterCropOperation>(size_); }
|
||||
CenterCrop::CenterCrop(std::vector<int32_t> size) : data_(std::make_shared<Data>(size)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> CenterCrop::Parse() { return std::make_shared<CenterCropOperation>(data_->size_); }
|
||||
|
||||
std::shared_ptr<TensorOperation> CenterCrop::Parse(const MapTargetDevice &env) {
|
||||
if (env == MapTargetDevice::kAscend310) {
|
||||
#ifdef ENABLE_ACL
|
||||
std::vector<uint32_t> usize_;
|
||||
usize_.reserve(size_.size());
|
||||
std::transform(size_.begin(), size_.end(), std::back_inserter(usize_), [](int32_t i) { return (uint32_t)i; });
|
||||
usize_.reserve(data_->size_.size());
|
||||
std::transform(data_->size_.begin(), data_->size_.end(), std::back_inserter(usize_),
|
||||
[](int32_t i) { return (uint32_t)i; });
|
||||
return std::make_shared<DvppCropJpegOperation>(usize_);
|
||||
#endif // ENABLE_ACL
|
||||
}
|
||||
return std::make_shared<CenterCropOperation>(size_);
|
||||
return std::make_shared<CenterCropOperation>(data_->size_);
|
||||
}
|
||||
|
||||
// Crop Transform Operation.
|
||||
Crop::Crop(std::vector<int32_t> coordinates, std::vector<int32_t> size) : coordinates_(coordinates), size_(size) {}
|
||||
struct Crop::Data {
|
||||
Data(const std::vector<int32_t> &coordinates, const std::vector<int32_t> &size)
|
||||
: coordinates_(coordinates), size_(size) {}
|
||||
std::vector<int32_t> coordinates_;
|
||||
std::vector<int32_t> size_;
|
||||
};
|
||||
|
||||
std::shared_ptr<TensorOperation> Crop::Parse() { return std::make_shared<CropOperation>(coordinates_, size_); }
|
||||
Crop::Crop(std::vector<int32_t> coordinates, std::vector<int32_t> size)
|
||||
: data_(std::make_shared<Data>(coordinates, size)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> Crop::Parse() {
|
||||
return std::make_shared<CropOperation>(data_->coordinates_, data_->size_);
|
||||
}
|
||||
|
||||
#ifndef ENABLE_ANDROID
|
||||
// CutMixBatch Transform Operation.
|
||||
struct CutMixBatch::Data {
|
||||
Data(ImageBatchFormat image_batch_format, float alpha, float prob)
|
||||
: image_batch_format_(image_batch_format), alpha_(alpha), prob_(prob) {}
|
||||
float alpha_;
|
||||
float prob_;
|
||||
ImageBatchFormat image_batch_format_;
|
||||
};
|
||||
|
||||
CutMixBatch::CutMixBatch(ImageBatchFormat image_batch_format, float alpha, float prob)
|
||||
: image_batch_format_(image_batch_format), alpha_(alpha), prob_(prob) {}
|
||||
: data_(std::make_shared<Data>(image_batch_format, alpha, prob)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> CutMixBatch::Parse() {
|
||||
return std::make_shared<CutMixBatchOperation>(image_batch_format_, alpha_, prob_);
|
||||
return std::make_shared<CutMixBatchOperation>(data_->image_batch_format_, data_->alpha_, data_->prob_);
|
||||
}
|
||||
|
||||
// CutOutOp.
|
||||
CutOut::CutOut(int32_t length, int32_t num_patches) : length_(length), num_patches_(num_patches) {}
|
||||
struct CutOut::Data {
|
||||
Data(int32_t length, int32_t num_patches) : length_(length), num_patches_(num_patches) {}
|
||||
int32_t length_;
|
||||
int32_t num_patches_;
|
||||
};
|
||||
|
||||
std::shared_ptr<TensorOperation> CutOut::Parse() { return std::make_shared<CutOutOperation>(length_, num_patches_); }
|
||||
CutOut::CutOut(int32_t length, int32_t num_patches) : data_(std::make_shared<Data>(length, num_patches)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> CutOut::Parse() {
|
||||
return std::make_shared<CutOutOperation>(data_->length_, data_->num_patches_);
|
||||
}
|
||||
#endif // not ENABLE_ANDROID
|
||||
|
||||
// Decode Transform Operation.
|
||||
Decode::Decode(bool rgb) : rgb_(rgb) {}
|
||||
std::shared_ptr<TensorOperation> Decode::Parse() { return std::make_shared<DecodeOperation>(rgb_); }
|
||||
struct Decode::Data {
|
||||
explicit Data(bool rgb) : rgb_(rgb) {}
|
||||
bool rgb_;
|
||||
};
|
||||
|
||||
Decode::Decode(bool rgb) : data_(std::make_shared<Data>(rgb)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> Decode::Parse() { return std::make_shared<DecodeOperation>(data_->rgb_); }
|
||||
|
||||
std::shared_ptr<TensorOperation> Decode::Parse(const MapTargetDevice &env) {
|
||||
if (env == MapTargetDevice::kAscend310) {
|
||||
|
|
@ -128,31 +196,42 @@ std::shared_ptr<TensorOperation> Decode::Parse(const MapTargetDevice &env) {
|
|||
return std::make_shared<DvppDecodeJpegOperation>();
|
||||
#endif // ENABLE_ACL
|
||||
}
|
||||
return std::make_shared<DecodeOperation>(rgb_);
|
||||
return std::make_shared<DecodeOperation>(data_->rgb_);
|
||||
}
|
||||
|
||||
#ifdef ENABLE_ACL
|
||||
// DvppDecodeResize Transform Operation.
|
||||
DvppDecodeResizeJpeg::DvppDecodeResizeJpeg(std::vector<uint32_t> resize) : resize_(resize) {}
|
||||
struct DvppDecodeResizeJpeg::Data {
|
||||
explicit Data(const std::vector<uint32_t> &resize) : resize_(resize) {}
|
||||
std::vector<uint32_t> resize_;
|
||||
};
|
||||
|
||||
DvppDecodeResizeJpeg::DvppDecodeResizeJpeg(std::vector<uint32_t> resize) : data_(std::make_shared<Data>(resize)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> DvppDecodeResizeJpeg::Parse() {
|
||||
return std::make_shared<DvppDecodeResizeOperation>(resize_);
|
||||
return std::make_shared<DvppDecodeResizeOperation>(data_->resize_);
|
||||
}
|
||||
|
||||
std::shared_ptr<TensorOperation> DvppDecodeResizeJpeg::Parse(const MapTargetDevice &env) {
|
||||
return std::make_shared<DvppDecodeResizeOperation>(resize_);
|
||||
return std::make_shared<DvppDecodeResizeOperation>(data_->resize_);
|
||||
}
|
||||
|
||||
// DvppDecodeResizeCrop Transform Operation.
|
||||
struct DvppDecodeResizeCropJpeg::Data {
|
||||
Data(const std::vector<uint32_t> &crop, const std::vector<uint32_t> &resize) : crop_(crop), resize_(resize) {}
|
||||
std::vector<uint32_t> crop_;
|
||||
std::vector<uint32_t> resize_;
|
||||
};
|
||||
|
||||
DvppDecodeResizeCropJpeg::DvppDecodeResizeCropJpeg(std::vector<uint32_t> crop, std::vector<uint32_t> resize)
|
||||
: crop_(crop), resize_(resize) {}
|
||||
: data_(std::make_shared<Data>(crop, resize)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> DvppDecodeResizeCropJpeg::Parse() {
|
||||
return std::make_shared<DvppDecodeResizeCropOperation>(crop_, resize_);
|
||||
return std::make_shared<DvppDecodeResizeCropOperation>(data_->crop_, data_->resize_);
|
||||
}
|
||||
|
||||
std::shared_ptr<TensorOperation> DvppDecodeResizeCropJpeg::Parse(const MapTargetDevice &env) {
|
||||
return std::make_shared<DvppDecodeResizeCropOperation>(crop_, resize_);
|
||||
return std::make_shared<DvppDecodeResizeCropOperation>(data_->crop_, data_->resize_);
|
||||
}
|
||||
|
||||
// DvppDecodePng Transform Operation.
|
||||
|
|
@ -181,174 +260,339 @@ Invert::Invert() {}
|
|||
std::shared_ptr<TensorOperation> Invert::Parse() { return std::make_shared<InvertOperation>(); }
|
||||
|
||||
// MixUpBatch Transform Operation.
|
||||
MixUpBatch::MixUpBatch(float alpha) : alpha_(alpha) {}
|
||||
struct MixUpBatch::Data {
|
||||
explicit Data(float alpha) : alpha_(alpha) {}
|
||||
float alpha_;
|
||||
};
|
||||
|
||||
std::shared_ptr<TensorOperation> MixUpBatch::Parse() { return std::make_shared<MixUpBatchOperation>(alpha_); }
|
||||
MixUpBatch::MixUpBatch(float alpha) : data_(std::make_shared<Data>(alpha)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> MixUpBatch::Parse() { return std::make_shared<MixUpBatchOperation>(data_->alpha_); }
|
||||
#endif // not ENABLE_ANDROID
|
||||
|
||||
// Normalize Transform Operation.
|
||||
Normalize::Normalize(std::vector<float> mean, std::vector<float> std) : mean_(mean), std_(std) {}
|
||||
struct Normalize::Data {
|
||||
Data(const std::vector<float> &mean, const std::vector<float> &std) : mean_(mean), std_(std) {}
|
||||
std::vector<float> mean_;
|
||||
std::vector<float> std_;
|
||||
};
|
||||
|
||||
std::shared_ptr<TensorOperation> Normalize::Parse() { return std::make_shared<NormalizeOperation>(mean_, std_); }
|
||||
Normalize::Normalize(std::vector<float> mean, std::vector<float> std) : data_(std::make_shared<Data>(mean, std)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> Normalize::Parse() {
|
||||
return std::make_shared<NormalizeOperation>(data_->mean_, data_->std_);
|
||||
}
|
||||
|
||||
std::shared_ptr<TensorOperation> Normalize::Parse(const MapTargetDevice &env) {
|
||||
if (env == MapTargetDevice::kAscend310) {
|
||||
#ifdef ENABLE_ACL
|
||||
return std::make_shared<DvppNormalizeOperation>(mean_, std_);
|
||||
return std::make_shared<DvppNormalizeOperation>(data_->mean_, data_->std_);
|
||||
#endif
|
||||
}
|
||||
return std::make_shared<NormalizeOperation>(mean_, std_);
|
||||
return std::make_shared<NormalizeOperation>(data_->mean_, data_->std_);
|
||||
}
|
||||
|
||||
#ifndef ENABLE_ANDROID
|
||||
// NormalizePad Transform Operation.
|
||||
NormalizePad::NormalizePad(const std::vector<float> &mean, const std::vector<float> &std, const std::string &dtype)
|
||||
: mean_(mean), std_(std), dtype_(dtype) {}
|
||||
struct NormalizePad::Data {
|
||||
Data(const std::vector<float> &mean, const std::vector<float> &std, const std::string &dtype)
|
||||
: mean_(mean), std_(std), dtype_(dtype) {}
|
||||
std::vector<float> mean_;
|
||||
std::vector<float> std_;
|
||||
std::string dtype_;
|
||||
};
|
||||
|
||||
NormalizePad::NormalizePad(const std::vector<float> &mean, const std::vector<float> &std,
|
||||
const std::vector<char> &dtype)
|
||||
: data_(std::make_shared<Data>(mean, std, CharToString(dtype))) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> NormalizePad::Parse() {
|
||||
return std::make_shared<NormalizePadOperation>(mean_, std_, dtype_);
|
||||
return std::make_shared<NormalizePadOperation>(data_->mean_, data_->std_, data_->dtype_);
|
||||
}
|
||||
|
||||
// Pad Transform Operation.
|
||||
struct Pad::Data {
|
||||
Data(const std::vector<int32_t> &padding, const std::vector<uint8_t> &fill_value, BorderType padding_mode)
|
||||
: padding_(padding), fill_value_(fill_value), padding_mode_(padding_mode) {}
|
||||
std::vector<int32_t> padding_;
|
||||
std::vector<uint8_t> fill_value_;
|
||||
BorderType padding_mode_;
|
||||
};
|
||||
|
||||
Pad::Pad(std::vector<int32_t> padding, std::vector<uint8_t> fill_value, BorderType padding_mode)
|
||||
: padding_(padding), fill_value_(fill_value), padding_mode_(padding_mode) {}
|
||||
: data_(std::make_shared<Data>(padding, fill_value, padding_mode)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> Pad::Parse() {
|
||||
return std::make_shared<PadOperation>(padding_, fill_value_, padding_mode_);
|
||||
return std::make_shared<PadOperation>(data_->padding_, data_->fill_value_, data_->padding_mode_);
|
||||
}
|
||||
|
||||
// RandomAffine Transform Operation.
|
||||
struct RandomAffine::Data {
|
||||
Data(const std::vector<float_t> °rees, const std::vector<float_t> &translate_range,
|
||||
const std::vector<float_t> &scale_range, const std::vector<float_t> &shear_ranges,
|
||||
InterpolationMode interpolation, const std::vector<uint8_t> &fill_value)
|
||||
: degrees_(degrees),
|
||||
translate_range_(translate_range),
|
||||
scale_range_(scale_range),
|
||||
shear_ranges_(shear_ranges),
|
||||
interpolation_(interpolation),
|
||||
fill_value_(fill_value) {}
|
||||
std::vector<float_t> degrees_; // min_degree, max_degree
|
||||
std::vector<float_t> translate_range_; // maximum x translation percentage, maximum y translation percentage
|
||||
std::vector<float_t> scale_range_; // min_scale, max_scale
|
||||
std::vector<float_t> shear_ranges_; // min_x_shear, max_x_shear, min_y_shear, max_y_shear
|
||||
InterpolationMode interpolation_;
|
||||
std::vector<uint8_t> fill_value_;
|
||||
};
|
||||
|
||||
RandomAffine::RandomAffine(const std::vector<float_t> °rees, const std::vector<float_t> &translate_range,
|
||||
const std::vector<float_t> &scale_range, const std::vector<float_t> &shear_ranges,
|
||||
InterpolationMode interpolation, const std::vector<uint8_t> &fill_value)
|
||||
: degrees_(degrees),
|
||||
translate_range_(translate_range),
|
||||
scale_range_(scale_range),
|
||||
shear_ranges_(shear_ranges),
|
||||
interpolation_(interpolation),
|
||||
fill_value_(fill_value) {}
|
||||
: data_(std::make_shared<Data>(degrees, translate_range, scale_range, shear_ranges, interpolation, fill_value)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomAffine::Parse() {
|
||||
return std::make_shared<RandomAffineOperation>(degrees_, translate_range_, scale_range_, shear_ranges_,
|
||||
interpolation_, fill_value_);
|
||||
return std::make_shared<RandomAffineOperation>(data_->degrees_, data_->translate_range_, data_->scale_range_,
|
||||
data_->shear_ranges_, data_->interpolation_, data_->fill_value_);
|
||||
}
|
||||
|
||||
// RandomColor Transform Operation.
|
||||
RandomColor::RandomColor(float t_lb, float t_ub) : t_lb_(t_lb), t_ub_(t_ub) {}
|
||||
struct RandomColor::Data {
|
||||
Data(float t_lb, float t_ub) : t_lb_(t_lb), t_ub_(t_ub) {}
|
||||
float t_lb_;
|
||||
float t_ub_;
|
||||
};
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomColor::Parse() { return std::make_shared<RandomColorOperation>(t_lb_, t_ub_); }
|
||||
RandomColor::RandomColor(float t_lb, float t_ub) : data_(std::make_shared<Data>(t_lb, t_ub)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomColor::Parse() {
|
||||
return std::make_shared<RandomColorOperation>(data_->t_lb_, data_->t_ub_);
|
||||
}
|
||||
|
||||
// RandomColorAdjust Transform Operation.
|
||||
struct RandomColorAdjust::Data {
|
||||
Data(const std::vector<float> &brightness, const std::vector<float> &contrast, const std::vector<float> &saturation,
|
||||
const std::vector<float> &hue)
|
||||
: brightness_(brightness), contrast_(contrast), saturation_(saturation), hue_(hue) {}
|
||||
std::vector<float> brightness_;
|
||||
std::vector<float> contrast_;
|
||||
std::vector<float> saturation_;
|
||||
std::vector<float> hue_;
|
||||
};
|
||||
|
||||
RandomColorAdjust::RandomColorAdjust(std::vector<float> brightness, std::vector<float> contrast,
|
||||
std::vector<float> saturation, std::vector<float> hue)
|
||||
: brightness_(brightness), contrast_(contrast), saturation_(saturation), hue_(hue) {}
|
||||
: data_(std::make_shared<Data>(brightness, contrast, saturation, hue)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomColorAdjust::Parse() {
|
||||
return std::make_shared<RandomColorAdjustOperation>(brightness_, contrast_, saturation_, hue_);
|
||||
return std::make_shared<RandomColorAdjustOperation>(data_->brightness_, data_->contrast_, data_->saturation_,
|
||||
data_->hue_);
|
||||
}
|
||||
|
||||
// RandomCrop Transform Operation.
|
||||
struct RandomCrop::Data {
|
||||
Data(const std::vector<int32_t> &size, const std::vector<int32_t> &padding, bool pad_if_needed,
|
||||
const std::vector<uint8_t> &fill_value, BorderType padding_mode)
|
||||
: size_(size),
|
||||
padding_(padding),
|
||||
pad_if_needed_(pad_if_needed),
|
||||
fill_value_(fill_value),
|
||||
padding_mode_(padding_mode) {}
|
||||
std::vector<int32_t> size_;
|
||||
std::vector<int32_t> padding_;
|
||||
bool pad_if_needed_;
|
||||
std::vector<uint8_t> fill_value_;
|
||||
BorderType padding_mode_;
|
||||
};
|
||||
|
||||
RandomCrop::RandomCrop(std::vector<int32_t> size, std::vector<int32_t> padding, bool pad_if_needed,
|
||||
std::vector<uint8_t> fill_value, BorderType padding_mode)
|
||||
: size_(size),
|
||||
padding_(padding),
|
||||
pad_if_needed_(pad_if_needed),
|
||||
fill_value_(fill_value),
|
||||
padding_mode_(padding_mode) {}
|
||||
: data_(std::make_shared<Data>(size, padding, pad_if_needed, fill_value, padding_mode)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomCrop::Parse() {
|
||||
return std::make_shared<RandomCropOperation>(size_, padding_, pad_if_needed_, fill_value_, padding_mode_);
|
||||
return std::make_shared<RandomCropOperation>(data_->size_, data_->padding_, data_->pad_if_needed_, data_->fill_value_,
|
||||
data_->padding_mode_);
|
||||
}
|
||||
|
||||
// RandomCropDecodeResize Transform Operation.
|
||||
struct RandomCropDecodeResize::Data {
|
||||
Data(const std::vector<int32_t> &size, const std::vector<float> &scale, const std::vector<float> &ratio,
|
||||
InterpolationMode interpolation, int32_t max_attempts)
|
||||
: size_(size), scale_(scale), ratio_(ratio), interpolation_(interpolation), max_attempts_(max_attempts) {}
|
||||
std::vector<int32_t> size_;
|
||||
std::vector<float> scale_;
|
||||
std::vector<float> ratio_;
|
||||
InterpolationMode interpolation_;
|
||||
int32_t max_attempts_;
|
||||
};
|
||||
|
||||
RandomCropDecodeResize::RandomCropDecodeResize(std::vector<int32_t> size, std::vector<float> scale,
|
||||
std::vector<float> ratio, InterpolationMode interpolation,
|
||||
int32_t max_attempts)
|
||||
: size_(size), scale_(scale), ratio_(ratio), interpolation_(interpolation), max_attempts_(max_attempts) {}
|
||||
: data_(std::make_shared<Data>(size, scale, ratio, interpolation, max_attempts)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomCropDecodeResize::Parse() {
|
||||
return std::make_shared<RandomCropDecodeResizeOperation>(size_, scale_, ratio_, interpolation_, max_attempts_);
|
||||
return std::make_shared<RandomCropDecodeResizeOperation>(data_->size_, data_->scale_, data_->ratio_,
|
||||
data_->interpolation_, data_->max_attempts_);
|
||||
}
|
||||
|
||||
// RandomCropWithBBox Transform Operation.
|
||||
struct RandomCropWithBBox::Data {
|
||||
Data(const std::vector<int32_t> &size, const std::vector<int32_t> &padding, bool pad_if_needed,
|
||||
const std::vector<uint8_t> &fill_value, BorderType padding_mode)
|
||||
: size_(size),
|
||||
padding_(padding),
|
||||
pad_if_needed_(pad_if_needed),
|
||||
fill_value_(fill_value),
|
||||
padding_mode_(padding_mode) {}
|
||||
std::vector<int32_t> size_;
|
||||
std::vector<int32_t> padding_;
|
||||
bool pad_if_needed_;
|
||||
std::vector<uint8_t> fill_value_;
|
||||
BorderType padding_mode_;
|
||||
};
|
||||
|
||||
RandomCropWithBBox::RandomCropWithBBox(std::vector<int32_t> size, std::vector<int32_t> padding, bool pad_if_needed,
|
||||
std::vector<uint8_t> fill_value, BorderType padding_mode)
|
||||
: size_(size),
|
||||
padding_(padding),
|
||||
pad_if_needed_(pad_if_needed),
|
||||
fill_value_(fill_value),
|
||||
padding_mode_(padding_mode) {}
|
||||
: data_(std::make_shared<Data>(size, padding, pad_if_needed, fill_value, padding_mode)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomCropWithBBox::Parse() {
|
||||
return std::make_shared<RandomCropWithBBoxOperation>(size_, padding_, pad_if_needed_, fill_value_, padding_mode_);
|
||||
return std::make_shared<RandomCropWithBBoxOperation>(data_->size_, data_->padding_, data_->pad_if_needed_,
|
||||
data_->fill_value_, data_->padding_mode_);
|
||||
}
|
||||
|
||||
// RandomHorizontalFlip.
|
||||
RandomHorizontalFlip::RandomHorizontalFlip(float prob) : probability_(prob) {}
|
||||
struct RandomHorizontalFlip::Data {
|
||||
explicit Data(float prob) : probability_(prob) {}
|
||||
float probability_;
|
||||
};
|
||||
|
||||
RandomHorizontalFlip::RandomHorizontalFlip(float prob) : data_(std::make_shared<Data>(prob)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomHorizontalFlip::Parse() {
|
||||
return std::make_shared<RandomHorizontalFlipOperation>(probability_);
|
||||
return std::make_shared<RandomHorizontalFlipOperation>(data_->probability_);
|
||||
}
|
||||
|
||||
// RandomHorizontalFlipWithBBox
|
||||
RandomHorizontalFlipWithBBox::RandomHorizontalFlipWithBBox(float prob) : probability_(prob) {}
|
||||
struct RandomHorizontalFlipWithBBox::Data {
|
||||
explicit Data(float prob) : probability_(prob) {}
|
||||
float probability_;
|
||||
};
|
||||
|
||||
RandomHorizontalFlipWithBBox::RandomHorizontalFlipWithBBox(float prob) : data_(std::make_shared<Data>(prob)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomHorizontalFlipWithBBox::Parse() {
|
||||
return std::make_shared<RandomHorizontalFlipWithBBoxOperation>(probability_);
|
||||
return std::make_shared<RandomHorizontalFlipWithBBoxOperation>(data_->probability_);
|
||||
}
|
||||
|
||||
// RandomPosterize Transform Operation.
|
||||
RandomPosterize::RandomPosterize(const std::vector<uint8_t> &bit_range) : bit_range_(bit_range) {}
|
||||
struct RandomPosterize::Data {
|
||||
explicit Data(const std::vector<uint8_t> &bit_range) : bit_range_(bit_range) {}
|
||||
std::vector<uint8_t> bit_range_;
|
||||
};
|
||||
|
||||
RandomPosterize::RandomPosterize(const std::vector<uint8_t> &bit_range) : data_(std::make_shared<Data>(bit_range)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomPosterize::Parse() {
|
||||
return std::make_shared<RandomPosterizeOperation>(bit_range_);
|
||||
return std::make_shared<RandomPosterizeOperation>(data_->bit_range_);
|
||||
}
|
||||
|
||||
// RandomResize Transform Operation.
|
||||
RandomResize::RandomResize(std::vector<int32_t> size) : size_(size) {}
|
||||
struct RandomResize::Data {
|
||||
explicit Data(const std::vector<int32_t> &size) : size_(size) {}
|
||||
std::vector<int32_t> size_;
|
||||
};
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomResize::Parse() { return std::make_shared<RandomResizeOperation>(size_); }
|
||||
RandomResize::RandomResize(std::vector<int32_t> size) : data_(std::make_shared<Data>(size)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomResize::Parse() { return std::make_shared<RandomResizeOperation>(data_->size_); }
|
||||
|
||||
// RandomResizeWithBBox Transform Operation.
|
||||
RandomResizeWithBBox::RandomResizeWithBBox(std::vector<int32_t> size) : size_(size) {}
|
||||
struct RandomResizeWithBBox::Data {
|
||||
explicit Data(const std::vector<int32_t> &size) : size_(size) {}
|
||||
std::vector<int32_t> size_;
|
||||
};
|
||||
|
||||
RandomResizeWithBBox::RandomResizeWithBBox(std::vector<int32_t> size) : data_(std::make_shared<Data>(size)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomResizeWithBBox::Parse() {
|
||||
return std::make_shared<RandomResizeWithBBoxOperation>(size_);
|
||||
return std::make_shared<RandomResizeWithBBoxOperation>(data_->size_);
|
||||
}
|
||||
|
||||
// RandomResizedCrop Transform Operation.
|
||||
struct RandomResizedCrop::Data {
|
||||
Data(const std::vector<int32_t> &size, const std::vector<float> &scale, const std::vector<float> &ratio,
|
||||
InterpolationMode interpolation, int32_t max_attempts)
|
||||
: size_(size), scale_(scale), ratio_(ratio), interpolation_(interpolation), max_attempts_(max_attempts) {}
|
||||
std::vector<int32_t> size_;
|
||||
std::vector<float> scale_;
|
||||
std::vector<float> ratio_;
|
||||
InterpolationMode interpolation_;
|
||||
int32_t max_attempts_;
|
||||
};
|
||||
|
||||
RandomResizedCrop::RandomResizedCrop(std::vector<int32_t> size, std::vector<float> scale, std::vector<float> ratio,
|
||||
InterpolationMode interpolation, int32_t max_attempts)
|
||||
: size_(size), scale_(scale), ratio_(ratio), interpolation_(interpolation), max_attempts_(max_attempts) {}
|
||||
: data_(std::make_shared<Data>(size, scale, ratio, interpolation, max_attempts)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomResizedCrop::Parse() {
|
||||
return std::make_shared<RandomResizedCropOperation>(size_, scale_, ratio_, interpolation_, max_attempts_);
|
||||
return std::make_shared<RandomResizedCropOperation>(data_->size_, data_->scale_, data_->ratio_, data_->interpolation_,
|
||||
data_->max_attempts_);
|
||||
}
|
||||
|
||||
// RandomResizedCrop Transform Operation.
|
||||
struct RandomResizedCropWithBBox::Data {
|
||||
Data(const std::vector<int32_t> &size, const std::vector<float> &scale, const std::vector<float> &ratio,
|
||||
InterpolationMode interpolation, int32_t max_attempts)
|
||||
: size_(size), scale_(scale), ratio_(ratio), interpolation_(interpolation), max_attempts_(max_attempts) {}
|
||||
std::vector<int32_t> size_;
|
||||
std::vector<float> scale_;
|
||||
std::vector<float> ratio_;
|
||||
InterpolationMode interpolation_;
|
||||
int32_t max_attempts_;
|
||||
};
|
||||
|
||||
RandomResizedCropWithBBox::RandomResizedCropWithBBox(std::vector<int32_t> size, std::vector<float> scale,
|
||||
std::vector<float> ratio, InterpolationMode interpolation,
|
||||
int32_t max_attempts)
|
||||
: size_(size), scale_(scale), ratio_(ratio), interpolation_(interpolation), max_attempts_(max_attempts) {}
|
||||
: data_(std::make_shared<Data>(size, scale, ratio, interpolation, max_attempts)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomResizedCropWithBBox::Parse() {
|
||||
return std::make_shared<RandomResizedCropWithBBoxOperation>(size_, scale_, ratio_, interpolation_, max_attempts_);
|
||||
return std::make_shared<RandomResizedCropWithBBoxOperation>(data_->size_, data_->scale_, data_->ratio_,
|
||||
data_->interpolation_, data_->max_attempts_);
|
||||
}
|
||||
|
||||
// RandomRotation Transform Operation.
|
||||
struct RandomRotation::Data {
|
||||
Data(const std::vector<float> °rees, InterpolationMode interpolation_mode, bool expand,
|
||||
const std::vector<float> ¢er, const std::vector<uint8_t> &fill_value)
|
||||
: degrees_(degrees),
|
||||
interpolation_mode_(interpolation_mode),
|
||||
expand_(expand),
|
||||
center_(center),
|
||||
fill_value_(fill_value) {}
|
||||
std::vector<float> degrees_;
|
||||
InterpolationMode interpolation_mode_;
|
||||
std::vector<float> center_;
|
||||
bool expand_;
|
||||
std::vector<uint8_t> fill_value_;
|
||||
};
|
||||
|
||||
RandomRotation::RandomRotation(std::vector<float> degrees, InterpolationMode interpolation_mode, bool expand,
|
||||
std::vector<float> center, std::vector<uint8_t> fill_value)
|
||||
: degrees_(degrees),
|
||||
interpolation_mode_(interpolation_mode),
|
||||
expand_(expand),
|
||||
center_(center),
|
||||
fill_value_(fill_value) {}
|
||||
: data_(std::make_shared<Data>(degrees, interpolation_mode, expand, center, fill_value)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomRotation::Parse() {
|
||||
return std::make_shared<RandomRotationOperation>(degrees_, interpolation_mode_, expand_, center_, fill_value_);
|
||||
return std::make_shared<RandomRotationOperation>(data_->degrees_, data_->interpolation_mode_, data_->expand_,
|
||||
data_->center_, data_->fill_value_);
|
||||
}
|
||||
|
||||
// RandomSelectSubpolicy Transform Operation.
|
||||
RandomSelectSubpolicy::RandomSelectSubpolicy(std::vector<std::vector<std::pair<TensorTransform *, double>>> policy) {
|
||||
struct RandomSelectSubpolicy::Data {
|
||||
std::vector<std::vector<std::pair<std::shared_ptr<TensorOperation>, double>>> policy_;
|
||||
};
|
||||
|
||||
RandomSelectSubpolicy::RandomSelectSubpolicy(std::vector<std::vector<std::pair<TensorTransform *, double>>> policy)
|
||||
: data_(std::make_shared<Data>()) {
|
||||
for (int32_t i = 0; i < policy.size(); i++) {
|
||||
std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy;
|
||||
|
||||
|
|
@ -358,12 +602,13 @@ RandomSelectSubpolicy::RandomSelectSubpolicy(std::vector<std::vector<std::pair<T
|
|||
double prob = policy[i][j].second;
|
||||
subpolicy.emplace_back(std::move(std::make_pair(operation, prob)));
|
||||
}
|
||||
policy_.emplace_back(subpolicy);
|
||||
data_->policy_.emplace_back(subpolicy);
|
||||
}
|
||||
}
|
||||
|
||||
RandomSelectSubpolicy::RandomSelectSubpolicy(
|
||||
std::vector<std::vector<std::pair<std::shared_ptr<TensorTransform>, double>>> policy) {
|
||||
std::vector<std::vector<std::pair<std::shared_ptr<TensorTransform>, double>>> policy)
|
||||
: data_(std::make_shared<Data>()) {
|
||||
for (int32_t i = 0; i < policy.size(); i++) {
|
||||
std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy;
|
||||
|
||||
|
|
@ -373,12 +618,13 @@ RandomSelectSubpolicy::RandomSelectSubpolicy(
|
|||
double prob = policy[i][j].second;
|
||||
subpolicy.emplace_back(std::move(std::make_pair(operation, prob)));
|
||||
}
|
||||
policy_.emplace_back(subpolicy);
|
||||
data_->policy_.emplace_back(subpolicy);
|
||||
}
|
||||
}
|
||||
|
||||
RandomSelectSubpolicy::RandomSelectSubpolicy(
|
||||
std::vector<std::vector<std::pair<std::reference_wrapper<TensorTransform>, double>>> policy) {
|
||||
std::vector<std::vector<std::pair<std::reference_wrapper<TensorTransform>, double>>> policy)
|
||||
: data_(std::make_shared<Data>()) {
|
||||
for (int32_t i = 0; i < policy.size(); i++) {
|
||||
std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy;
|
||||
|
||||
|
|
@ -388,64 +634,102 @@ RandomSelectSubpolicy::RandomSelectSubpolicy(
|
|||
double prob = policy[i][j].second;
|
||||
subpolicy.emplace_back(std::move(std::make_pair(operation, prob)));
|
||||
}
|
||||
policy_.emplace_back(subpolicy);
|
||||
data_->policy_.emplace_back(subpolicy);
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomSelectSubpolicy::Parse() {
|
||||
return std::make_shared<RandomSelectSubpolicyOperation>(policy_);
|
||||
return std::make_shared<RandomSelectSubpolicyOperation>(data_->policy_);
|
||||
}
|
||||
|
||||
// RandomSharpness Transform Operation.
|
||||
RandomSharpness::RandomSharpness(std::vector<float> degrees) : degrees_(degrees) {}
|
||||
struct RandomSharpness::Data {
|
||||
explicit Data(const std::vector<float> °rees) : degrees_(degrees) {}
|
||||
std::vector<float> degrees_;
|
||||
};
|
||||
|
||||
RandomSharpness::RandomSharpness(std::vector<float> degrees) : data_(std::make_shared<Data>(degrees)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomSharpness::Parse() {
|
||||
return std::make_shared<RandomSharpnessOperation>(degrees_);
|
||||
return std::make_shared<RandomSharpnessOperation>(data_->degrees_);
|
||||
}
|
||||
|
||||
// RandomSolarize Transform Operation.
|
||||
RandomSolarize::RandomSolarize(std::vector<uint8_t> threshold) : threshold_(threshold) {}
|
||||
struct RandomSolarize::Data {
|
||||
explicit Data(const std::vector<uint8_t> &threshold) : threshold_(threshold) {}
|
||||
std::vector<uint8_t> threshold_;
|
||||
};
|
||||
|
||||
RandomSolarize::RandomSolarize(std::vector<uint8_t> threshold) : data_(std::make_shared<Data>(threshold)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomSolarize::Parse() {
|
||||
return std::make_shared<RandomSolarizeOperation>(threshold_);
|
||||
return std::make_shared<RandomSolarizeOperation>(data_->threshold_);
|
||||
}
|
||||
|
||||
// RandomVerticalFlip Transform Operation.
|
||||
RandomVerticalFlip::RandomVerticalFlip(float prob) : probability_(prob) {}
|
||||
struct RandomVerticalFlip::Data {
|
||||
explicit Data(float prob) : probability_(prob) {}
|
||||
float probability_;
|
||||
};
|
||||
|
||||
RandomVerticalFlip::RandomVerticalFlip(float prob) : data_(std::make_shared<Data>(prob)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomVerticalFlip::Parse() {
|
||||
return std::make_shared<RandomVerticalFlipOperation>(probability_);
|
||||
return std::make_shared<RandomVerticalFlipOperation>(data_->probability_);
|
||||
}
|
||||
|
||||
// RandomVerticalFlipWithBBox Transform Operation.
|
||||
RandomVerticalFlipWithBBox::RandomVerticalFlipWithBBox(float prob) : probability_(prob) {}
|
||||
struct RandomVerticalFlipWithBBox::Data {
|
||||
explicit Data(float prob) : probability_(prob) {}
|
||||
float probability_;
|
||||
};
|
||||
|
||||
RandomVerticalFlipWithBBox::RandomVerticalFlipWithBBox(float prob) : data_(std::make_shared<Data>(prob)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> RandomVerticalFlipWithBBox::Parse() {
|
||||
return std::make_shared<RandomVerticalFlipWithBBoxOperation>(probability_);
|
||||
return std::make_shared<RandomVerticalFlipWithBBoxOperation>(data_->probability_);
|
||||
}
|
||||
|
||||
// Rescale Transform Operation.
|
||||
Rescale::Rescale(float rescale, float shift) : rescale_(rescale), shift_(shift) {}
|
||||
struct Rescale::Data {
|
||||
Data(float rescale, float shift) : rescale_(rescale), shift_(shift) {}
|
||||
float rescale_;
|
||||
float shift_;
|
||||
};
|
||||
|
||||
std::shared_ptr<TensorOperation> Rescale::Parse() { return std::make_shared<RescaleOperation>(rescale_, shift_); }
|
||||
Rescale::Rescale(float rescale, float shift) : data_(std::make_shared<Data>(rescale, shift)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> Rescale::Parse() {
|
||||
return std::make_shared<RescaleOperation>(data_->rescale_, data_->shift_);
|
||||
}
|
||||
#endif // not ENABLE_ANDROID
|
||||
|
||||
// Resize Transform Operation.
|
||||
Resize::Resize(std::vector<int32_t> size, InterpolationMode interpolation)
|
||||
: size_(size), interpolation_(interpolation) {}
|
||||
struct Resize::Data {
|
||||
Data(const std::vector<int32_t> &size, InterpolationMode interpolation)
|
||||
: size_(size), interpolation_(interpolation) {}
|
||||
std::vector<int32_t> size_;
|
||||
InterpolationMode interpolation_;
|
||||
};
|
||||
|
||||
std::shared_ptr<TensorOperation> Resize::Parse() { return std::make_shared<ResizeOperation>(size_, interpolation_); }
|
||||
Resize::Resize(std::vector<int32_t> size, InterpolationMode interpolation)
|
||||
: data_(std::make_shared<Data>(size, interpolation)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> Resize::Parse() {
|
||||
return std::make_shared<ResizeOperation>(data_->size_, data_->interpolation_);
|
||||
}
|
||||
|
||||
std::shared_ptr<TensorOperation> Resize::Parse(const MapTargetDevice &env) {
|
||||
if (env == MapTargetDevice::kAscend310) {
|
||||
#ifdef ENABLE_ACL
|
||||
std::vector<uint32_t> usize_;
|
||||
usize_.reserve(size_.size());
|
||||
std::transform(size_.begin(), size_.end(), std::back_inserter(usize_), [](int32_t i) { return (uint32_t)i; });
|
||||
usize_.reserve(data_->size_.size());
|
||||
std::transform(data_->size_.begin(), data_->size_.end(), std::back_inserter(usize_),
|
||||
[](int32_t i) { return (uint32_t)i; });
|
||||
return std::make_shared<DvppResizeJpegOperation>(usize_);
|
||||
#endif // ENABLE_ACL
|
||||
}
|
||||
return std::make_shared<ResizeOperation>(size_, interpolation_);
|
||||
return std::make_shared<ResizeOperation>(data_->size_, data_->interpolation_);
|
||||
}
|
||||
|
||||
#ifdef ENABLE_ANDROID
|
||||
|
|
@ -457,11 +741,18 @@ std::shared_ptr<TensorOperation> Rotate::Parse() { return std::make_shared<Rotat
|
|||
|
||||
#ifndef ENABLE_ANDROID
|
||||
// ResizeWithBBox Transform Operation.
|
||||
struct ResizeWithBBox::Data {
|
||||
Data(const std::vector<int32_t> &size, InterpolationMode interpolation)
|
||||
: size_(size), interpolation_(interpolation) {}
|
||||
std::vector<int32_t> size_;
|
||||
InterpolationMode interpolation_;
|
||||
};
|
||||
|
||||
ResizeWithBBox::ResizeWithBBox(std::vector<int32_t> size, InterpolationMode interpolation)
|
||||
: size_(size), interpolation_(interpolation) {}
|
||||
: data_(std::make_shared<Data>(size, interpolation)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> ResizeWithBBox::Parse() {
|
||||
return std::make_shared<ResizeWithBBoxOperation>(size_, interpolation_);
|
||||
return std::make_shared<ResizeWithBBoxOperation>(data_->size_, data_->interpolation_);
|
||||
}
|
||||
|
||||
// RgbaToBgr Transform Operation.
|
||||
|
|
@ -475,19 +766,36 @@ RGBA2RGB::RGBA2RGB() {}
|
|||
std::shared_ptr<TensorOperation> RGBA2RGB::Parse() { return std::make_shared<RgbaToRgbOperation>(); }
|
||||
|
||||
// SoftDvppDecodeRandomCropResizeJpeg Transform Operation.
|
||||
struct SoftDvppDecodeRandomCropResizeJpeg::Data {
|
||||
Data(const std::vector<int32_t> &size, const std::vector<float> &scale, const std::vector<float> &ratio,
|
||||
int32_t max_attempts)
|
||||
: size_(size), scale_(scale), ratio_(ratio), max_attempts_(max_attempts) {}
|
||||
std::vector<int32_t> size_;
|
||||
std::vector<float> scale_;
|
||||
std::vector<float> ratio_;
|
||||
int32_t max_attempts_;
|
||||
};
|
||||
|
||||
SoftDvppDecodeRandomCropResizeJpeg::SoftDvppDecodeRandomCropResizeJpeg(std::vector<int32_t> size,
|
||||
std::vector<float> scale,
|
||||
std::vector<float> ratio, int32_t max_attempts)
|
||||
: size_(size), scale_(scale), ratio_(ratio), max_attempts_(max_attempts) {}
|
||||
: data_(std::make_shared<Data>(size, scale, ratio, max_attempts)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> SoftDvppDecodeRandomCropResizeJpeg::Parse() {
|
||||
return std::make_shared<SoftDvppDecodeRandomCropResizeJpegOperation>(size_, scale_, ratio_, max_attempts_);
|
||||
return std::make_shared<SoftDvppDecodeRandomCropResizeJpegOperation>(data_->size_, data_->scale_, data_->ratio_,
|
||||
data_->max_attempts_);
|
||||
}
|
||||
|
||||
// SoftDvppDecodeResizeJpeg Transform Operation.
|
||||
SoftDvppDecodeResizeJpeg::SoftDvppDecodeResizeJpeg(std::vector<int32_t> size) : size_(size) {}
|
||||
struct SoftDvppDecodeResizeJpeg::Data {
|
||||
explicit Data(const std::vector<int32_t> &size) : size_(size) {}
|
||||
std::vector<int32_t> size_;
|
||||
};
|
||||
|
||||
SoftDvppDecodeResizeJpeg::SoftDvppDecodeResizeJpeg(std::vector<int32_t> size) : data_(std::make_shared<Data>(size)) {}
|
||||
|
||||
std::shared_ptr<TensorOperation> SoftDvppDecodeResizeJpeg::Parse() {
|
||||
return std::make_shared<SoftDvppDecodeResizeJpegOperation>(size_);
|
||||
return std::make_shared<SoftDvppDecodeResizeJpegOperation>(data_->size_);
|
||||
}
|
||||
|
||||
// SwapRedBlue Transform Operation.
|
||||
|
|
@ -496,27 +804,36 @@ SwapRedBlue::SwapRedBlue() {}
|
|||
std::shared_ptr<TensorOperation> SwapRedBlue::Parse() { return std::make_shared<SwapRedBlueOperation>(); }
|
||||
|
||||
// UniformAug Transform Operation.
|
||||
UniformAugment::UniformAugment(const std::vector<TensorTransform *> &transforms, int32_t num_ops) : num_ops_(num_ops) {
|
||||
struct UniformAugment::Data {
|
||||
std::vector<std::shared_ptr<TensorOperation>> transforms_;
|
||||
int32_t num_ops_;
|
||||
};
|
||||
|
||||
UniformAugment::UniformAugment(const std::vector<TensorTransform *> &transforms, int32_t num_ops)
|
||||
: data_(std::make_shared<Data>()) {
|
||||
(void)std::transform(
|
||||
transforms.begin(), transforms.end(), std::back_inserter(transforms_),
|
||||
transforms.begin(), transforms.end(), std::back_inserter(data_->transforms_),
|
||||
[](TensorTransform *op) -> std::shared_ptr<TensorOperation> { return op ? op->Parse() : nullptr; });
|
||||
data_->num_ops_ = num_ops;
|
||||
}
|
||||
|
||||
UniformAugment::UniformAugment(const std::vector<std::shared_ptr<TensorTransform>> &transforms, int32_t num_ops)
|
||||
: num_ops_(num_ops) {
|
||||
: data_(std::make_shared<Data>()) {
|
||||
(void)std::transform(
|
||||
transforms.begin(), transforms.end(), std::back_inserter(transforms_),
|
||||
transforms.begin(), transforms.end(), std::back_inserter(data_->transforms_),
|
||||
[](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> { return op ? op->Parse() : nullptr; });
|
||||
data_->num_ops_ = num_ops;
|
||||
}
|
||||
|
||||
UniformAugment::UniformAugment(const std::vector<std::reference_wrapper<TensorTransform>> &transforms, int32_t num_ops)
|
||||
: num_ops_(num_ops) {
|
||||
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
|
||||
: data_(std::make_shared<Data>()) {
|
||||
(void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(data_->transforms_),
|
||||
[](TensorTransform &op) -> std::shared_ptr<TensorOperation> { return op.Parse(); });
|
||||
data_->num_ops_ = num_ops;
|
||||
}
|
||||
|
||||
std::shared_ptr<TensorOperation> UniformAugment::Parse() {
|
||||
return std::make_shared<UniformAugOperation>(transforms_, num_ops_);
|
||||
return std::make_shared<UniformAugOperation>(data_->transforms_, data_->num_ops_);
|
||||
}
|
||||
#endif // not ENABLE_ANDROID
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@
|
|||
#include <string>
|
||||
#include <optional>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "include/api/dual_abi_helper.h"
|
||||
#include "minddata/dataset/engine/cache/cache_client.h"
|
||||
#include "minddata/dataset/engine/datasetops/cache_op.h"
|
||||
#include "minddata/dataset/engine/ir/cache/dataset_cache.h"
|
||||
|
|
@ -39,13 +41,13 @@ class DatasetCacheImpl : public DatasetCache {
|
|||
/// \param port optional port (default=50052).
|
||||
/// \param num_connections optional number of connections (default=12).
|
||||
/// \param prefetch_sz optional prefetch size (default=20).
|
||||
DatasetCacheImpl(session_id_type id, uint64_t mem_sz, bool spill, std::optional<std::string> hostname,
|
||||
DatasetCacheImpl(session_id_type id, uint64_t mem_sz, bool spill, std::optional<std::vector<char>> hostname,
|
||||
std::optional<int32_t> port, std::optional<int32_t> num_connections,
|
||||
std::optional<int32_t> prefetch_sz)
|
||||
: session_id_(id),
|
||||
cache_mem_sz_(mem_sz),
|
||||
spill_(spill),
|
||||
hostname_(std::move(hostname)),
|
||||
hostname_(OptionalCharToString(hostname)),
|
||||
port_(std::move(port)),
|
||||
num_connections_(std::move(num_connections)),
|
||||
prefetch_sz_(std::move(prefetch_sz)) {}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@
|
|||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "include/api/dual_abi_helper.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
|
@ -70,7 +72,12 @@ int32_t get_callback_timeout();
|
|||
|
||||
/// \brief Function to load configuration from a file.
|
||||
/// \param[in] file path of the configuration file to be loaded.
|
||||
bool load(std::string file);
|
||||
/// \note This api exists because std::string will constrained by ABI compile macro but char don't.
|
||||
bool load(const std::vector<char> &file);
|
||||
|
||||
/// \brief Function to load configuration from a file.
|
||||
/// \param[in] file path of the configuration file to be loaded.
|
||||
inline bool load(std::string file) { return load(StringToChar(file)); }
|
||||
|
||||
} // namespace config
|
||||
} // namespace dataset
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -17,10 +17,12 @@
|
|||
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_ITERATOR_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_ITERATOR_H_
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
#include "include/api/dual_abi_helper.h"
|
||||
#include "include/api/status.h"
|
||||
#include "include/api/types.h"
|
||||
|
||||
|
|
@ -39,6 +41,7 @@ class IteratorConsumer;
|
|||
class Dataset;
|
||||
|
||||
using MSTensorMap = std::unordered_map<std::string, mindspore::MSTensor>;
|
||||
using MSTensorMapChar = std::map<std::vector<char>, mindspore::MSTensor>;
|
||||
using MSTensorVec = std::vector<mindspore::MSTensor>;
|
||||
|
||||
// Abstract class for iterating over the dataset.
|
||||
|
|
@ -60,7 +63,18 @@ class Iterator {
|
|||
/// \note Type of return data is a map(with column name).
|
||||
/// \param[out] row - the output tensor row.
|
||||
/// \return - a Status error code, returns OK if no error encountered.
|
||||
Status GetNextRow(MSTensorMap *row);
|
||||
Status GetNextRow(MSTensorMap *row) {
|
||||
MSTensorMapChar row_;
|
||||
row_.clear();
|
||||
row->clear();
|
||||
Status s = GetNextRowCharIF(&row_);
|
||||
TensorMapCharToString(&row_, row);
|
||||
return s;
|
||||
}
|
||||
|
||||
// Char interface(CharIF) of GetNextRow
|
||||
// This api exists because std::string will constrained by ABI compile macro but char don't.
|
||||
Status GetNextRowCharIF(MSTensorMapChar *row);
|
||||
|
||||
/// \brief Function to get the next row from the data pipeline.
|
||||
/// \note Type of return data is a vector(without column name).
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@
|
|||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "include/api/dual_abi_helper.h"
|
||||
#include "include/api/status.h"
|
||||
#include "minddata/dataset/include/constants.h"
|
||||
#include "minddata/dataset/include/transforms.h"
|
||||
|
|
@ -64,11 +65,8 @@ class BasicTokenizer : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
bool lower_case_;
|
||||
bool keep_whitespace_;
|
||||
NormalizeForm normalize_form_;
|
||||
bool preserve_unused_token_;
|
||||
bool with_offsets_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Tokenizer used for Bert text process.
|
||||
|
|
@ -94,7 +92,14 @@ class BertTokenizer : public TensorTransform {
|
|||
int32_t max_bytes_per_token = 100, const std::string &unknown_token = "[UNK]",
|
||||
bool lower_case = false, bool keep_whitespace = false,
|
||||
const NormalizeForm normalize_form = NormalizeForm::kNone, bool preserve_unused_token = true,
|
||||
bool with_offsets = false);
|
||||
bool with_offsets = false)
|
||||
: BertTokenizer(vocab, StringToChar(suffix_indicator), max_bytes_per_token, StringToChar(unknown_token),
|
||||
lower_case, keep_whitespace, normalize_form, preserve_unused_token, with_offsets) {}
|
||||
|
||||
explicit BertTokenizer(const std::shared_ptr<Vocab> &vocab, const std::vector<char> &suffix_indicator,
|
||||
int32_t max_bytes_per_token, const std::vector<char> &unknown_token, bool lower_case,
|
||||
bool keep_whitespace, const NormalizeForm normalize_form, bool preserve_unused_token,
|
||||
bool with_offsets);
|
||||
|
||||
/// \brief Destructor
|
||||
~BertTokenizer() = default;
|
||||
|
|
@ -104,15 +109,8 @@ class BertTokenizer : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::shared_ptr<Vocab> vocab_;
|
||||
std::string suffix_indicator_;
|
||||
int32_t max_bytes_per_token_;
|
||||
std::string unknown_token_;
|
||||
bool lower_case_;
|
||||
bool keep_whitespace_;
|
||||
NormalizeForm normalize_form_;
|
||||
bool preserve_unused_token_;
|
||||
bool with_offsets_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Apply case fold operation on UTF-8 string tensor.
|
||||
|
|
@ -146,7 +144,11 @@ class JiebaTokenizer : public TensorTransform {
|
|||
/// - JiebaMode.kMIX, tokenize with a mix of MPSegment and HMMSegment algorithm.
|
||||
/// \param[in] with_offsets If or not output offsets of tokens (default=false).
|
||||
explicit JiebaTokenizer(const std::string &hmm_path, const std::string &mp_path,
|
||||
const JiebaMode &mode = JiebaMode::kMix, bool with_offsets = false);
|
||||
const JiebaMode &mode = JiebaMode::kMix, bool with_offsets = false)
|
||||
: JiebaTokenizer(StringToChar(hmm_path), StringToChar(mp_path), mode, with_offsets) {}
|
||||
|
||||
explicit JiebaTokenizer(const std::vector<char> &hmm_path, const std::vector<char> &mp_path, const JiebaMode &mode,
|
||||
bool with_offsets);
|
||||
|
||||
/// \brief Destructor
|
||||
~JiebaTokenizer() = default;
|
||||
|
|
@ -158,11 +160,8 @@ class JiebaTokenizer : public TensorTransform {
|
|||
Status AddWord(const std::string &word, int64_t freq = 0);
|
||||
|
||||
private:
|
||||
std::string hmm_path_;
|
||||
std::string mp_path_;
|
||||
JiebaMode mode_;
|
||||
bool with_offsets_;
|
||||
std::vector<std::pair<std::string, int64_t>> words_list_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Look up a word into an id according to the input vocabulary table.
|
||||
|
|
@ -175,7 +174,11 @@ class Lookup : public TensorTransform {
|
|||
/// specify unknown_token when word being out of Vocabulary (default={}).
|
||||
/// \param[in] data_type type of the tensor after lookup, typically int32.
|
||||
explicit Lookup(const std::shared_ptr<Vocab> &vocab, const std::optional<std::string> &unknown_token = {},
|
||||
const std::string &data_type = "int32");
|
||||
const std::string &data_type = "int32")
|
||||
: Lookup(vocab, OptionalStringToChar(unknown_token), StringToChar(data_type)) {}
|
||||
|
||||
explicit Lookup(const std::shared_ptr<Vocab> &vocab, const std::optional<std::vector<char>> &unknown_token,
|
||||
const std::vector<char> &data_type);
|
||||
|
||||
/// \brief Destructor
|
||||
~Lookup() = default;
|
||||
|
|
@ -185,9 +188,8 @@ class Lookup : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::shared_ptr<Vocab> vocab_;
|
||||
std::optional<std::string> unknown_token_;
|
||||
std::string data_type_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief TensorOp to generate n-gram from a 1-D string Tensor.
|
||||
|
|
@ -203,7 +205,11 @@ class Ngram : public TensorTransform {
|
|||
/// be capped at n-1. right_pad=("-":2) would pad right side of the sequence with "--" (default={"", 0}}).
|
||||
/// \param[in] separator Symbol used to join strings together (default=" ").
|
||||
explicit Ngram(const std::vector<int32_t> &ngrams, const std::pair<std::string, int32_t> &left_pad = {"", 0},
|
||||
const std::pair<std::string, int32_t> &right_pad = {"", 0}, const std::string &separator = " ");
|
||||
const std::pair<std::string, int32_t> &right_pad = {"", 0}, const std::string &separator = " ")
|
||||
: Ngram(ngrams, PairStringToChar(left_pad), PairStringToChar(right_pad), StringToChar(separator)) {}
|
||||
|
||||
explicit Ngram(const std::vector<int32_t> &ngrams, const std::pair<std::vector<char>, int32_t> &left_pad,
|
||||
const std::pair<std::vector<char>, int32_t> &right_pad, const std::vector<char> &separator);
|
||||
|
||||
/// \brief Destructor
|
||||
~Ngram() = default;
|
||||
|
|
@ -213,10 +219,8 @@ class Ngram : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> ngrams_;
|
||||
std::pair<std::string, int32_t> left_pad_;
|
||||
std::pair<std::string, int32_t> right_pad_;
|
||||
std::string separator_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
#ifndef _WIN32
|
||||
|
|
@ -243,7 +247,8 @@ class NormalizeUTF8 : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
NormalizeForm normalize_form_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Replace UTF-8 string tensor with 'replace' according to regular expression 'pattern'.
|
||||
|
|
@ -254,7 +259,10 @@ class RegexReplace : public TensorTransform {
|
|||
/// \param[in] replace The string to replace matched element.
|
||||
/// \param[in] replace_all Confirm whether to replace all. If false, only replace first matched element;
|
||||
/// if true, replace all matched elements (default=true).
|
||||
explicit RegexReplace(std::string pattern, std::string replace, bool replace_all = true);
|
||||
explicit RegexReplace(std::string pattern, std::string replace, bool replace_all = true)
|
||||
: RegexReplace(StringToChar(pattern), StringToChar(replace), replace_all) {}
|
||||
|
||||
explicit RegexReplace(const std::vector<char> &pattern, const std::vector<char> &replace, bool replace_all);
|
||||
|
||||
/// \brief Destructor
|
||||
~RegexReplace() = default;
|
||||
|
|
@ -264,9 +272,8 @@ class RegexReplace : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::string pattern_;
|
||||
std::string replace_;
|
||||
bool replace_all_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Tokenize a scalar tensor of UTF-8 string by regex expression pattern.
|
||||
|
|
@ -278,7 +285,11 @@ class RegexTokenizer : public TensorTransform {
|
|||
/// matched by 'keep_delim_pattern'. The default value is an empty string ("")
|
||||
/// which means that delimiters will not be kept as an output token (default="").
|
||||
/// \param[in] with_offsets If or not output offsets of tokens (default=false).
|
||||
explicit RegexTokenizer(std::string delim_pattern, std::string keep_delim_pattern = "", bool with_offsets = false);
|
||||
explicit RegexTokenizer(std::string delim_pattern, std::string keep_delim_pattern = "", bool with_offsets = false)
|
||||
: RegexTokenizer(StringToChar(delim_pattern), StringToChar(keep_delim_pattern), with_offsets) {}
|
||||
|
||||
explicit RegexTokenizer(const std::vector<char> &delim_pattern, const std::vector<char> &keep_delim_pattern,
|
||||
bool with_offsets);
|
||||
|
||||
/// \brief Destructor
|
||||
~RegexTokenizer() = default;
|
||||
|
|
@ -288,9 +299,8 @@ class RegexTokenizer : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::string delim_pattern_;
|
||||
std::string keep_delim_pattern_;
|
||||
bool with_offsets_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
#endif
|
||||
|
||||
|
|
@ -306,7 +316,10 @@ class SentencePieceTokenizer : public TensorTransform {
|
|||
/// \brief Constructor.
|
||||
/// \param[in] vocab_path vocab model file path.
|
||||
/// \param[in] out_type The type of output.
|
||||
SentencePieceTokenizer(const std::string &vocab_path, mindspore::dataset::SPieceTokenizerOutType out_type);
|
||||
SentencePieceTokenizer(const std::string &vocab_path, mindspore::dataset::SPieceTokenizerOutType out_type)
|
||||
: SentencePieceTokenizer(StringToChar(vocab_path), out_type) {}
|
||||
|
||||
SentencePieceTokenizer(const std::vector<char> &vocab_path, mindspore::dataset::SPieceTokenizerOutType out_type);
|
||||
|
||||
/// \brief Destructor
|
||||
~SentencePieceTokenizer() = default;
|
||||
|
|
@ -316,10 +329,8 @@ class SentencePieceTokenizer : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::shared_ptr<SentencePieceVocab> vocab_;
|
||||
std::string vocab_path_;
|
||||
SPieceTokenizerLoadType load_type_;
|
||||
SPieceTokenizerOutType out_type_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief TensorOp to construct a tensor from data (only 1-D for now), where each element in the dimension
|
||||
|
|
@ -340,8 +351,8 @@ class SlidingWindow : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
int32_t width_;
|
||||
int32_t axis_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Tensor operation to convert every element of a string tensor to a number.
|
||||
|
|
@ -353,7 +364,9 @@ class ToNumber : public TensorTransform {
|
|||
public:
|
||||
/// \brief Constructor.
|
||||
/// \param[in] data_type of the tensor to be casted to. Must be a numeric type.
|
||||
explicit ToNumber(const std::string &data_type);
|
||||
explicit ToNumber(const std::string &data_type) : ToNumber(StringToChar(data_type)) {}
|
||||
|
||||
explicit ToNumber(const std::vector<char> &data_type);
|
||||
|
||||
/// \brief Destructor
|
||||
~ToNumber() = default;
|
||||
|
|
@ -363,7 +376,8 @@ class ToNumber : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::string data_type_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Truncate a pair of rank-1 tensors such that the total length is less than max_length.
|
||||
|
|
@ -381,7 +395,8 @@ class TruncateSequencePair : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
int32_t max_length_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Tokenize a scalar tensor of UTF-8 string to Unicode characters.
|
||||
|
|
@ -399,7 +414,8 @@ class UnicodeCharTokenizer : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
bool with_offsets_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
#ifndef _WIN32
|
||||
|
|
@ -419,8 +435,8 @@ class UnicodeScriptTokenizer : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
bool keep_whitespace_;
|
||||
bool with_offsets_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Tokenize a scalar tensor of UTF-8 string on ICU4C defined whitespaces.
|
||||
|
|
@ -438,7 +454,8 @@ class WhitespaceTokenizer : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
bool with_offsets_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
#endif
|
||||
} // namespace text
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@
|
|||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "include/api/dual_abi_helper.h"
|
||||
#include "include/api/status.h"
|
||||
#include "minddata/dataset/include/constants.h"
|
||||
|
||||
|
|
@ -72,7 +73,8 @@ class Compose : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<TensorOperation>> transforms_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Duplicate Op.
|
||||
|
|
@ -107,7 +109,8 @@ class OneHot : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
float num_classes_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomApply Op.
|
||||
|
|
@ -129,8 +132,8 @@ class RandomApply : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<TensorOperation>> transforms_;
|
||||
double prob_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomChoice Op.
|
||||
|
|
@ -151,7 +154,8 @@ class RandomChoice : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<TensorOperation>> transforms_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief TypeCast Op.
|
||||
|
|
@ -160,7 +164,9 @@ class TypeCast : public TensorTransform {
|
|||
public:
|
||||
/// \brief Constructor.
|
||||
/// \param[in] data_type mindspore.dtype to be cast to.
|
||||
explicit TypeCast(std::string data_type);
|
||||
explicit TypeCast(std::string data_type) : TypeCast(StringToChar(data_type)) {}
|
||||
|
||||
explicit TypeCast(const std::vector<char> &data_type);
|
||||
|
||||
/// \brief Destructor
|
||||
~TypeCast() = default;
|
||||
|
|
@ -170,7 +176,8 @@ class TypeCast : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::string data_type_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Unique Op.
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@
|
|||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "include/api/dual_abi_helper.h"
|
||||
#include "include/api/status.h"
|
||||
#include "minddata/dataset/include/constants.h"
|
||||
#include "minddata/dataset/include/transforms.h"
|
||||
|
|
@ -52,8 +53,8 @@ class AutoContrast : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
float cutoff_;
|
||||
std::vector<uint32_t> ignore_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief BoundingBoxAugment TensorTransform.
|
||||
|
|
@ -83,8 +84,8 @@ class BoundingBoxAugment : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::shared_ptr<TensorOperation> transform_;
|
||||
float ratio_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Constructor to apply CutMix on a batch of images
|
||||
|
|
@ -106,9 +107,8 @@ class CutMixBatch : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
float alpha_;
|
||||
float prob_;
|
||||
ImageBatchFormat image_batch_format_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief CutOut TensorOp
|
||||
|
|
@ -128,8 +128,8 @@ class CutOut : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
int32_t length_;
|
||||
int32_t num_patches_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Equalize TensorTransform.
|
||||
|
|
@ -194,7 +194,8 @@ class MixUpBatch : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
float alpha_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief NormalizePad TensorTransform.
|
||||
|
|
@ -210,7 +211,10 @@ class NormalizePad : public TensorTransform {
|
|||
/// \param[in] dtype The output datatype of Tensor.
|
||||
/// The standard deviation values must be "float32" or "float16"（default = "float32"）
|
||||
explicit NormalizePad(const std::vector<float> &mean, const std::vector<float> &std,
|
||||
const std::string &dtype = "float32");
|
||||
const std::string &dtype = "float32")
|
||||
: NormalizePad(mean, std, StringToChar(dtype)) {}
|
||||
|
||||
explicit NormalizePad(const std::vector<float> &mean, const std::vector<float> &std, const std::vector<char> &dtype);
|
||||
|
||||
/// \brief Destructor.
|
||||
~NormalizePad() = default;
|
||||
|
|
@ -220,9 +224,8 @@ class NormalizePad : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<float> mean_;
|
||||
std::vector<float> std_;
|
||||
std::string dtype_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Pad TensorOp
|
||||
|
|
@ -257,9 +260,8 @@ class Pad : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> padding_;
|
||||
std::vector<uint8_t> fill_value_;
|
||||
BorderType padding_mode_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Blends an image with its grayscale version with random weights
|
||||
|
|
@ -280,8 +282,8 @@ class RandomColor : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
float t_lb_;
|
||||
float t_ub_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomColorAdjust TensorTransform.
|
||||
|
|
@ -309,10 +311,8 @@ class RandomColorAdjust : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<float> brightness_;
|
||||
std::vector<float> contrast_;
|
||||
std::vector<float> saturation_;
|
||||
std::vector<float> hue_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomCrop TensorTransform.
|
||||
|
|
@ -346,11 +346,8 @@ class RandomCrop : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> size_;
|
||||
std::vector<int32_t> padding_;
|
||||
bool pad_if_needed_;
|
||||
std::vector<uint8_t> fill_value_;
|
||||
BorderType padding_mode_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomCropDecodeResize TensorTransform.
|
||||
|
|
@ -381,11 +378,8 @@ class RandomCropDecodeResize : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> size_;
|
||||
std::vector<float> scale_;
|
||||
std::vector<float> ratio_;
|
||||
InterpolationMode interpolation_;
|
||||
int32_t max_attempts_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomCropWithBBox TensorTransform.
|
||||
|
|
@ -421,11 +415,8 @@ class RandomCropWithBBox : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> size_;
|
||||
std::vector<int32_t> padding_;
|
||||
bool pad_if_needed_;
|
||||
std::vector<uint8_t> fill_value_;
|
||||
BorderType padding_mode_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomHorizontalFlip TensorTransform.
|
||||
|
|
@ -444,7 +435,8 @@ class RandomHorizontalFlip : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
float probability_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomHorizontalFlipWithBBox TensorTransform.
|
||||
|
|
@ -463,7 +455,8 @@ class RandomHorizontalFlipWithBBox : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
float probability_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomPosterize TensorTransform.
|
||||
|
|
@ -482,7 +475,8 @@ class RandomPosterize : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<uint8_t> bit_range_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomResize TensorTransform.
|
||||
|
|
@ -503,7 +497,8 @@ class RandomResize : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> size_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomResizeWithBBox TensorTransform.
|
||||
|
|
@ -525,7 +520,8 @@ class RandomResizeWithBBox : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> size_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomResizedCrop TensorTransform.
|
||||
|
|
@ -555,11 +551,8 @@ class RandomResizedCrop : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> size_;
|
||||
std::vector<float> scale_;
|
||||
std::vector<float> ratio_;
|
||||
InterpolationMode interpolation_;
|
||||
int32_t max_attempts_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomResizedCropWithBBox TensorTransform.
|
||||
|
|
@ -589,11 +582,8 @@ class RandomResizedCropWithBBox : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> size_;
|
||||
std::vector<float> scale_;
|
||||
std::vector<float> ratio_;
|
||||
InterpolationMode interpolation_;
|
||||
int32_t max_attempts_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomRotation TensorOp
|
||||
|
|
@ -620,11 +610,8 @@ class RandomRotation : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<float> degrees_;
|
||||
InterpolationMode interpolation_mode_;
|
||||
std::vector<float> center_;
|
||||
bool expand_;
|
||||
std::vector<uint8_t> fill_value_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomSelectSubpolicy TensorTransform.
|
||||
|
|
@ -655,7 +642,8 @@ class RandomSelectSubpolicy : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<std::vector<std::pair<std::shared_ptr<TensorOperation>, double>>> policy_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomSharpness TensorTransform.
|
||||
|
|
@ -675,7 +663,8 @@ class RandomSharpness : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<float> degrees_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomSolarize TensorTransform.
|
||||
|
|
@ -695,7 +684,8 @@ class RandomSolarize : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<uint8_t> threshold_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomVerticalFlip TensorTransform.
|
||||
|
|
@ -714,7 +704,8 @@ class RandomVerticalFlip : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
float probability_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomVerticalFlipWithBBox TensorTransform.
|
||||
|
|
@ -733,7 +724,8 @@ class RandomVerticalFlipWithBBox : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
float probability_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RescaleOperation TensorTransform.
|
||||
|
|
@ -753,8 +745,8 @@ class Rescale : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
float rescale_;
|
||||
float shift_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief ResizeWithBBox TensorTransform.
|
||||
|
|
@ -776,8 +768,8 @@ class ResizeWithBBox : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> size_;
|
||||
InterpolationMode interpolation_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RgbaToBgr TensorTransform.
|
||||
|
|
@ -833,10 +825,8 @@ class SoftDvppDecodeRandomCropResizeJpeg : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> size_;
|
||||
std::vector<float> scale_;
|
||||
std::vector<float> ratio_;
|
||||
int32_t max_attempts_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief SoftDvppDecodeResizeJpeg TensorTransform.
|
||||
|
|
@ -864,7 +854,8 @@ class SoftDvppDecodeResizeJpeg : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> size_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief SwapRedBlue TensorOp
|
||||
|
|
@ -909,8 +900,8 @@ class UniformAugment : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<TensorOperation>> transforms_;
|
||||
int32_t num_ops_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
} // namespace vision
|
||||
|
|
|
|||
|
|
@ -50,7 +50,8 @@ class DvppDecodeResizeJpeg : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) override;
|
||||
|
||||
private:
|
||||
std::vector<uint32_t> resize_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
class DvppDecodeResizeCropJpeg : public TensorTransform {
|
||||
|
|
@ -70,8 +71,8 @@ class DvppDecodeResizeCropJpeg : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) override;
|
||||
|
||||
private:
|
||||
std::vector<uint32_t> crop_;
|
||||
std::vector<uint32_t> resize_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
class DvppDecodePng : public TensorTransform {
|
||||
|
|
|
|||
|
|
@ -62,12 +62,8 @@ class Affine : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
float degrees_;
|
||||
std::vector<float> translation_;
|
||||
float scale_;
|
||||
std::vector<float> shear_;
|
||||
InterpolationMode interpolation_;
|
||||
std::vector<uint8_t> fill_value_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief CenterCrop TensorTransform.
|
||||
|
|
@ -90,7 +86,8 @@ class CenterCrop : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> size_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Crop TensorTransform.
|
||||
|
|
@ -112,8 +109,8 @@ class Crop : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> coordinates_;
|
||||
std::vector<int32_t> size_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Decode TensorTransform.
|
||||
|
|
@ -134,7 +131,8 @@ class Decode : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) override;
|
||||
|
||||
private:
|
||||
bool rgb_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Normalize TensorTransform.
|
||||
|
|
@ -158,8 +156,8 @@ class Normalize : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) override;
|
||||
|
||||
private:
|
||||
std::vector<float> mean_;
|
||||
std::vector<float> std_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomAffine TensorTransform.
|
||||
|
|
@ -196,12 +194,8 @@ class RandomAffine : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<float_t> degrees_; // min_degree, max_degree
|
||||
std::vector<float_t> translate_range_; // maximum x translation percentage, maximum y translation percentage
|
||||
std::vector<float_t> scale_range_; // min_scale, max_scale
|
||||
std::vector<float_t> shear_ranges_; // min_x_shear, max_x_shear, min_y_shear, max_y_shear
|
||||
InterpolationMode interpolation_;
|
||||
std::vector<uint8_t> fill_value_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Resize TensorTransform.
|
||||
|
|
@ -225,8 +219,8 @@ class Resize : public TensorTransform {
|
|||
std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> size_;
|
||||
InterpolationMode interpolation_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Rotate TensorTransform.
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
|
|
@ -28,8 +29,10 @@
|
|||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "include/api/dual_abi_helper.h"
|
||||
#include "include/iterator.h"
|
||||
#include "include/samplers.h"
|
||||
#include "include/transforms.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
|
@ -94,11 +97,13 @@ class Dataset : public std::enable_shared_from_this<Dataset> {
|
|||
|
||||
/// \brief Gets the column names
|
||||
/// \return Names of the columns. If failed, return an empty vector
|
||||
std::vector<std::string> GetColumnNames();
|
||||
std::vector<std::string> GetColumnNames() { return VectorCharToString(GetColumnNamesCharIF()); }
|
||||
|
||||
/// \brief Gets the class indexing
|
||||
/// \return a map of ClassIndexing. If failed, return an empty map
|
||||
std::vector<std::pair<std::string, std::vector<int32_t>>> GetClassIndexing();
|
||||
std::vector<std::pair<std::string, std::vector<int32_t>>> GetClassIndexing() {
|
||||
return ClassIndexCharToString(GetClassIndexingCharIF());
|
||||
}
|
||||
|
||||
/// \brief Setter function for runtime number of workers
|
||||
/// \param[in] num_workers The number of threads in this operator
|
||||
|
|
@ -110,7 +115,9 @@ class Dataset : public std::enable_shared_from_this<Dataset> {
|
|||
/// \param[in] num_epochs Number of epochs to run through the pipeline, default -1 which means infinite epochs.
|
||||
/// An empty row is returned at the end of each epoch
|
||||
/// \return Shared pointer to the Iterator
|
||||
std::shared_ptr<Iterator> CreateIterator(std::vector<std::string> columns = {}, int32_t num_epochs = -1);
|
||||
std::shared_ptr<Iterator> CreateIterator(std::vector<std::string> columns = {}, int32_t num_epochs = -1) {
|
||||
return CreateIteratorCharIF(VectorStringToChar(columns), num_epochs);
|
||||
}
|
||||
|
||||
/// \brief Function to create a BatchDataset
|
||||
/// \notes Combines batch_size number of consecutive rows into batches
|
||||
|
|
@ -138,14 +145,49 @@ class Dataset : public std::enable_shared_from_this<Dataset> {
|
|||
/// \param[in] project_columns A list of column names to project
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current MapDataset
|
||||
std::shared_ptr<MapDataset> Map(std::vector<std::shared_ptr<TensorOperation>> operations,
|
||||
std::shared_ptr<MapDataset> Map(std::vector<TensorTransform *> operations,
|
||||
const std::vector<std::string> &input_columns = {},
|
||||
const std::vector<std::string> &output_columns = {},
|
||||
const std::vector<std::string> &project_columns = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr,
|
||||
std::vector<std::shared_ptr<DSCallback>> callbacks = {}) {
|
||||
return std::make_shared<MapDataset>(shared_from_this(), operations, input_columns, output_columns, project_columns,
|
||||
cache, callbacks);
|
||||
std::vector<std::shared_ptr<TensorOperation>> transform_ops;
|
||||
(void)std::transform(
|
||||
operations.begin(), operations.end(), std::back_inserter(transform_ops),
|
||||
[](TensorTransform *op) -> std::shared_ptr<TensorOperation> { return op != nullptr ? op->Parse() : nullptr; });
|
||||
return std::make_shared<MapDataset>(shared_from_this(), transform_ops, VectorStringToChar(input_columns),
|
||||
VectorStringToChar(output_columns), VectorStringToChar(project_columns), cache,
|
||||
callbacks);
|
||||
}
|
||||
|
||||
std::shared_ptr<MapDataset> Map(std::vector<std::shared_ptr<TensorTransform>> operations,
|
||||
const std::vector<std::string> &input_columns = {},
|
||||
const std::vector<std::string> &output_columns = {},
|
||||
const std::vector<std::string> &project_columns = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr,
|
||||
std::vector<std::shared_ptr<DSCallback>> callbacks = {}) {
|
||||
std::vector<std::shared_ptr<TensorOperation>> transform_ops;
|
||||
(void)std::transform(operations.begin(), operations.end(), std::back_inserter(transform_ops),
|
||||
[](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> {
|
||||
return op != nullptr ? op->Parse() : nullptr;
|
||||
});
|
||||
return std::make_shared<MapDataset>(shared_from_this(), transform_ops, VectorStringToChar(input_columns),
|
||||
VectorStringToChar(output_columns), VectorStringToChar(project_columns), cache,
|
||||
callbacks);
|
||||
}
|
||||
|
||||
std::shared_ptr<MapDataset> Map(const std::vector<std::reference_wrapper<TensorTransform>> operations,
|
||||
const std::vector<std::string> &input_columns = {},
|
||||
const std::vector<std::string> &output_columns = {},
|
||||
const std::vector<std::string> &project_columns = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr,
|
||||
std::vector<std::shared_ptr<DSCallback>> callbacks = {}) {
|
||||
std::vector<std::shared_ptr<TensorOperation>> transform_ops;
|
||||
(void)std::transform(operations.begin(), operations.end(), std::back_inserter(transform_ops),
|
||||
[](TensorTransform &op) -> std::shared_ptr<TensorOperation> { return op.Parse(); });
|
||||
return std::make_shared<MapDataset>(shared_from_this(), transform_ops, VectorStringToChar(input_columns),
|
||||
VectorStringToChar(output_columns), VectorStringToChar(project_columns), cache,
|
||||
callbacks);
|
||||
}
|
||||
|
||||
/// \brief Function to create a Project Dataset
|
||||
|
|
@ -153,7 +195,7 @@ class Dataset : public std::enable_shared_from_this<Dataset> {
|
|||
/// \param[in] columns The name of columns to project
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<ProjectDataset> Project(const std::vector<std::string> &columns) {
|
||||
return std::make_shared<ProjectDataset>(shared_from_this(), columns);
|
||||
return std::make_shared<ProjectDataset>(shared_from_this(), VectorStringToChar(columns));
|
||||
}
|
||||
|
||||
/// \brief Function to create a Shuffle Dataset
|
||||
|
|
@ -169,6 +211,16 @@ class Dataset : public std::enable_shared_from_this<Dataset> {
|
|||
protected:
|
||||
std::shared_ptr<TreeGetters> tree_getters_;
|
||||
std::shared_ptr<DatasetNode> ir_node_;
|
||||
|
||||
private:
|
||||
// Char interface(CharIF) of GetColumnNames
|
||||
std::vector<std::vector<char>> GetColumnNamesCharIF();
|
||||
|
||||
// Char interface(CharIF) of GetClassIndexing
|
||||
std::vector<std::pair<std::vector<char>, std::vector<int32_t>>> GetClassIndexingCharIF();
|
||||
|
||||
// Char interface(CharIF) of CreateIterator
|
||||
std::shared_ptr<Iterator> CreateIteratorCharIF(std::vector<std::vector<char>> columns, int32_t num_epochs);
|
||||
};
|
||||
|
||||
class BatchDataset : public Dataset {
|
||||
|
|
@ -180,15 +232,15 @@ class BatchDataset : public Dataset {
|
|||
class MapDataset : public Dataset {
|
||||
public:
|
||||
MapDataset(std::shared_ptr<Dataset> input, std::vector<std::shared_ptr<TensorOperation>> operations,
|
||||
const std::vector<std::string> &input_columns, const std::vector<std::string> &output_columns,
|
||||
const std::vector<std::string> &project_columns, const std::shared_ptr<DatasetCache> &cache,
|
||||
const std::vector<std::vector<char>> &input_columns, const std::vector<std::vector<char>> &output_columns,
|
||||
const std::vector<std::vector<char>> &project_columns, const std::shared_ptr<DatasetCache> &cache,
|
||||
std::vector<std::shared_ptr<DSCallback>> callbacks);
|
||||
~MapDataset() = default;
|
||||
};
|
||||
|
||||
class ProjectDataset : public Dataset {
|
||||
public:
|
||||
ProjectDataset(std::shared_ptr<Dataset> input, const std::vector<std::string> &columns);
|
||||
ProjectDataset(std::shared_ptr<Dataset> input, const std::vector<std::vector<char>> &columns);
|
||||
~ProjectDataset() = default;
|
||||
};
|
||||
|
||||
|
|
@ -201,14 +253,22 @@ class ShuffleDataset : public Dataset {
|
|||
/// \brief Function to create a SchemaObj
|
||||
/// \param[in] schema_file Path of schema file
|
||||
/// \return Shared pointer to the current schema
|
||||
std::shared_ptr<SchemaObj> Schema(const std::string &schema_file = "");
|
||||
std::shared_ptr<SchemaObj> SchemaCharIF(const std::vector<char> &schema_file);
|
||||
|
||||
inline std::shared_ptr<SchemaObj> Schema(const std::string &schema_file = "") {
|
||||
return SchemaCharIF(StringToChar(schema_file));
|
||||
}
|
||||
class AlbumDataset : public Dataset {
|
||||
public:
|
||||
AlbumDataset(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names = {}, bool decode = false,
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema,
|
||||
const std::vector<std::vector<char>> &column_names, bool decode, const std::shared_ptr<Sampler> &sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache);
|
||||
AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema,
|
||||
const std::vector<std::vector<char>> &column_names, bool decode, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache);
|
||||
AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema,
|
||||
const std::vector<std::vector<char>> &column_names, bool decode,
|
||||
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
|
||||
~AlbumDataset() = default;
|
||||
};
|
||||
|
||||
|
|
@ -219,20 +279,58 @@ class AlbumDataset : public Dataset {
|
|||
/// \param[in] column_names Column names used to specify columns to load, if empty, will read all columns.
|
||||
/// (default = {})
|
||||
/// \param[in] decode the option to decode the images in dataset (default = false)
|
||||
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
|
||||
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
|
||||
/// given,
|
||||
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names = {}, bool decode = false,
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names = {}, bool decode = false,
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr) {
|
||||
return std::make_shared<AlbumDataset>(StringToChar(dataset_dir), StringToChar(data_schema),
|
||||
VectorStringToChar(column_names), decode, sampler, cache);
|
||||
}
|
||||
/// \brief Function to create an AlbumDataset
|
||||
/// \notes The generated dataset is specified through setting a schema
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] data_schema Path to dataset schema file
|
||||
/// \param[in] column_names Column names used to specify columns to load
|
||||
/// \param[in] decode the option to decode the images in dataset
|
||||
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names, bool decode, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr) {
|
||||
return std::make_shared<AlbumDataset>(StringToChar(dataset_dir), StringToChar(data_schema),
|
||||
VectorStringToChar(column_names), decode, sampler, cache);
|
||||
}
|
||||
/// \brief Function to create an AlbumDataset
|
||||
/// \notes The generated dataset is specified through setting a schema
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] data_schema Path to dataset schema file
|
||||
/// \param[in] column_names Column names used to specify columns to load
|
||||
/// \param[in] decode the option to decode the images in dataset
|
||||
/// \param[in] sampler Sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names, bool decode,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr) {
|
||||
return std::make_shared<AlbumDataset>(StringToChar(dataset_dir), StringToChar(data_schema),
|
||||
VectorStringToChar(column_names), decode, sampler, cache);
|
||||
}
|
||||
|
||||
class MnistDataset : public Dataset {
|
||||
public:
|
||||
explicit MnistDataset(const std::string &dataset_dir, const std::string &usage = "all",
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
|
||||
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
|
||||
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache);
|
||||
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
|
||||
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
|
||||
~MnistDataset() = default;
|
||||
};
|
||||
|
||||
|
|
@ -240,13 +338,41 @@ class MnistDataset : public Dataset {
|
|||
/// \notes The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] usage of MNIST, can be "train", "test" or "all" (default = "all").
|
||||
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
|
||||
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
|
||||
/// given,
|
||||
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current MnistDataset
|
||||
std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage = "all",
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage = "all",
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr) {
|
||||
return std::make_shared<MnistDataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
|
||||
}
|
||||
|
||||
/// \brief Function to create a MnistDataset
|
||||
/// \notes The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] usage of MNIST, can be "train", "test" or "all"
|
||||
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current MnistDataset
|
||||
inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr) {
|
||||
return std::make_shared<MnistDataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
|
||||
}
|
||||
|
||||
/// \brief Function to create a MnistDataset
|
||||
/// \notes The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] usage of MNIST, can be "train", "test" or "all"
|
||||
/// \param[in] sampler Sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current MnistDataset
|
||||
inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr) {
|
||||
return std::make_shared<MnistDataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
|
||||
}
|
||||
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
|
|
|||
|
|
@ -26,16 +26,27 @@
|
|||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
class DeviceResource;
|
||||
// class to run tensor operations in eager mode
|
||||
class Execute {
|
||||
public:
|
||||
/// \brief Constructor
|
||||
explicit Execute(std::shared_ptr<TensorOperation> op);
|
||||
// FIXME - Temporarily overload Execute to support both TensorOperation and TensorTransform
|
||||
explicit Execute(std::shared_ptr<TensorOperation> op, MapTargetDevice deviceType = MapTargetDevice::kCpu);
|
||||
explicit Execute(std::shared_ptr<TensorTransform> op, MapTargetDevice deviceType = MapTargetDevice::kCpu);
|
||||
// explicit Execute(TensorTransform op, MapTargetDevice deviceType = MapTargetDevice::KCpu);
|
||||
explicit Execute(TensorTransform *op, MapTargetDevice deviceType = MapTargetDevice::kCpu);
|
||||
|
||||
explicit Execute(std::vector<std::shared_ptr<TensorOperation>> ops);
|
||||
explicit Execute(std::vector<std::shared_ptr<TensorOperation>> ops,
|
||||
MapTargetDevice deviceType = MapTargetDevice::kCpu);
|
||||
explicit Execute(std::vector<std::shared_ptr<TensorTransform>> ops,
|
||||
MapTargetDevice deviceType = MapTargetDevice::kCpu);
|
||||
explicit Execute(const std::vector<std::reference_wrapper<TensorTransform>> ops,
|
||||
MapTargetDevice deviceType = MapTargetDevice::kCpu);
|
||||
explicit Execute(std::vector<TensorTransform *> ops, MapTargetDevice deviceType = MapTargetDevice::kCpu);
|
||||
|
||||
/// \brief Destructor
|
||||
~Execute() = default;
|
||||
~Execute();
|
||||
|
||||
/// \brief callable function to execute the TensorOperation in eager mode
|
||||
/// \param[in] input Tensor to be transformed
|
||||
|
|
@ -49,8 +60,16 @@ class Execute {
|
|||
/// \return - Status
|
||||
Status operator()(const std::vector<mindspore::MSTensor> &input_tensor_list, std::vector<mindspore::MSTensor> *out);
|
||||
|
||||
Status DeviceMemoryRelease();
|
||||
|
||||
private:
|
||||
Status validate_device_();
|
||||
|
||||
std::vector<std::shared_ptr<TensorOperation>> ops_;
|
||||
|
||||
MapTargetDevice device_type_;
|
||||
|
||||
std::shared_ptr<DeviceResource> device_resource_;
|
||||
};
|
||||
|
||||
} // namespace dataset
|
||||
|
|
|
|||
|
|
@ -17,10 +17,12 @@
|
|||
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_ITERATOR_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_ITERATOR_H_
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
#include "include/api/dual_abi_helper.h"
|
||||
#include "include/api/status.h"
|
||||
#include "include/api/types.h"
|
||||
|
||||
|
|
@ -39,6 +41,7 @@ class IteratorConsumer;
|
|||
class Dataset;
|
||||
|
||||
using MSTensorMap = std::unordered_map<std::string, mindspore::MSTensor>;
|
||||
using MSTensorMapChar = std::map<std::vector<char>, mindspore::MSTensor>;
|
||||
using MSTensorVec = std::vector<mindspore::MSTensor>;
|
||||
|
||||
// Abstract class for iterating over the dataset.
|
||||
|
|
@ -60,7 +63,18 @@ class Iterator {
|
|||
/// \note Type of return data is a map(with column name).
|
||||
/// \param[out] row - the output tensor row.
|
||||
/// \return - a Status error code, returns OK if no error encountered.
|
||||
Status GetNextRow(MSTensorMap *row);
|
||||
Status GetNextRow(MSTensorMap *row) {
|
||||
MSTensorMapChar row_;
|
||||
row_.clear();
|
||||
row->clear();
|
||||
Status s = GetNextRowCharIF(&row_);
|
||||
TensorMapCharToString(&row_, row);
|
||||
return s;
|
||||
}
|
||||
|
||||
// Char interface(CharIF) of GetNextRow
|
||||
// This api exists because std::string will constrained by ABI compile macro but char don't.
|
||||
Status GetNextRowCharIF(MSTensorMapChar *row);
|
||||
|
||||
/// \brief Function to get the next row from the data pipeline.
|
||||
/// \note Type of return data is a vector(without column name).
|
||||
|
|
|
|||
|
|
@ -26,143 +26,59 @@
|
|||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
// Internal Sampler class forward declaration
|
||||
class SamplerRT;
|
||||
class SamplerObj;
|
||||
|
||||
// Abstract class to represent a sampler in the data pipeline.
|
||||
/// \class Sampler samplers.h
|
||||
/// \brief An abstract base class to represent a sampler in the data pipeline.
|
||||
class Sampler : std::enable_shared_from_this<Sampler> {
|
||||
friend class AlbumDataset;
|
||||
friend class MindDataDataset;
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
class SamplerObj {
|
||||
public:
|
||||
/// \brief Constructor
|
||||
SamplerObj();
|
||||
Sampler() {}
|
||||
|
||||
/// \brief Destructor
|
||||
~SamplerObj() = default;
|
||||
~Sampler() = default;
|
||||
|
||||
/// \brief Pure virtual function for derived class to implement parameters validation
|
||||
/// \return The Status code of the function. It returns OK status if parameters are valid.
|
||||
virtual Status ValidateParams() = 0;
|
||||
|
||||
/// \brief Pure virtual function to convert a SamplerObj class into a runtime sampler object
|
||||
/// \return Shared pointers to the newly created Sampler
|
||||
virtual std::shared_ptr<SamplerRT> SamplerBuild() = 0;
|
||||
|
||||
/// \brief Pure virtual function to copy a SamplerObj class
|
||||
/// \return Shared pointers to the newly copied SamplerObj
|
||||
virtual std::shared_ptr<SamplerObj> SamplerCopy() = 0;
|
||||
|
||||
/// \brief Function for derived class to get the shard id of sampler
|
||||
/// \return The shard id of the derived sampler
|
||||
virtual int64_t ShardId() { return 0; }
|
||||
|
||||
/// \brief Adds a child to the sampler
|
||||
/// \param[in] child The sampler to be added as child
|
||||
/// \return the Status code returned
|
||||
Status AddChildSampler(std::shared_ptr<SamplerObj> child);
|
||||
|
||||
std::vector<std::shared_ptr<SamplerObj>> GetChild() { return children_; }
|
||||
/// \brief A virtual function to add a child sampler.
|
||||
/// \param[in] child The child sampler to be added as a children of this sampler.
|
||||
virtual void AddChild(std::shared_ptr<Sampler> child) { children_.push_back(child); }
|
||||
|
||||
protected:
|
||||
/// \brief A function that calls build on the children of this sampler
|
||||
/// \param[in] sampler The samplerRT object built from this sampler
|
||||
void BuildChildren(std::shared_ptr<SamplerRT> sampler);
|
||||
/// \brief Pure virtual function to convert a Sampler class into an IR Sampler object.
|
||||
/// \return shared pointer to the newly created TensorOperation.
|
||||
virtual std::shared_ptr<SamplerObj> Parse() = 0;
|
||||
|
||||
std::vector<std::shared_ptr<SamplerObj>> children_;
|
||||
std::vector<std::shared_ptr<Sampler>> children_;
|
||||
};
|
||||
|
||||
class DistributedSamplerObj;
|
||||
class PKSamplerObj;
|
||||
class PreBuiltSamplerObj;
|
||||
class RandomSamplerObj;
|
||||
class SequentialSamplerObj;
|
||||
class SubsetSamplerObj;
|
||||
class SubsetRandomSamplerObj;
|
||||
class WeightedRandomSamplerObj;
|
||||
/// \brief A class to represent a Distributed Sampler in the data pipeline.
|
||||
/// \notes A Sampler that accesses a shard of the dataset.
|
||||
class DistributedSampler : public Sampler {
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
/// Function to create a Distributed Sampler.
|
||||
/// \notes A Sampler that access a shard of the dataset.
|
||||
/// \param[in] num_shards - Number of shards to divide the dataset into.
|
||||
/// \param[in] shard_id - Shard ID of the current shard within num_shards.
|
||||
/// \param[in] shuffle - If true, the indices are shuffled.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \param[in] seed - The seed in use when shuffle is true.
|
||||
/// \param[in] offset - The starting position where access to elements in the dataset begins.
|
||||
/// \param[in] even_dist - If true, each shard would return the same number of rows (default to true).
|
||||
/// If false the total rows returned by all the shards would not have overlap.
|
||||
/// \return Shared pointer to the current Sampler.
|
||||
std::shared_ptr<DistributedSamplerObj> DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle = true,
|
||||
int64_t num_samples = 0, uint32_t seed = 1,
|
||||
int64_t offset = -1, bool even_dist = true);
|
||||
|
||||
/// Function to create a PK Sampler.
|
||||
/// \notes Samples K elements for each P class in the dataset.
|
||||
/// This will sample all classes.
|
||||
/// \param[in] num_val - Number of elements to sample for each class.
|
||||
/// \param[in] shuffle - If true, the class IDs are shuffled.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \return Shared pointer to the current Sampler.
|
||||
std::shared_ptr<PKSamplerObj> PKSampler(int64_t num_val, bool shuffle = false, int64_t num_samples = 0);
|
||||
|
||||
/// Function to create a Random Sampler.
|
||||
/// \notes Samples the elements randomly.
|
||||
/// \param[in] replacement - If true, put the sample ID back for the next draw.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \return Shared pointer to the current Sampler.
|
||||
std::shared_ptr<RandomSamplerObj> RandomSampler(bool replacement = false, int64_t num_samples = 0);
|
||||
|
||||
/// Function to create a Sequential Sampler.
|
||||
/// \notes Samples the dataset elements sequentially, same as not having a sampler.
|
||||
/// \param[in] start_index - Index to start sampling at (default to start at first id).
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \return Shared pointer to the current Sampler.
|
||||
std::shared_ptr<SequentialSamplerObj> SequentialSampler(int64_t start_index = 0, int64_t num_samples = 0);
|
||||
|
||||
/// Function to create a Subset Sampler.
|
||||
/// \notes Samples the elements from a sequence of indices.
|
||||
/// \param[in] indices - A vector sequence of indices.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \return Shared pointer to the current Sampler.
|
||||
std::shared_ptr<SubsetSamplerObj> SubsetSampler(std::vector<int64_t> indices, int64_t num_samples = 0);
|
||||
|
||||
/// Function to create a Subset Random Sampler.
|
||||
/// \notes Samples the elements randomly from a sequence of indices.
|
||||
/// \param[in] indices - A vector sequence of indices.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \return Shared pointer to the current Sampler.
|
||||
std::shared_ptr<SubsetRandomSamplerObj> SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples = 0);
|
||||
|
||||
/// Function to create a Weighted Random Sampler.
|
||||
/// \notes Samples the elements from [0, len(weights) - 1] randomly with the given
|
||||
/// weights (probabilities).
|
||||
/// \param[in] weights - A vector sequence of weights, not necessarily summing up to 1.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \param[in] replacement - If true, put the sample ID back for the next draw.
|
||||
/// \return Shared pointer to the current Sampler.
|
||||
std::shared_ptr<WeightedRandomSamplerObj> WeightedRandomSampler(std::vector<double> weights, int64_t num_samples = 0,
|
||||
bool replacement = true);
|
||||
|
||||
/* ####################################### Derived Sampler classes ################################# */
|
||||
class DistributedSamplerObj : public SamplerObj {
|
||||
public:
|
||||
DistributedSamplerObj(int64_t num_shards, int64_t shard_id, bool shuffle, int64_t num_samples, uint32_t seed,
|
||||
int64_t offset, bool even_dist);
|
||||
/// \brief Constructor
|
||||
/// \param[in] num_shards - Number of shards to divide the dataset into.
|
||||
/// \param[in] shard_id - Shard ID of the current shard within num_shards.
|
||||
/// \param[in] shuffle - If true, the indices are shuffled.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \param[in] seed - The seed in use when shuffle is true.
|
||||
/// \param[in] offset - The starting position where access to elements in the dataset begins.
|
||||
/// \param[in] even_dist - If true, each shard would return the same number of rows (default to true).
|
||||
/// If false the total rows returned by all the shards would not have overlap.
|
||||
explicit DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle = true, int64_t num_samples = 0,
|
||||
uint32_t seed = 1, int64_t offset = -1, bool even_dist = true);
|
||||
/// \brief Destructor.
|
||||
~DistributedSampler() = default;
|
||||
|
||||
virtual ~DistributedSamplerObj() = default;
|
||||
|
||||
std::shared_ptr<SamplerRT> SamplerBuild() override;
|
||||
|
||||
std::shared_ptr<SamplerObj> SamplerCopy() override {
|
||||
auto sampler = std::make_shared<DistributedSamplerObj>(num_shards_, shard_id_, shuffle_, num_samples_, seed_,
|
||||
offset_, even_dist_);
|
||||
for (auto child : children_) {
|
||||
sampler->AddChildSampler(child);
|
||||
}
|
||||
return sampler;
|
||||
}
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
/// \brief Function to get the shard id of sampler
|
||||
/// \return The shard id of sampler
|
||||
int64_t ShardId() override { return shard_id_; }
|
||||
protected:
|
||||
/// \brief Function to convert a Sampler into an IR SamplerObj.
|
||||
/// \return shared pointer to the newly created SamplerObj.
|
||||
std::shared_ptr<SamplerObj> Parse() override;
|
||||
|
||||
private:
|
||||
int64_t num_shards_;
|
||||
|
|
@ -174,23 +90,26 @@ class DistributedSamplerObj : public SamplerObj {
|
|||
bool even_dist_;
|
||||
};
|
||||
|
||||
class PKSamplerObj : public SamplerObj {
|
||||
/// \brief A class to represent a PK Sampler in the data pipeline.
|
||||
/// \notes Samples K elements for each P class in the dataset.
|
||||
/// This will sample all classes.
|
||||
class PKSampler : public Sampler {
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
public:
|
||||
PKSamplerObj(int64_t num_val, bool shuffle, int64_t num_samples);
|
||||
/// \brief Constructor
|
||||
/// \param[in] num_val - Number of elements to sample for each class.
|
||||
/// \param[in] shuffle - If true, the class IDs are shuffled.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
explicit PKSampler(int64_t num_val, bool shuffle = false, int64_t num_samples = 0);
|
||||
|
||||
virtual ~PKSamplerObj() = default;
|
||||
/// \brief Destructor.
|
||||
~PKSampler() = default;
|
||||
|
||||
std::shared_ptr<SamplerRT> SamplerBuild() override;
|
||||
|
||||
std::shared_ptr<SamplerObj> SamplerCopy() override {
|
||||
auto sampler = std::make_shared<PKSamplerObj>(num_val_, shuffle_, num_samples_);
|
||||
for (auto child : children_) {
|
||||
sampler->AddChildSampler(child);
|
||||
}
|
||||
return sampler;
|
||||
}
|
||||
|
||||
Status ValidateParams() override;
|
||||
protected:
|
||||
/// \brief Function to convert a Sampler into an IR SamplerObj.
|
||||
/// \return shared pointer to the newly created SamplerObj.
|
||||
std::shared_ptr<SamplerObj> Parse() override;
|
||||
|
||||
private:
|
||||
int64_t num_val_;
|
||||
|
|
@ -198,131 +117,120 @@ class PKSamplerObj : public SamplerObj {
|
|||
int64_t num_samples_;
|
||||
};
|
||||
|
||||
class PreBuiltSamplerObj : public SamplerObj {
|
||||
/// \brief A class to represent a Random Sampler in the data pipeline.
|
||||
/// \notes Samples the elements randomly.
|
||||
class RandomSampler : public Sampler {
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
public:
|
||||
explicit PreBuiltSamplerObj(std::shared_ptr<SamplerRT> sampler);
|
||||
/// \brief Constructor
|
||||
/// \param[in] replacement - If true, put the sample ID back for the next draw.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
explicit RandomSampler(bool replacement = false, int64_t num_samples = 0);
|
||||
|
||||
~PreBuiltSamplerObj() = default;
|
||||
/// \brief Destructor.
|
||||
~RandomSampler() = default;
|
||||
|
||||
std::shared_ptr<SamplerRT> SamplerBuild() override;
|
||||
|
||||
std::shared_ptr<SamplerObj> SamplerCopy() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
private:
|
||||
std::shared_ptr<SamplerRT> sp_;
|
||||
};
|
||||
|
||||
class RandomSamplerObj : public SamplerObj {
|
||||
public:
|
||||
RandomSamplerObj(bool replacement, int64_t num_samples, bool reshuffle_each_epoch = true);
|
||||
|
||||
virtual ~RandomSamplerObj() = default;
|
||||
|
||||
std::shared_ptr<SamplerRT> SamplerBuild() override;
|
||||
|
||||
std::shared_ptr<SamplerObj> SamplerCopy() override {
|
||||
auto sampler = std::make_shared<RandomSamplerObj>(replacement_, num_samples_, reshuffle_each_epoch_);
|
||||
for (auto child : children_) {
|
||||
sampler->AddChildSampler(child);
|
||||
}
|
||||
return sampler;
|
||||
}
|
||||
|
||||
Status ValidateParams() override;
|
||||
protected:
|
||||
/// \brief Function to convert a Sampler into an IR SamplerObj.
|
||||
/// \return shared pointer to the newly created SamplerObj.
|
||||
std::shared_ptr<SamplerObj> Parse() override;
|
||||
|
||||
private:
|
||||
bool replacement_;
|
||||
int64_t num_samples_;
|
||||
bool reshuffle_each_epoch_;
|
||||
};
|
||||
|
||||
class SequentialSamplerObj : public SamplerObj {
|
||||
/// \brief A class to represent a Sequential Sampler in the data pipeline.
|
||||
/// \notes Samples the dataset elements sequentially, same as not having a sampler.
|
||||
class SequentialSampler : public Sampler {
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
public:
|
||||
SequentialSamplerObj(int64_t start_index, int64_t num_samples);
|
||||
/// \brief Constructor
|
||||
/// \param[in] start_index - Index to start sampling at (default to start at first id).
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
explicit SequentialSampler(int64_t start_index = 0, int64_t num_samples = 0);
|
||||
|
||||
virtual ~SequentialSamplerObj() = default;
|
||||
/// \brief Destructor.
|
||||
~SequentialSampler() = default;
|
||||
|
||||
std::shared_ptr<SamplerRT> SamplerBuild() override;
|
||||
|
||||
std::shared_ptr<SamplerObj> SamplerCopy() override {
|
||||
auto sampler = std::make_shared<SequentialSamplerObj>(start_index_, num_samples_);
|
||||
for (auto child : children_) {
|
||||
sampler->AddChildSampler(child);
|
||||
}
|
||||
return sampler;
|
||||
}
|
||||
|
||||
Status ValidateParams() override;
|
||||
protected:
|
||||
/// \brief Function to convert a Sampler into an IR SamplerObj.
|
||||
/// \return shared pointer to the newly created SamplerObj.
|
||||
std::shared_ptr<SamplerObj> Parse() override;
|
||||
|
||||
private:
|
||||
int64_t start_index_;
|
||||
int64_t num_samples_;
|
||||
};
|
||||
|
||||
class SubsetSamplerObj : public SamplerObj {
|
||||
/// \brief A class to represent a Subset Sampler in the data pipeline.
|
||||
/// \notes Samples the elements from a sequence of indices.
|
||||
class SubsetSampler : public Sampler {
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
public:
|
||||
SubsetSamplerObj(std::vector<int64_t> indices, int64_t num_samples);
|
||||
/// \brief Constructor
|
||||
/// \param[in] indices - A vector sequence of indices.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
explicit SubsetSampler(std::vector<int64_t> indices, int64_t num_samples = 0);
|
||||
|
||||
virtual ~SubsetSamplerObj() = default;
|
||||
|
||||
std::shared_ptr<SamplerRT> SamplerBuild() override;
|
||||
|
||||
std::shared_ptr<SamplerObj> SamplerCopy() override {
|
||||
auto sampler = std::make_shared<SubsetSamplerObj>(indices_, num_samples_);
|
||||
for (auto child : children_) {
|
||||
sampler->AddChildSampler(child);
|
||||
}
|
||||
return sampler;
|
||||
}
|
||||
|
||||
Status ValidateParams() override;
|
||||
/// \brief Destructor.
|
||||
~SubsetSampler() = default;
|
||||
|
||||
protected:
|
||||
const std::vector<int64_t> indices_;
|
||||
/// \brief Function to convert a Sampler into an IR SamplerObj.
|
||||
/// \return shared pointer to the newly created SamplerObj.
|
||||
std::shared_ptr<SamplerObj> Parse() override;
|
||||
|
||||
std::vector<int64_t> indices_;
|
||||
int64_t num_samples_;
|
||||
};
|
||||
|
||||
class SubsetRandomSamplerObj : public SubsetSamplerObj {
|
||||
/// \brief A class to represent a Subset Random Sampler in the data pipeline.
|
||||
/// \notes Samples the elements randomly from a sequence of indices.
|
||||
class SubsetRandomSampler : public SubsetSampler {
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
public:
|
||||
SubsetRandomSamplerObj(std::vector<int64_t> indices, int64_t num_samples);
|
||||
/// \brief Constructor
|
||||
/// \param[in] indices - A vector sequence of indices.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
explicit SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples = 0);
|
||||
|
||||
~SubsetRandomSamplerObj() = default;
|
||||
/// \brief Destructor.
|
||||
~SubsetRandomSampler() = default;
|
||||
|
||||
std::shared_ptr<SamplerRT> SamplerBuild() override;
|
||||
|
||||
std::shared_ptr<SamplerObj> SamplerCopy() override {
|
||||
auto sampler = std::make_shared<SubsetRandomSamplerObj>(indices_, num_samples_);
|
||||
for (auto child : children_) {
|
||||
sampler->AddChildSampler(child);
|
||||
}
|
||||
return sampler;
|
||||
}
|
||||
|
||||
private:
|
||||
protected:
|
||||
/// \brief Function to convert a Sampler into an IR SamplerObj.
|
||||
/// \return shared pointer to the newly created SamplerObj.
|
||||
std::shared_ptr<SamplerObj> Parse() override;
|
||||
};
|
||||
|
||||
class WeightedRandomSamplerObj : public SamplerObj {
|
||||
/// \brief A class to represent a Weighted Random Sampler in the data pipeline.
|
||||
/// \notes Samples the elements from [0, len(weights) - 1] randomly with the given
|
||||
/// weights (probabilities).
|
||||
class WeightedRandomSampler : public Sampler {
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
public:
|
||||
explicit WeightedRandomSamplerObj(std::vector<double> weights, int64_t num_samples = 0, bool replacement = true);
|
||||
/// \brief Constructor
|
||||
/// \param[in] weights - A vector sequence of weights, not necessarily summing up to 1.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \param[in] replacement - If true, put the sample ID back for the next draw.
|
||||
explicit WeightedRandomSampler(std::vector<double> weights, int64_t num_samples = 0, bool replacement = true);
|
||||
|
||||
virtual ~WeightedRandomSamplerObj() = default;
|
||||
/// \brief Destructor.
|
||||
~WeightedRandomSampler() = default;
|
||||
|
||||
std::shared_ptr<SamplerRT> SamplerBuild() override;
|
||||
|
||||
std::shared_ptr<SamplerObj> SamplerCopy() override {
|
||||
auto sampler = std::make_shared<WeightedRandomSamplerObj>(weights_, num_samples_, replacement_);
|
||||
for (auto child : children_) {
|
||||
sampler->AddChildSampler(child);
|
||||
}
|
||||
return sampler;
|
||||
}
|
||||
|
||||
Status ValidateParams() override;
|
||||
protected:
|
||||
/// \brief Function to convert a Sampler into an IR SamplerObj.
|
||||
/// \return shared pointer to the newly created SamplerObj.
|
||||
std::shared_ptr<SamplerObj> Parse() override;
|
||||
|
||||
private:
|
||||
const std::vector<double> weights_;
|
||||
std::vector<double> weights_;
|
||||
int64_t num_samples_;
|
||||
bool replacement_;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -20,231 +20,178 @@
|
|||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "include/constants.h"
|
||||
#include "include/api/dual_abi_helper.h"
|
||||
#include "include/api/status.h"
|
||||
#include "include/constants.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
class TensorOp;
|
||||
class TensorOperation;
|
||||
|
||||
// Char arrays storing name of corresponding classes (in alphabetical order)
|
||||
constexpr char kComposeOperation[] = "Compose";
|
||||
constexpr char kDuplicateOperation[] = "Duplicate";
|
||||
constexpr char kOneHotOperation[] = "OneHot";
|
||||
constexpr char kPreBuiltOperation[] = "PreBuilt";
|
||||
constexpr char kRandomApplyOperation[] = "RandomApply";
|
||||
constexpr char kRandomChoiceOperation[] = "RandomChoice";
|
||||
constexpr char kRandomSelectSubpolicyOperation[] = "RandomSelectSubpolicy";
|
||||
constexpr char kTypeCastOperation[] = "TypeCast";
|
||||
constexpr char kUniqueOperation[] = "Unique";
|
||||
|
||||
// Abstract class to represent a dataset in the data pipeline.
|
||||
class TensorOperation : public std::enable_shared_from_this<TensorOperation> {
|
||||
// Abstract class to represent a tensor transform operation in the data pipeline.
|
||||
/// \class TensorTransform transforms.h
|
||||
/// \brief A base class to represent a tensor transform operation in the data pipeline.
|
||||
class TensorTransform : public std::enable_shared_from_this<TensorTransform> {
|
||||
public:
|
||||
/// \brief Constructor
|
||||
TensorOperation() : random_op_(false) {}
|
||||
|
||||
/// \brief Constructor
|
||||
explicit TensorOperation(bool random) : random_op_(random) {}
|
||||
TensorTransform() {}
|
||||
|
||||
/// \brief Destructor
|
||||
~TensorOperation() = default;
|
||||
~TensorTransform() = default;
|
||||
|
||||
/// \brief Pure virtual function to convert a TensorOperation class into a runtime TensorOp object.
|
||||
/// \return shared pointer to the newly created TensorOp.
|
||||
virtual std::shared_ptr<TensorOp> Build() = 0;
|
||||
/// \brief Pure virtual function to convert a TensorTransform class into a IR TensorOperation object.
|
||||
/// \return shared pointer to the newly created TensorOperation.
|
||||
virtual std::shared_ptr<TensorOperation> Parse() = 0;
|
||||
|
||||
virtual Status ValidateParams() = 0;
|
||||
|
||||
virtual std::string Name() const = 0;
|
||||
|
||||
/// \brief Check whether the operation is deterministic.
|
||||
/// \return true if this op is a random op (returns non-deterministic result e.g. RandomCrop)
|
||||
bool IsRandomOp() const { return random_op_; }
|
||||
|
||||
protected:
|
||||
bool random_op_;
|
||||
/// \brief Virtual function to convert a TensorTransform class into a IR TensorOperation object.
|
||||
/// \param[in] env A string to determine the running environment
|
||||
/// \return shared pointer to the newly created TensorOperation.
|
||||
virtual std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) { return nullptr; }
|
||||
};
|
||||
|
||||
// Helper function to validate fill value
|
||||
Status ValidateVectorFillvalue(const std::string &transform_name, const std::vector<uint8_t> &fill_value);
|
||||
|
||||
// Helper function to validate probability
|
||||
Status ValidateProbability(const std::string &transform_name, const float &probability);
|
||||
|
||||
// Helper function to validate padding
|
||||
Status ValidateVectorPadding(const std::string &transform_name, const std::vector<int32_t> &padding);
|
||||
|
||||
// Helper function to validate size
|
||||
Status ValidateVectorPositive(const std::string &transform_name, const std::vector<int32_t> &size);
|
||||
|
||||
// Helper function to validate transforms
|
||||
Status ValidateVectorTransforms(const std::string &transform_name,
|
||||
const std::vector<std::shared_ptr<TensorOperation>> &transforms);
|
||||
|
||||
// Helper function to compare float value
|
||||
bool CmpFloat(const float &a, const float &b, float epsilon = 0.0000000001f);
|
||||
|
||||
// Transform operations for performing data transformation.
|
||||
namespace transforms {
|
||||
|
||||
// Transform Op classes (in alphabetical order)
|
||||
class ComposeOperation;
|
||||
class DuplicateOperation;
|
||||
class OneHotOperation;
|
||||
class PreBuiltOperation;
|
||||
class RandomApplyOperation;
|
||||
class RandomChoiceOperation;
|
||||
class TypeCastOperation;
|
||||
|
||||
/// \brief Function to create a Compose TensorOperation.
|
||||
/// \brief Compose Op.
|
||||
/// \notes Compose a list of transforms into a single transform.
|
||||
/// \param[in] transforms A vector of transformations to be applied.
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<ComposeOperation> Compose(const std::vector<std::shared_ptr<TensorOperation>> &transforms);
|
||||
class Compose : public TensorTransform {
|
||||
public:
|
||||
/// \brief Constructor.
|
||||
/// \param[in] transforms A vector of transformations to be applied.
|
||||
explicit Compose(const std::vector<TensorTransform *> &transforms);
|
||||
explicit Compose(const std::vector<std::shared_ptr<TensorTransform>> &transforms);
|
||||
explicit Compose(const std::vector<std::reference_wrapper<TensorTransform>> &transforms);
|
||||
|
||||
/// \brief Function to create a Duplicate TensorOperation.
|
||||
/// \brief Destructor
|
||||
~Compose() = default;
|
||||
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Duplicate Op.
|
||||
/// \notes Duplicate the input tensor to a new output tensor.
|
||||
/// The input tensor is carried over to the output list.
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<DuplicateOperation> Duplicate();
|
||||
class Duplicate : public TensorTransform {
|
||||
public:
|
||||
/// \brief Constructor.
|
||||
Duplicate();
|
||||
|
||||
/// \brief Function to create a OneHot TensorOperation.
|
||||
/// \brief Destructor
|
||||
~Duplicate() = default;
|
||||
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
};
|
||||
|
||||
/// \brief OneHot Op.
|
||||
/// \notes Convert the labels into OneHot format.
|
||||
/// \param[in] num_classes number of classes.
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<OneHotOperation> OneHot(int32_t num_classes);
|
||||
class OneHot : public TensorTransform {
|
||||
public:
|
||||
/// \brief Constructor.
|
||||
/// \param[in] num_classes number of classes.
|
||||
explicit OneHot(int32_t num_classes);
|
||||
|
||||
/// \brief Function to create a RandomApply TensorOperation.
|
||||
/// \brief Destructor
|
||||
~OneHot() = default;
|
||||
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomApply Op.
|
||||
/// \notes Randomly perform a series of transforms with a given probability.
|
||||
/// \param[in] transforms A vector of transformations to be applied.
|
||||
/// \param[in] prob The probability to apply the transformation list (default=0.5)
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<RandomApplyOperation> RandomApply(const std::vector<std::shared_ptr<TensorOperation>> &transforms,
|
||||
double prob = 0.5);
|
||||
class RandomApply : public TensorTransform {
|
||||
public:
|
||||
/// \brief Constructor.
|
||||
/// \param[in] transforms A vector of transformations to be applied.
|
||||
/// \param[in] prob The probability to apply the transformation list (default=0.5)
|
||||
explicit RandomApply(const std::vector<TensorTransform *> &transforms, double prob = 0.5);
|
||||
explicit RandomApply(const std::vector<std::shared_ptr<TensorTransform>> &transforms, double prob = 0.5);
|
||||
explicit RandomApply(const std::vector<std::reference_wrapper<TensorTransform>> &transforms, double prob = 0.5);
|
||||
|
||||
/// \brief Function to create a RandomChoice TensorOperation.
|
||||
/// \brief Destructor
|
||||
~RandomApply() = default;
|
||||
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief RandomChoice Op.
|
||||
/// \notes Randomly selects one transform from a list of transforms to perform operation.
|
||||
/// \param[in] transforms A vector of transformations to be chosen from to apply.
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<RandomChoiceOperation> RandomChoice(const std::vector<std::shared_ptr<TensorOperation>> &transforms);
|
||||
class RandomChoice : public TensorTransform {
|
||||
public:
|
||||
/// \brief Constructor.
|
||||
/// \param[in] transforms A vector of transformations to be chosen from to apply.
|
||||
explicit RandomChoice(const std::vector<TensorTransform *> &transforms);
|
||||
explicit RandomChoice(const std::vector<std::shared_ptr<TensorTransform>> &transforms);
|
||||
explicit RandomChoice(const std::vector<std::reference_wrapper<TensorTransform>> &transforms);
|
||||
|
||||
/// \brief Function to create a TypeCast TensorOperation.
|
||||
/// \brief Destructor
|
||||
~RandomChoice() = default;
|
||||
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief TypeCast Op.
|
||||
/// \notes Tensor operation to cast to a given MindSpore data type.
|
||||
/// \param[in] data_type mindspore.dtype to be cast to.
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<TypeCastOperation> TypeCast(std::string data_type);
|
||||
|
||||
/* ####################################### Derived TensorOperation classes ################################# */
|
||||
|
||||
class ComposeOperation : public TensorOperation {
|
||||
class TypeCast : public TensorTransform {
|
||||
public:
|
||||
explicit ComposeOperation(const std::vector<std::shared_ptr<TensorOperation>> &transforms);
|
||||
/// \brief Constructor.
|
||||
/// \param[in] data_type mindspore.dtype to be cast to.
|
||||
explicit TypeCast(std::string data_type) : TypeCast(StringToChar(data_type)) {}
|
||||
|
||||
~ComposeOperation() = default;
|
||||
explicit TypeCast(const std::vector<char> &data_type);
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
/// \brief Destructor
|
||||
~TypeCast() = default;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kComposeOperation; }
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<TensorOperation>> transforms_;
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
class DuplicateOperation : public TensorOperation {
|
||||
/// \brief Unique Op.
|
||||
/// \notes Return an output tensor containing all the unique elements of the input tensor in
|
||||
/// the same order that they occur in the input tensor.
|
||||
class Unique : public TensorTransform {
|
||||
public:
|
||||
DuplicateOperation() = default;
|
||||
/// \brief Constructor.
|
||||
Unique();
|
||||
|
||||
~DuplicateOperation() = default;
|
||||
/// \brief Destructor
|
||||
~Unique() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kDuplicateOperation; }
|
||||
};
|
||||
|
||||
class OneHotOperation : public TensorOperation {
|
||||
public:
|
||||
explicit OneHotOperation(int32_t num_classes_);
|
||||
|
||||
~OneHotOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kOneHotOperation; }
|
||||
|
||||
private:
|
||||
float num_classes_;
|
||||
};
|
||||
|
||||
class PreBuiltOperation : public TensorOperation {
|
||||
public:
|
||||
explicit PreBuiltOperation(std::shared_ptr<TensorOp> tensor_op);
|
||||
|
||||
~PreBuiltOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kPreBuiltOperation; }
|
||||
|
||||
private:
|
||||
std::shared_ptr<TensorOp> op_;
|
||||
};
|
||||
|
||||
class RandomApplyOperation : public TensorOperation {
|
||||
public:
|
||||
explicit RandomApplyOperation(const std::vector<std::shared_ptr<TensorOperation>> &transforms, double prob);
|
||||
|
||||
~RandomApplyOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kRandomApplyOperation; }
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<TensorOperation>> transforms_;
|
||||
double prob_;
|
||||
};
|
||||
|
||||
class RandomChoiceOperation : public TensorOperation {
|
||||
public:
|
||||
explicit RandomChoiceOperation(const std::vector<std::shared_ptr<TensorOperation>> &transforms);
|
||||
|
||||
~RandomChoiceOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kRandomChoiceOperation; }
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<TensorOperation>> transforms_;
|
||||
};
|
||||
class TypeCastOperation : public TensorOperation {
|
||||
public:
|
||||
explicit TypeCastOperation(std::string data_type);
|
||||
|
||||
~TypeCastOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kTypeCastOperation; }
|
||||
|
||||
private:
|
||||
std::string data_type_;
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
};
|
||||
} // namespace transforms
|
||||
} // namespace dataset
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@
|
|||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "include/constants.h"
|
||||
#include "include/transforms.h"
|
||||
|
||||
namespace mindspore {
|
||||
|
|
@ -31,167 +31,210 @@ namespace dataset {
|
|||
// Transform operations for performing computer vision.
|
||||
namespace vision {
|
||||
|
||||
// Char arrays storing name of corresponding classes (in alphabetical order)
|
||||
constexpr char kCenterCropOperation[] = "CenterCrop";
|
||||
constexpr char kCropOperation[] = "Crop";
|
||||
constexpr char kDecodeOperation[] = "Decode";
|
||||
constexpr char kNormalizeOperation[] = "Normalize";
|
||||
constexpr char kResizeOperation[] = "Resize";
|
||||
constexpr char kRotateOperation[] = "Rotate";
|
||||
// Transform Op classes (in alphabetical order)
|
||||
class CenterCropOperation;
|
||||
class CropOperation;
|
||||
class DecodeOperation;
|
||||
class NormalizeOperation;
|
||||
class ResizeOperation;
|
||||
// Forward Declarations
|
||||
class RotateOperation;
|
||||
|
||||
/// \brief Function to create a CenterCrop TensorOperation.
|
||||
/// \brief Affine TensorTransform.
|
||||
/// \notes Apply affine transform on input image.
|
||||
class Affine : public TensorTransform {
|
||||
public:
|
||||
/// \brief Constructor.
|
||||
/// \param[in] degrees The degrees to rotate the image by
|
||||
/// \param[in] translation The value representing vertical and horizontal translation (default = {0.0, 0.0})
|
||||
/// The first value represent the x axis translation while the second represents y axis translation.
|
||||
/// \param[in] scale The scaling factor for the image (default = 0.0)
|
||||
/// \param[in] shear A float vector of size 2, representing the shear degrees (default = {0.0, 0.0})
|
||||
/// \param[in] interpolation An enum for the mode of interpolation
|
||||
/// \param[in] fill_value A vector representing the value to fill the area outside the transform
|
||||
/// in the output image. If 1 value is provided, it is used for all RGB channels.
|
||||
/// If 3 values are provided, it is used to fill R, G, B channels respectively.
|
||||
explicit Affine(float_t degrees, const std::vector<float> &translation = {0.0, 0.0}, float scale = 0.0,
|
||||
const std::vector<float> &shear = {0.0, 0.0},
|
||||
InterpolationMode interpolation = InterpolationMode::kNearestNeighbour,
|
||||
const std::vector<uint8_t> &fill_value = {0, 0, 0});
|
||||
|
||||
/// \brief Destructor.
|
||||
~Affine() = default;
|
||||
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
/// \brief CenterCrop TensorTransform.
|
||||
/// \notes Crops the input image at the center to the given size.
|
||||
/// \param[in] size A vector representing the output size of the cropped image.
|
||||
/// If size is a single value, a square crop of size (size, size) is returned.
|
||||
/// If size has 2 values, it should be (height, width).
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<CenterCropOperation> CenterCrop(std::vector<int32_t> size);
|
||||
class CenterCrop : public TensorTransform {
|
||||
public:
|
||||
/// \brief Constructor.
|
||||
/// \param[in] size A vector representing the output size of the cropped image.
|
||||
/// If size is a single value, a square crop of size (size, size) is returned.
|
||||
/// If size has 2 values, it should be (height, width).
|
||||
explicit CenterCrop(std::vector<int32_t> size);
|
||||
|
||||
/// \brief Function to create a Crop TensorOp
|
||||
/// \brief Destructor.
|
||||
~CenterCrop() = default;
|
||||
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) override;
|
||||
|
||||
private:
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Crop TensorTransform.
|
||||
/// \notes Crop an image based on location and crop size
|
||||
/// \param[in] coordinates Starting location of crop. Must be a vector of two values, in the form of {x_coor, y_coor}
|
||||
/// \param[in] size Size of the cropped area.
|
||||
/// If size is a single value, a square crop of size (size, size) is returned.
|
||||
/// If size has 2 values, it should be (height, width).
|
||||
/// \return Shared pointer to the current TensorOp
|
||||
std::shared_ptr<CropOperation> Crop(std::vector<int32_t> coordinates, std::vector<int32_t> size);
|
||||
class Crop : public TensorTransform {
|
||||
public:
|
||||
/// \brief Constructor.
|
||||
/// \param[in] coordinates Starting location of crop. Must be a vector of two values, in the form of {x_coor, y_coor}
|
||||
/// \param[in] size Size of the cropped area.
|
||||
/// If size is a single value, a square crop of size (size, size) is returned.
|
||||
/// If size has 2 values, it should be (height, width).
|
||||
Crop(std::vector<int32_t> coordinates, std::vector<int32_t> size);
|
||||
|
||||
/// \brief Function to create a Decode TensorOperation.
|
||||
/// \brief Destructor.
|
||||
~Crop() = default;
|
||||
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Decode TensorTransform.
|
||||
/// \notes Decode the input image in RGB mode.
|
||||
/// \param[in] rgb A boolean of whether to decode in RGB mode or not.
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<DecodeOperation> Decode(bool rgb = true);
|
||||
class Decode : public TensorTransform {
|
||||
public:
|
||||
/// \brief Constructor.
|
||||
/// \param[in] rgb A boolean of whether to decode in RGB mode or not.
|
||||
explicit Decode(bool rgb = true);
|
||||
|
||||
/// \brief Function to create a Normalize TensorOperation.
|
||||
/// \brief Destructor.
|
||||
~Decode() = default;
|
||||
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) override;
|
||||
|
||||
private:
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Normalize TensorTransform.
|
||||
/// \notes Normalize the input image with respect to mean and standard deviation.
|
||||
/// \param[in] mean A vector of mean values for each channel, w.r.t channel order.
|
||||
/// The mean values must be in range [0.0, 255.0].
|
||||
/// \param[in] std A vector of standard deviations for each channel, w.r.t. channel order.
|
||||
/// The standard deviation values must be in range (0.0, 255.0]
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<NormalizeOperation> Normalize(std::vector<float> mean, std::vector<float> std);
|
||||
class Normalize : public TensorTransform {
|
||||
public:
|
||||
/// \brief Constructor.
|
||||
/// \param[in] mean A vector of mean values for each channel, w.r.t channel order.
|
||||
/// The mean values must be in range [0.0, 255.0].
|
||||
/// \param[in] std A vector of standard deviations for each channel, w.r.t. channel order.
|
||||
/// The standard deviation values must be in range (0.0, 255.0]
|
||||
Normalize(std::vector<float> mean, std::vector<float> std);
|
||||
|
||||
/// \brief Function to create a Resize TensorOperation.
|
||||
/// \brief Destructor.
|
||||
~Normalize() = default;
|
||||
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
class RandomAffine : public TensorTransform {
|
||||
public:
|
||||
/// \brief Constructor.
|
||||
/// \param[in] degrees A float vector of size 2, representing the starting and ending degree
|
||||
/// \param[in] translate_range A float vector of size 2 or 4, representing percentages of translation on x and y axes.
|
||||
/// if size is 2, (min_dx, max_dx, 0, 0)
|
||||
/// if size is 4, (min_dx, max_dx, min_dy, max_dy)
|
||||
/// all values are in range [-1, 1]
|
||||
/// \param[in] scale_range A float vector of size 2, representing the starting and ending scales in the range.
|
||||
/// \param[in] shear_ranges A float vector of size 2 or 4, representing the starting and ending shear degrees
|
||||
/// vertically and horizontally.
|
||||
/// if size is 2, (min_shear_x, max_shear_x, 0, 0)
|
||||
/// if size is 4, (min_shear_x, max_shear_x, min_shear_y, max_shear_y)
|
||||
/// \param[in] interpolation An enum for the mode of interpolation
|
||||
/// \param[in] fill_value A vector representing the value to fill the area outside the transform
|
||||
/// in the output image. If 1 value is provided, it is used for all RGB channels.
|
||||
/// If 3 values are provided, it is used to fill R, G, B channels respectively.
|
||||
explicit RandomAffine(const std::vector<float_t> °rees,
|
||||
const std::vector<float_t> &translate_range = {0.0, 0.0, 0.0, 0.0},
|
||||
const std::vector<float_t> &scale_range = {1.0, 1.0},
|
||||
const std::vector<float_t> &shear_ranges = {0.0, 0.0, 0.0, 0.0},
|
||||
InterpolationMode interpolation = InterpolationMode::kNearestNeighbour,
|
||||
const std::vector<uint8_t> &fill_value = {0, 0, 0});
|
||||
|
||||
/// \brief Destructor.
|
||||
~RandomAffine() = default;
|
||||
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Resize TensorTransform.
|
||||
/// \notes Resize the input image to the given size.
|
||||
/// \param[in] size A vector representing the output size of the resized image.
|
||||
/// If size is a single value, the image will be resized to this value with
|
||||
/// the same image aspect ratio. If size has 2 values, it should be (height, width).
|
||||
/// \param[in] interpolation An enum for the mode of interpolation
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<ResizeOperation> Resize(std::vector<int32_t> size,
|
||||
InterpolationMode interpolation = InterpolationMode::kLinear);
|
||||
/// \brief Applies an rotate transformation to an image.
|
||||
class Resize : public TensorTransform {
|
||||
public:
|
||||
/// \brief Constructor.
|
||||
/// \param[in] size A vector representing the output size of the resized image.
|
||||
/// If size is a single value, the image will be resized to this value with
|
||||
/// the same image aspect ratio. If size has 2 values, it should be (height, width).
|
||||
/// \param[in] interpolation An enum for the mode of interpolation
|
||||
explicit Resize(std::vector<int32_t> size, InterpolationMode interpolation = InterpolationMode::kLinear);
|
||||
|
||||
/// \brief Destructor.
|
||||
~Resize() = default;
|
||||
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) override;
|
||||
|
||||
private:
|
||||
struct Data;
|
||||
std::shared_ptr<Data> data_;
|
||||
};
|
||||
|
||||
/// \brief Rotate TensorTransform.
|
||||
/// \notes Rotate the input image using a specified angle id.
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<RotateOperation> Rotate();
|
||||
|
||||
class CenterCropOperation : public TensorOperation {
|
||||
class Rotate : public TensorTransform {
|
||||
public:
|
||||
explicit CenterCropOperation(std::vector<int32_t> size);
|
||||
/// \brief Constructor.
|
||||
Rotate();
|
||||
|
||||
~CenterCropOperation() = default;
|
||||
/// \brief Destructor.
|
||||
~Rotate() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kCenterCropOperation; }
|
||||
/// \brief Function to convert TensorTransform object into a TensorOperation object.
|
||||
/// \return Shared pointer to TensorOperation object.
|
||||
std::shared_ptr<TensorOperation> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> size_;
|
||||
std::shared_ptr<RotateOperation> op_;
|
||||
};
|
||||
|
||||
class CropOperation : public TensorOperation {
|
||||
public:
|
||||
CropOperation(std::vector<int32_t> coordinates, std::vector<int32_t> size);
|
||||
|
||||
~CropOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kCropOperation; }
|
||||
|
||||
private:
|
||||
std::vector<int32_t> coordinates_;
|
||||
std::vector<int32_t> size_;
|
||||
};
|
||||
class DecodeOperation : public TensorOperation {
|
||||
public:
|
||||
explicit DecodeOperation(bool rgb = true);
|
||||
|
||||
~DecodeOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kDecodeOperation; }
|
||||
|
||||
private:
|
||||
bool rgb_;
|
||||
};
|
||||
|
||||
class NormalizeOperation : public TensorOperation {
|
||||
public:
|
||||
NormalizeOperation(std::vector<float> mean, std::vector<float> std);
|
||||
|
||||
~NormalizeOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kNormalizeOperation; }
|
||||
|
||||
private:
|
||||
std::vector<float> mean_;
|
||||
std::vector<float> std_;
|
||||
};
|
||||
|
||||
class ResizeOperation : public TensorOperation {
|
||||
public:
|
||||
explicit ResizeOperation(std::vector<int32_t> size,
|
||||
InterpolationMode interpolation_mode = InterpolationMode::kLinear);
|
||||
|
||||
~ResizeOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kResizeOperation; }
|
||||
|
||||
private:
|
||||
std::vector<int32_t> size_;
|
||||
InterpolationMode interpolation_;
|
||||
};
|
||||
|
||||
class RotateOperation : public TensorOperation {
|
||||
public:
|
||||
RotateOperation();
|
||||
|
||||
~RotateOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kRotateOperation; }
|
||||
|
||||
void setAngle(uint64_t angle_id);
|
||||
|
||||
private:
|
||||
std::shared_ptr<TensorOp> rotate_op;
|
||||
};
|
||||
} // namespace vision
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
|
|
|||
Loading…
Reference in New Issue