p12346870
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!23092 minddata sync code review 

Merge pull request !23092 from luoyang/issues
This commit is contained in:
i-robot 2021-09-17 09:45:04 +00:00 committed by Gitee
parent 78b6fd17d6 28d21c13a0
commit 53e64f65e5
51 changed files with 563 additions and 329 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/api/dual_abi_helper.h
View File

@ -151,6 +151,9 @@ inline std::map<std::string, T> PadInfoCharToString(const std::map<std::vector<c
template <class T>
inline void TensorMapCharToString(const std::map<std::vector<char>, T> *c, std::unordered_map<std::string, T> *s) {
if (c == nullptr || s == nullptr) {
return;
}
for (auto ch : *c) {
auto key = std::string(ch.first.begin(), ch.first.end());
auto val = ch.second;

4
mindspore/ccsrc/minddata/dataset/api/text.cc
View File

@ -314,9 +314,9 @@ std::shared_ptr<TensorOperation> RegexTokenizer::Parse() {
// SentencePieceTokenizer
struct SentencePieceTokenizer::Data {
Data(const std::shared_ptr<SentencePieceVocab> &vocab, SPieceTokenizerOutType out_type)
: vocab_(vocab), out_type_(out_type) {}
: vocab_(vocab), vocab_path_(""), out_type_(out_type) {}
Data(const std::vector<char> &vocab_path, SPieceTokenizerOutType out_type)
: vocab_path_(CharToString(vocab_path)), out_type_(out_type) {}
: vocab_(nullptr), vocab_path_(CharToString(vocab_path)), out_type_(out_type) {}
std::shared_ptr<SentencePieceVocab> vocab_;
std::string vocab_path_;
SPieceTokenizerOutType out_type_;

38
mindspore/ccsrc/minddata/dataset/api/vision.cc
View File

@ -194,8 +194,11 @@ std::shared_ptr<TensorOperation> CenterCrop::Parse(const MapTargetDevice &env) {
[](int32_t i) { return (uint32_t)i; });
return std::make_shared<DvppCropJpegOperation>(usize_);
#endif // ENABLE_ACL
} else if (env == MapTargetDevice::kCpu) {
return std::make_shared<CenterCropOperation>(data_->size_);
}
return std::make_shared<CenterCropOperation>(data_->size_);
MS_LOG(ERROR) << "Unsupported MapTargetDevice, only supported kCpu and kAscend310.";
return nullptr;
}
#ifndef ENABLE_ANDROID
@ -273,8 +276,11 @@ std::shared_ptr<TensorOperation> Decode::Parse(const MapTargetDevice &env) {
#ifdef ENABLE_ACL
return std::make_shared<DvppDecodeJpegOperation>();
#endif // ENABLE_ACL
} else if (env == MapTargetDevice::kCpu) {
return std::make_shared<DecodeOperation>(data_->rgb_);
}
return std::make_shared<DecodeOperation>(data_->rgb_);
MS_LOG(ERROR) << "Unsupported MapTargetDevice, only supported kCpu and kAscend310.";
return nullptr;
}
#ifdef ENABLE_ACL
@ -291,7 +297,11 @@ std::shared_ptr<TensorOperation> DvppDecodeResizeJpeg::Parse() {
}
std::shared_ptr<TensorOperation> DvppDecodeResizeJpeg::Parse(const MapTargetDevice &env) {
return std::make_shared<DvppDecodeResizeOperation>(data_->resize_);
if (env == MapTargetDevice::kAscend310) {
return std::make_shared<DvppDecodeResizeOperation>(data_->resize_);
}
MS_LOG(ERROR) << "Unsupported MapTargetDevice, only supported kAscend310.";
return nullptr;
}
// DvppDecodeResizeCrop Transform Operation.
@ -309,7 +319,11 @@ std::shared_ptr<TensorOperation> DvppDecodeResizeCropJpeg::Parse() {
}
std::shared_ptr<TensorOperation> DvppDecodeResizeCropJpeg::Parse(const MapTargetDevice &env) {
return std::make_shared<DvppDecodeResizeCropOperation>(data_->crop_, data_->resize_);
if (env == MapTargetDevice::kAscend310) {
return std::make_shared<DvppDecodeResizeCropOperation>(data_->crop_, data_->resize_);
}
MS_LOG(ERROR) << "Unsupported MapTargetDevice, only supported kAscend310.";
return nullptr;
}
// DvppDecodePng Transform Operation.
@ -318,7 +332,11 @@ DvppDecodePng::DvppDecodePng() {}
std::shared_ptr<TensorOperation> DvppDecodePng::Parse() { return std::make_shared<DvppDecodePngOperation>(); }
std::shared_ptr<TensorOperation> DvppDecodePng::Parse(const MapTargetDevice &env) {
return std::make_shared<DvppDecodePngOperation>();
if (env == MapTargetDevice::kAscend310) {
return std::make_shared<DvppDecodePngOperation>();
}
MS_LOG(ERROR) << "Unsupported MapTargetDevice, only supported kAscend310.";
return nullptr;
}
#endif // ENABLE_ACL
@ -389,8 +407,11 @@ std::shared_ptr<TensorOperation> Normalize::Parse(const MapTargetDevice &env) {
#ifdef ENABLE_ACL
return std::make_shared<DvppNormalizeOperation>(data_->mean_, data_->std_);
#endif // ENABLE_ACL
} else if (env == MapTargetDevice::kCpu) {
return std::make_shared<NormalizeOperation>(data_->mean_, data_->std_);
}
return std::make_shared<NormalizeOperation>(data_->mean_, data_->std_);
MS_LOG(ERROR) << "Unsupported MapTargetDevice, only supported kCpu and kAscend310.";
return nullptr;
}
#ifndef ENABLE_ANDROID
@ -828,8 +849,11 @@ std::shared_ptr<TensorOperation> Resize::Parse(const MapTargetDevice &env) {
[](int32_t i) { return (uint32_t)i; });
return std::make_shared<DvppResizeJpegOperation>(usize_);
#endif // ENABLE_ACL
} else if (env == MapTargetDevice::kCpu) {
return std::make_shared<ResizeOperation>(data_->size_, data_->interpolation_);
}
return std::make_shared<ResizeOperation>(data_->size_, data_->interpolation_);
MS_LOG(ERROR) << "Unsupported MapTargetDevice, only supported kCpu and kAscend310.";
return nullptr;
}
// ResizePreserveAR Transform Operation.

4
mindspore/ccsrc/minddata/dataset/core/tensor_helpers.cc
View File

@ -34,6 +34,10 @@ void IndexGeneratorHelper(int8_t depth, std::vector<dsize_t> *numbers,
int8_t new_depth = depth - 1;
// depth is always less than or equal to numbers->size() (based on the caller functions)
size_t curr_ind = static_cast<size_t>(numbers->size() - static_cast<size_t>(depth));
if (curr_ind >= slice_list.size()) {
MS_LOG(ERROR) << "The index is out of range in slice_list.";
return;
}
if (slice_list[curr_ind].slice_.valid()) {
dsize_t increment = slice_list[curr_ind].slice_.step_;

1
mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc
View File

@ -100,6 +100,7 @@ Status DatasetOp::RemoveChild(std::shared_ptr<DatasetOp> child) {
}
Status DatasetOp::InsertAsParent(std::shared_ptr<DatasetOp> to_add) {
RETURN_UNEXPECTED_IF_NULL(to_add);
for (auto &prev_parent : this->parent_) {
RETURN_IF_NOT_OK(prev_parent->RemoveChild(shared_from_this()));
RETURN_IF_NOT_OK(prev_parent->AddChild(to_add));

3
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc
View File

@ -270,6 +270,9 @@ Status CocoOp::ParseAnnotationIds() {
}
std::ifstream in(realpath.value());
if (!in.is_open()) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open annotation file: " + annotation_path_);
}
in >> js;
} catch (const std::exception &err) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open JSON file: " + annotation_path_ + ".");

2
mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc
View File

@ -33,6 +33,8 @@ namespace dataset {
// Constructor
ExecutionTree::ExecutionTree() : id_count_(0), tree_state_(kDeTStateInit) {
tg_ = std::make_unique<TaskGroup>();
root_ = nullptr;
prepare_flags_ = 0;
#ifndef ENABLE_SECURITY
profiling_manager_ = std::make_unique<ProfilingManager>(this);
#endif

7
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/build_sentence_piece_vocab_node.cc
View File

@ -70,6 +70,13 @@ Status BuildSentenceVocabNode::ValidateParams() {
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (model_type_ != SentencePieceModel::kUnigram && model_type_ != SentencePieceModel::kBpe &&
model_type_ != SentencePieceModel::kChar && model_type_ != SentencePieceModel::kWord) {
std::string err_msg = "BuildSentenceVocabNode: Invalid SentencePieceModel, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (vocab_size_ <= 0) {
std::string err_msg =
"BuildSentenceVocabNode: vocab_size should be positive, but got: " + std::to_string(vocab_size_);

6
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/clue_node.cc
View File

@ -60,6 +60,12 @@ Status CLUENode::ValidateParams() {
RETURN_IF_NOT_OK(ValidateStringValue("CLUENode", usage_, {"train", "test", "eval"}));
if (shuffle_ != ShuffleMode::kFalse && shuffle_ != ShuffleMode::kFiles && shuffle_ != ShuffleMode::kGlobal) {
std::string err_msg = "CLUENode: Invalid ShuffleMode, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (num_samples_ < 0) {
std::string err_msg = "CLUENode: Invalid number of samples: " + std::to_string(num_samples_);
MS_LOG(ERROR) << err_msg;

5
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/coco_node.cc
View File

@ -63,6 +63,11 @@ Status CocoNode::ValidateParams() {
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (access(annotation_file_.c_str(), R_OK) == -1) {
std::string err_msg = "CocoNode: No access to specified annotation file: " + annotation_file_;
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
RETURN_IF_NOT_OK(ValidateStringValue("CocoNode", task_, {"Detection", "Stuff", "Panoptic", "Keypoint"}));

6
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/csv_node.cc
View File

@ -70,6 +70,12 @@ Status CSVNode::ValidateParams() {
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (shuffle_ != ShuffleMode::kFalse && shuffle_ != ShuffleMode::kFiles && shuffle_ != ShuffleMode::kGlobal) {
std::string err_msg = "CSVNode: Invalid ShuffleMode, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (num_samples_ < 0) {
std::string err_msg = "CSVNode: Invalid number of samples: " + std::to_string(num_samples_);
MS_LOG(ERROR) << err_msg;

5
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/manifest_node.cc
View File

@ -77,6 +77,11 @@ Status ManifestNode::ValidateParams() {
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (access(dataset_file_.c_str(), R_OK) == -1) {
std::string err_msg = "ManifestNode: No access to specified annotation file: " + dataset_file_;
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
RETURN_IF_NOT_OK(ValidateDatasetSampler("ManifestNode", sampler_));

6
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/text_file_node.cc
View File

@ -58,6 +58,12 @@ Status TextFileNode::ValidateParams() {
RETURN_IF_NOT_OK(DatasetNode::ValidateParams());
RETURN_IF_NOT_OK(ValidateDatasetFilesParam("TextFileNode", dataset_files_));
if (shuffle_ != ShuffleMode::kFalse && shuffle_ != ShuffleMode::kFiles && shuffle_ != ShuffleMode::kGlobal) {
std::string err_msg = "TextFileNode: Invalid ShuffleMode, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (num_samples_ < 0) {
std::string err_msg = "TextFileNode: Invalid number of samples: " + std::to_string(num_samples_);
MS_LOG(ERROR) << err_msg;

7
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc
View File

@ -52,6 +52,13 @@ void TFRecordNode::Print(std::ostream &out) const {
// Validator for TFRecordNode
Status TFRecordNode::ValidateParams() {
RETURN_IF_NOT_OK(DatasetNode::ValidateParams());
if (shuffle_ != ShuffleMode::kFalse && shuffle_ != ShuffleMode::kFiles && shuffle_ != ShuffleMode::kGlobal) {
std::string err_msg = "TFRecordNode: Invalid ShuffleMode, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (dataset_files_.empty()) {
std::string err_msg = "TFRecordNode: dataset_files is not specified.";
MS_LOG(ERROR) << err_msg;

16
mindspore/ccsrc/minddata/dataset/include/dataset/data_helper.h
View File

@ -45,6 +45,7 @@ class DataHelper {
/// Creates the output directory if doesn't exist
/// \param[in] in_dir Image folder directory that takes in images
/// \param[in] out_dir Directory containing output json files
/// \return Status The status code returned
Status CreateAlbum(const std::string &in_dir, const std::string &out_dir) {
return CreateAlbumIF(StringToChar(in_dir), StringToChar(out_dir));
}
@ -339,6 +340,7 @@ class DataHelper {
}
/// \brief Write pointer to bin, use pointer to avoid memcpy
/// \note The value of `length`` must be equal to the length of `data`
/// \param[in] in_file File name to write to
/// \param[in] data Pointer to data
/// \param[in] length Length of values to write from pointer
@ -346,16 +348,22 @@ class DataHelper {
template <typename T>
Status WriteBinFile(const std::string &in_file, T *data, size_t length) {
try {
if (data == nullptr) {
return Status(kMDUnexpectedError, "input data can not be null");
}
std::ofstream o(in_file, std::ios::binary | std::ios::out);
if (!o.is_open()) {
return Status(kMDUnexpectedError, "Error opening Bin file to write");
}
o.write(reinterpret_cast<const char *>(data), std::streamsize(length * sizeof(T)));
if (!o.good()) {
return Status(kMDUnexpectedError, "Error writing Bin file");
}
o.close();
}
// Catch any exception and convert to Status return code
catch (const std::exception &err) {
return Status(kMDUnexpectedError, "Write bin file failed ");
return Status(kMDUnexpectedError, "Write bin file failed");
}
return Status::OK();
}
@ -370,7 +378,7 @@ class DataHelper {
size_t DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr, const size_t &buffer_size);
/// \brief Helper function to delete key in json file
/// note This function will return okay even if key not found
/// \note This function will return okay even if key not found
/// \param[in] in_file Json file to remove key from
/// \param[in] key The key to remove
/// \return Status The status code returned
@ -383,9 +391,9 @@ class DataHelper {
void Print(std::ostream &out) const;
/// \brief << Stream output operator overload
/// \notes This allows you to write the debug print info using stream operators
/// \note This allows you to write the debug print info using stream operators
/// \param out Reference to the output stream being overloaded
/// \param ds Reference to the DataSchema to display
/// \param dh Reference to the DataSchema to display
/// \return The output stream must be returned
friend std::ostream &operator<<(std::ostream &out, const DataHelper &dh) {
dh.Print(out);

258
mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h
View File

@ -932,9 +932,9 @@ class CelebADataset : public Dataset {
/// \param[in] decode Decode the images after reading (default=false).
/// \param[in] extensions Set of file extensions to be included in the dataset (default={}).
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, bool decode,
const std::set<std::vector<char>> &extensions, const std::shared_ptr<DatasetCache> &cache);
CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, bool decode, const std::set<std::vector<char>> &extensions,
const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of CelebADataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
@ -943,9 +943,8 @@ class CelebADataset : public Dataset {
/// \param[in] decode Decode the images after reading (default=false).
/// \param[in] extensions Set of file extensions to be included in the dataset (default={}).
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
bool decode, const std::set<std::vector<char>> &extensions,
const std::shared_ptr<DatasetCache> &cache);
CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
bool decode, const std::set<std::vector<char>> &extensions, const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of CelebADataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
@ -954,9 +953,9 @@ class CelebADataset : public Dataset {
/// \param[in] decode Decode the images after reading (default=false).
/// \param[in] extensions Set of file extensions to be included in the dataset (default={}).
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, bool decode,
const std::set<std::vector<char>> &extensions, const std::shared_ptr<DatasetCache> &cache);
CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, bool decode,
const std::set<std::vector<char>> &extensions, const std::shared_ptr<DatasetCache> &cache);
/// \brief Destructor of CelebADataset.
~CelebADataset() = default;
@ -1027,24 +1026,24 @@ class Cifar10Dataset : public Dataset {
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given, a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()).
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of Cifar10Dataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage Part of dataset of CIFAR10, can be "train", "test" or "all".
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of Cifar10Dataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage Part of dataset of CIFAR10, can be "train", "test" or "all".
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
/// \brief Destructor of Cifar10Dataset.
~Cifar10Dataset() = default;
@ -1101,24 +1100,24 @@ class Cifar100Dataset : public Dataset {
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given, a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()).
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of Cifar100Dataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage Part of dataset of CIFAR100, can be "train", "test" or "all".
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of Cifar100Dataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage Part of dataset of CIFAR100, can be "train", "test" or "all".
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
/// \brief Destructor of Cifar100Dataset.
~Cifar100Dataset() = default;
@ -1181,9 +1180,9 @@ class CityscapesDataset : public Dataset {
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given, a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()).
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
explicit CityscapesDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::vector<char> &quality_mode, const std::vector<char> &task, bool decode,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
CityscapesDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::vector<char> &quality_mode, const std::vector<char> &task, bool decode,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of CityscapesDataset.
/// \param[in] dataset_dir The dataset dir to be read.
@ -1196,9 +1195,9 @@ class CityscapesDataset : public Dataset {
/// \param[in] decode Decode the images after reading.
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
explicit CityscapesDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::vector<char> &quality_mode, const std::vector<char> &task, bool decode,
const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache);
CityscapesDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::vector<char> &quality_mode, const std::vector<char> &task, bool decode,
const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of CityscapesDataset.
/// \param[in] dataset_dir The dataset dir to be read.
@ -1211,9 +1210,9 @@ class CityscapesDataset : public Dataset {
/// \param[in] decode Decode the images after reading.
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
explicit CityscapesDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::vector<char> &quality_mode, const std::vector<char> &task, bool decode,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
CityscapesDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::vector<char> &quality_mode, const std::vector<char> &task, bool decode,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
/// \brief Destructor of CityscapesDataset.
~CityscapesDataset() = default;
@ -1303,9 +1302,9 @@ class CLUEDataset : public Dataset {
/// \param[in] shard_id The shard ID within num_shards. This argument should be
/// specified only when num_shards is also specified (Default = 0).
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit CLUEDataset(const std::vector<std::vector<char>> &dataset_files, const std::vector<char> &task,
const std::vector<char> &usage, int64_t num_samples, ShuffleMode shuffle, int32_t num_shards,
int32_t shard_id, const std::shared_ptr<DatasetCache> &cache);
CLUEDataset(const std::vector<std::vector<char>> &dataset_files, const std::vector<char> &task,
const std::vector<char> &usage, int64_t num_samples, ShuffleMode shuffle, int32_t num_shards,
int32_t shard_id, const std::shared_ptr<DatasetCache> &cache);
/// \brief Destructor of CLUEDataset.
~CLUEDataset() = default;
@ -1484,10 +1483,10 @@ class CSVDataset : public Dataset {
/// \param[in] shard_id The shard ID within num_shards. This argument should be
/// specified only when num_shards is also specified (Default = 0).
/// \param[in] cache Tensor cache to use.(default=nullptr which means no cache is used).
explicit CSVDataset(const std::vector<std::vector<char>> &dataset_files, char field_delim,
const std::vector<std::shared_ptr<CsvBase>> &column_defaults,
const std::vector<std::vector<char>> &column_names, int64_t num_samples, ShuffleMode shuffle,
int32_t num_shards, int32_t shard_id, const std::shared_ptr<DatasetCache> &cache);
CSVDataset(const std::vector<std::vector<char>> &dataset_files, char field_delim,
const std::vector<std::shared_ptr<CsvBase>> &column_defaults,
const std::vector<std::vector<char>> &column_names, int64_t num_samples, ShuffleMode shuffle,
int32_t num_shards, int32_t shard_id, const std::shared_ptr<DatasetCache> &cache);
/// \brief Destructor of CSVDataset.
~CSVDataset() = default;
@ -1538,9 +1537,9 @@ class DIV2KDataset : public Dataset {
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given, a `RandomSampler` will be used to randomly iterate the entire dataset.
/// \param[in] cache Tensor cache to use.
explicit DIV2KDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::vector<char> &downgrade, int32_t scale, bool decode,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
DIV2KDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const std::vector<char> &downgrade,
int32_t scale, bool decode, const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of DIV2KDataset.
/// \param[in] dataset_dir The dataset dir to be read.
@ -1551,9 +1550,8 @@ class DIV2KDataset : public Dataset {
/// \param[in] decode Decode the images after reading.
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use.
explicit DIV2KDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::vector<char> &downgrade, int32_t scale, bool decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
DIV2KDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const std::vector<char> &downgrade,
int32_t scale, bool decode, const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of DIV2KDataset.
/// \param[in] dataset_dir The dataset dir to be read.
@ -1564,9 +1562,9 @@ class DIV2KDataset : public Dataset {
/// \param[in] decode Decode the images after reading.
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use.
explicit DIV2KDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::vector<char> &downgrade, int32_t scale, bool decode,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
DIV2KDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const std::vector<char> &downgrade,
int32_t scale, bool decode, const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache);
/// \brief Destructor of DIV2KDataset.
~DIV2KDataset() = default;
@ -1641,8 +1639,8 @@ class FlickrDataset : public Dataset {
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given, a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()).
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
explicit FlickrDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file, bool decode,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
FlickrDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file, bool decode,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of FlickrDataset.
/// \param[in] dataset_dir The dataset dir to be read
@ -1650,8 +1648,8 @@ class FlickrDataset : public Dataset {
/// \param[in] decode Decode the images after reading.
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
explicit FlickrDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file, bool decode,
const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache);
FlickrDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file, bool decode,
const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of FlickrDataset.
/// \param[in] dataset_dir The dataset dir to be read
@ -1659,8 +1657,8 @@ class FlickrDataset : public Dataset {
/// \param[in] decode Decode the images after reading.
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
explicit FlickrDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file, bool decode,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
FlickrDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file, bool decode,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
/// \brief Destructor of FlickrDataset.
~FlickrDataset() = default;
@ -1725,10 +1723,10 @@ class ImageFolderDataset : public Dataset {
/// \param[in] extensions File extensions to be read.
/// \param[in] class_indexing a class name to label map.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode,
const std::shared_ptr<Sampler> &sampler, const std::set<std::vector<char>> &extensions,
const std::map<std::vector<char>, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache);
ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, const std::shared_ptr<Sampler> &sampler,
const std::set<std::vector<char>> &extensions,
const std::map<std::vector<char>, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of ImageFolderDataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
@ -1737,10 +1735,10 @@ class ImageFolderDataset : public Dataset {
/// \param[in] extensions File extensions to be read.
/// \param[in] class_indexing a class name to label map.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, const Sampler *sampler,
const std::set<std::vector<char>> &extensions,
const std::map<std::vector<char>, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache);
ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, const Sampler *sampler,
const std::set<std::vector<char>> &extensions,
const std::map<std::vector<char>, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of ImageFolderDataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
@ -1749,11 +1747,10 @@ class ImageFolderDataset : public Dataset {
/// \param[in] extensions File extensions to be read.
/// \param[in] class_indexing a class name to label map.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode,
const std::reference_wrapper<Sampler> sampler,
const std::set<std::vector<char>> &extensions,
const std::map<std::vector<char>, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache);
ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, const std::reference_wrapper<Sampler> sampler,
const std::set<std::vector<char>> &extensions,
const std::map<std::vector<char>, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache);
/// \brief Destructor of ImageFolderDataset.
~ImageFolderDataset() = default;
@ -1833,10 +1830,9 @@ class ManifestDataset : public Dataset {
/// names will be sorted alphabetically and each class will be given a unique index starting from 0).
/// \param[in] decode Decode the images after reading (default=false).
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler,
const std::map<std::vector<char>, int32_t> &class_indexing, bool decode,
const std::shared_ptr<DatasetCache> &cache);
ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of ManifestDataset.
/// \param[in] dataset_file The dataset file to be read.
@ -1846,9 +1842,9 @@ class ManifestDataset : public Dataset {
/// names will be sorted alphabetically and each class will be given a unique index starting from 0).
/// \param[in] decode Decode the images after reading (default=false).
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage,
const Sampler *sampler, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, const std::shared_ptr<DatasetCache> &cache);
ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage, const Sampler *sampler,
const std::map<std::vector<char>, int32_t> &class_indexing, bool decode,
const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of ManifestDataset.
/// \param[in] dataset_file The dataset file to be read.
@ -1858,10 +1854,10 @@ class ManifestDataset : public Dataset {
/// names will be sorted alphabetically and each class will be given a unique index starting from 0).
/// \param[in] decode Decode the images after reading (default=false).
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler,
const std::map<std::vector<char>, int32_t> &class_indexing, bool decode,
const std::shared_ptr<DatasetCache> &cache);
ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler,
const std::map<std::vector<char>, int32_t> &class_indexing, bool decode,
const std::shared_ptr<DatasetCache> &cache);
/// \brief Destructor of ManifestDataset.
~ManifestDataset() = default;
@ -1945,10 +1941,10 @@ class MindDataDataset : public Dataset {
/// ShuffleMode::kGlobal - Shuffle both the files and samples.
/// ShuffleMode::kInfile - Shuffle samples in file.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
const std::shared_ptr<Sampler> &sampler, const nlohmann::json *padded_sample,
int64_t num_padded, ShuffleMode shuffle_mode = ShuffleMode::kGlobal,
const std::shared_ptr<DatasetCache> &cache = nullptr);
MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
const std::shared_ptr<Sampler> &sampler, const nlohmann::json *padded_sample, int64_t num_padded,
ShuffleMode shuffle_mode = ShuffleMode::kGlobal,
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Constructor of MindDataDataset.
/// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source
@ -1965,10 +1961,10 @@ class MindDataDataset : public Dataset {
/// ShuffleMode::kGlobal - Shuffle both the files and samples.
/// ShuffleMode::kInfile - Shuffle samples in file.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
const Sampler *sampler, const nlohmann::json *padded_sample, int64_t num_padded,
ShuffleMode shuffle_mode = ShuffleMode::kGlobal,
const std::shared_ptr<DatasetCache> &cache = nullptr);
MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
const Sampler *sampler, const nlohmann::json *padded_sample, int64_t num_padded,
ShuffleMode shuffle_mode = ShuffleMode::kGlobal,
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Constructor of MindDataDataset.
/// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source
@ -1985,10 +1981,10 @@ class MindDataDataset : public Dataset {
/// ShuffleMode::kGlobal - Shuffle both the files and samples.
/// ShuffleMode::kInfile - Shuffle samples in file.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample,
int64_t num_padded, ShuffleMode shuffle_mode = ShuffleMode::kGlobal,
const std::shared_ptr<DatasetCache> &cache = nullptr);
MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample,
int64_t num_padded, ShuffleMode shuffle_mode = ShuffleMode::kGlobal,
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Constructor of MindDataDataset.
/// \param[in] dataset_files List of dataset files to be read directly.
@ -2004,11 +2000,11 @@ class MindDataDataset : public Dataset {
/// ShuffleMode::kGlobal - Shuffle both the files and samples.
/// ShuffleMode::kInfile - Shuffle data within each file.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list, const std::shared_ptr<Sampler> &sampler,
const nlohmann::json *padded_sample, int64_t num_padded,
ShuffleMode shuffle_mode = ShuffleMode::kGlobal,
const std::shared_ptr<DatasetCache> &cache = nullptr);
MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list, const std::shared_ptr<Sampler> &sampler,
const nlohmann::json *padded_sample, int64_t num_padded,
ShuffleMode shuffle_mode = ShuffleMode::kGlobal,
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Constructor of MindDataDataset.
/// \param[in] dataset_files List of dataset files to be read directly.
@ -2024,11 +2020,11 @@ class MindDataDataset : public Dataset {
/// ShuffleMode::kGlobal - Shuffle both the files and samples.
/// ShuffleMode::kInfile - Shuffle data within each file.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list, const Sampler *sampler,
const nlohmann::json *padded_sample, int64_t num_padded,
ShuffleMode shuffle_mode = ShuffleMode::kGlobal,
const std::shared_ptr<DatasetCache> &cache = nullptr);
MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list, const Sampler *sampler,
const nlohmann::json *padded_sample, int64_t num_padded,
ShuffleMode shuffle_mode = ShuffleMode::kGlobal,
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Constructor of MindDataDataset.
/// \param[in] dataset_files List of dataset files to be read directly.
@ -2044,11 +2040,11 @@ class MindDataDataset : public Dataset {
/// ShuffleMode::kGlobal - Shuffle both the files and samples.
/// ShuffleMode::kInfile - Shuffle samples in file.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample,
int64_t num_padded, ShuffleMode shuffle_mode = ShuffleMode::kGlobal,
const std::shared_ptr<DatasetCache> &cache = nullptr);
MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list, const std::reference_wrapper<Sampler> sampler,
const nlohmann::json *padded_sample, int64_t num_padded,
ShuffleMode shuffle_mode = ShuffleMode::kGlobal,
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Destructor of MindDataDataset.
~MindDataDataset() = default;
@ -2214,24 +2210,24 @@ class MnistDataset : public Dataset {
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given, a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()).
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of MnistDataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage Part of dataset of MNIST, can be "train", "test" or "all".
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of MnistDataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage Part of dataset of MNIST, can be "train", "test" or "all".
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
/// Destructor of MnistDataset.
~MnistDataset() = default;
@ -2342,24 +2338,24 @@ class SBUDataset : public Dataset {
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given, a `RandomSampler` will be used to randomly iterate the entire dataset.
/// \param[in] cache Tensor cache to use.
explicit SBUDataset(const std::vector<char> &dataset_dir, bool decode, const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache);
SBUDataset(const std::vector<char> &dataset_dir, bool decode, const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of SBUDataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] decode Decode the images after reading.
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use.
explicit SBUDataset(const std::vector<char> &dataset_dir, bool decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
SBUDataset(const std::vector<char> &dataset_dir, bool decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of SBUDataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] decode Decode the images after reading.
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use.
explicit SBUDataset(const std::vector<char> &dataset_dir, bool decode, const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache);
SBUDataset(const std::vector<char> &dataset_dir, bool decode, const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache);
/// Destructor of SBUDataset.
~SBUDataset() = default;
@ -2422,9 +2418,8 @@ class TextFileDataset : public Dataset {
/// \param[in] shard_id The shard ID within num_shards. This argument should be
/// specified only when num_shards is also specified (Default = 0).
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit TextFileDataset(const std::vector<std::vector<char>> &dataset_files, int64_t num_samples,
ShuffleMode shuffle, int32_t num_shards, int32_t shard_id,
const std::shared_ptr<DatasetCache> &cache);
TextFileDataset(const std::vector<std::vector<char>> &dataset_files, int64_t num_samples, ShuffleMode shuffle,
int32_t num_shards, int32_t shard_id, const std::shared_ptr<DatasetCache> &cache);
/// Destructor of TextFileDataset.
~TextFileDataset() = default;
@ -2582,9 +2577,8 @@ class USPSDataset : public Dataset {
/// \param[in] shard_id The shard ID within num_shards. This argument should be
/// specified only when num_shards is also specified (Default = 0).
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
explicit USPSDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, int64_t num_samples,
ShuffleMode shuffle, int32_t num_shards, int32_t shard_id,
const std::shared_ptr<DatasetCache> &cache);
USPSDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, int64_t num_samples,
ShuffleMode shuffle, int32_t num_shards, int32_t shard_id, const std::shared_ptr<DatasetCache> &cache);
/// Destructor of USPSDataset.
~USPSDataset() = default;
@ -2628,10 +2622,9 @@ class VOCDataset : public Dataset {
/// given, a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()).
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
/// \param[in] extra_metadata Flag to add extra meta-data to row (default=false).
explicit VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task,
const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache,
bool extra_metadata);
VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task, const std::vector<char> &usage,
const std::map<std::vector<char>, int32_t> &class_indexing, bool decode,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache, bool extra_metadata);
/// \brief Constructor of VOCDataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
@ -2642,10 +2635,9 @@ class VOCDataset : public Dataset {
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
/// \param[in] extra_metadata Flag to add extra meta-data to row (default=false).
explicit VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task,
const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache,
bool extra_metadata);
VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task, const std::vector<char> &usage,
const std::map<std::vector<char>, int32_t> &class_indexing, bool decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache, bool extra_metadata);
/// \brief Constructor of VOCDataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
@ -2656,10 +2648,10 @@ class VOCDataset : public Dataset {
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
/// \param[in] extra_metadata Flag to add extra meta-data to row (default=false).
explicit VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task,
const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache, bool extra_metadata);
VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task, const std::vector<char> &usage,
const std::map<std::vector<char>, int32_t> &class_indexing, bool decode,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache,
bool extra_metadata);
/// Destructor of VOCDataset.
~VOCDataset() = default;

28
mindspore/ccsrc/minddata/dataset/include/dataset/text.h
View File

@ -113,10 +113,10 @@ class BertTokenizer final : public TensorTransform {
/// \param[in] preserve_unused_token If true, do not split special tokens like '[CLS]', '[SEP]', '[UNK]', '[PAD]' and
/// '[MASK]' (default=true).
/// \param[in] with_offsets Whether to output offsets of tokens (default=false).
explicit BertTokenizer(const std::shared_ptr<Vocab> &vocab, const std::vector<char> &suffix_indicator,
int32_t max_bytes_per_token, const std::vector<char> &unknown_token, bool lower_case,
bool keep_whitespace, const NormalizeForm normalize_form, bool preserve_unused_token,
bool with_offsets);
BertTokenizer(const std::shared_ptr<Vocab> &vocab, const std::vector<char> &suffix_indicator,
int32_t max_bytes_per_token, const std::vector<char> &unknown_token, bool lower_case,
bool keep_whitespace, const NormalizeForm normalize_form, bool preserve_unused_token,
bool with_offsets);
/// \brief Destructor
~BertTokenizer() = default;
@ -162,8 +162,8 @@ class JiebaTokenizer final : public TensorTransform {
/// - JiebaMode.kHMM, tokenizes with Hidden Markov Model Segment algorithm.
/// - JiebaMode.kMIX, tokenizes with a mix of MPSegment and HMMSegment algorithms.
/// \param[in] with_offsets Whether to output offsets of tokens (default=false).
explicit JiebaTokenizer(const std::string &hmm_path, const std::string &mp_path,
const JiebaMode &mode = JiebaMode::kMix, bool with_offsets = false)
JiebaTokenizer(const std::string &hmm_path, const std::string &mp_path, const JiebaMode &mode = JiebaMode::kMix,
bool with_offsets = false)
: JiebaTokenizer(StringToChar(hmm_path), StringToChar(mp_path), mode, with_offsets) {}
/// \brief Constructor.
@ -177,8 +177,8 @@ class JiebaTokenizer final : public TensorTransform {
/// - JiebaMode.kHMM, tokenizes with Hidden Markov Model Segment algorithm.
/// - JiebaMode.kMIX, tokenizes with a mix of MPSegment and HMMSegment algorithms.
/// \param[in] with_offsets Whether to output offsets of tokens (default=false).
explicit JiebaTokenizer(const std::vector<char> &hmm_path, const std::vector<char> &mp_path, const JiebaMode &mode,
bool with_offsets);
JiebaTokenizer(const std::vector<char> &hmm_path, const std::vector<char> &mp_path, const JiebaMode &mode,
bool with_offsets);
/// \brief Destructor
~JiebaTokenizer() = default;
@ -255,8 +255,8 @@ class Lookup final : public TensorTransform {
/// runtime error will be thrown (default={}, means no unknown_token is specified).
/// \param[in] data_type mindspore::DataType of the tensor after lookup; must be numeric, including bool.
/// (default=mindspore::DataType::kNumberTypeInt32).
explicit Lookup(const std::shared_ptr<Vocab> &vocab, const std::optional<std::vector<char>> &unknown_token,
mindspore::DataType data_type = mindspore::DataType::kNumberTypeInt32);
Lookup(const std::shared_ptr<Vocab> &vocab, const std::optional<std::vector<char>> &unknown_token,
mindspore::DataType data_type = mindspore::DataType::kNumberTypeInt32);
/// \brief Destructor
~Lookup() = default;
@ -296,8 +296,8 @@ class Ngram final : public TensorTransform {
/// \param[in] right_pad {"pad_token", pad_width}. Padding performed on right side of the sequence.pad_width will
/// be capped at n-1. right_pad=("-",2) would pad the right side of the sequence with "--" (default={"", 0}}).
/// \param[in] separator Symbol used to join strings together (default=" ").
explicit Ngram(const std::vector<int32_t> &ngrams, const std::pair<std::vector<char>, int32_t> &left_pad,
const std::pair<std::vector<char>, int32_t> &right_pad, const std::vector<char> &separator);
Ngram(const std::vector<int32_t> &ngrams, const std::pair<std::vector<char>, int32_t> &left_pad,
const std::pair<std::vector<char>, int32_t> &right_pad, const std::vector<char> &separator);
/// \brief Destructor
~Ngram() = default;
@ -348,7 +348,7 @@ class RegexReplace final : public TensorTransform {
/// \param[in] replace The string to replace the matched element.
/// \param[in] replace_all Confirm whether to replace all. If false, only replace the first matched element;
/// if true, replace all matched elements (default=true).
explicit RegexReplace(std::string pattern, std::string replace, bool replace_all = true)
RegexReplace(std::string pattern, std::string replace, bool replace_all = true)
: RegexReplace(StringToChar(pattern), StringToChar(replace), replace_all) {}
/// \brief Constructor.
@ -356,7 +356,7 @@ class RegexReplace final : public TensorTransform {
/// \param[in] replace The string to replace the matched element.
/// \param[in] replace_all Confirm whether to replace all. If false, only replace the first matched element;
/// if true, replace all matched elements (default=true).
explicit RegexReplace(const std::vector<char> &pattern, const std::vector<char> &replace, bool replace_all);
RegexReplace(const std::vector<char> &pattern, const std::vector<char> &replace, bool replace_all);
/// \brief Destructor
~RegexReplace() = default;

7
mindspore/ccsrc/minddata/dataset/include/dataset/vision.h
View File

@ -269,11 +269,10 @@ class NormalizePad final : public TensorTransform {
/// The standard deviation values must be in range (0.0, 255.0].
/// \param[in] dtype The output datatype of Tensor.
/// The standard deviation values must be "float32" or "float16"（default = "float32"）.
explicit NormalizePad(const std::vector<float> &mean, const std::vector<float> &std,
const std::string &dtype = "float32")
NormalizePad(const std::vector<float> &mean, const std::vector<float> &std, const std::string &dtype = "float32")
: NormalizePad(mean, std, StringToChar(dtype)) {}
explicit NormalizePad(const std::vector<float> &mean, const std::vector<float> &std, const std::vector<char> &dtype);
NormalizePad(const std::vector<float> &mean, const std::vector<float> &std, const std::vector<char> &dtype);
/// \brief Destructor.
~NormalizePad() = default;
@ -332,7 +331,7 @@ class RandomColor final : public TensorTransform {
/// \brief Constructor.
/// \param[in] t_lb Lower bound random weights.
/// \param[in] t_ub Upper bound random weights.
explicit RandomColor(float t_lb, float t_ub);
RandomColor(float t_lb, float t_ub);
/// \brief Destructor.
~RandomColor() = default;

2
mindspore/ccsrc/minddata/dataset/include/dataset/vision_ascend.h
View File

@ -64,7 +64,7 @@ class DvppDecodeResizeCropJpeg final : public TensorTransform {
/// \brief Constructor.
/// \param[in] crop Parameter vector of two integers for each dimension after final crop, with respect to H,W order.
/// \param[in] resize Parameter vector of two integers for each dimension after resize, with respect to H,W order.
explicit DvppDecodeResizeCropJpeg(std::vector<uint32_t> crop, std::vector<uint32_t> resize);
DvppDecodeResizeCropJpeg(std::vector<uint32_t> crop, std::vector<uint32_t> resize);
/// \brief Destructor.
~DvppDecodeResizeCropJpeg() = default;

3
mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.cc
View File

@ -354,10 +354,11 @@ Status ToFloat16(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *
RETURN_IF_NOT_OK(Tensor::CreateEmpty(input->shape(), new_type, output));
auto in_itr = input->begin<float>();
auto in_end = input->end<float>();
auto out_itr = (*output)->begin<float16>();
auto out_end = (*output)->end<float16>();
for (; out_itr != out_end; ++in_itr, ++out_itr) {
for (; (in_itr != in_end) && (out_itr != out_end); ++in_itr, ++out_itr) {
float element = *in_itr;
float float16_max = static_cast<float>(std::numeric_limits<float16>::max());
float float16_min = static_cast<float>(std::numeric_limits<float16>::lowest());

28
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_crop_jpeg_op.cc
View File

@ -38,21 +38,22 @@ Status DvppCropJpegOp::Compute(const std::shared_ptr<DeviceTensor> &input, std::
}
APP_ERROR ret = processor_->JPEG_C(last_step);
if (ret != APP_ERR_OK) {
processor_->Release();
ret = processor_->Release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release memory failed.");
std::string error = "Error in dvpp crop processing:" + std::to_string(ret);
RETURN_STATUS_UNEXPECTED(error);
}
std::shared_ptr<DvppDataInfo> CropOut(processor_->Get_Croped_DeviceData());
const TensorShape dvpp_shape({1, 1, 1});
const DataType dvpp_data_type(DataType::DE_UINT8);
mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output);
(*output)->SetAttributes(CropOut->data, CropOut->dataSize, CropOut->width, CropOut->widthStride, CropOut->height,
CropOut->heightStride);
RETURN_IF_NOT_OK(mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output));
RETURN_IF_NOT_OK((*output)->SetAttributes(CropOut->data, CropOut->dataSize, CropOut->width, CropOut->widthStride,
CropOut->height, CropOut->heightStride));
if (!((*output)->HasDeviceData())) {
std::string error = "[ERROR] Fail to get the Output result from device memory!";
RETURN_STATUS_UNEXPECTED(error);
}
} catch (const cv::Exception &e) {
} catch (const std::exception &e) {
std::string error = "[ERROR] Fail in DvppCropJpegOp:" + std::string(e.what());
RETURN_STATUS_UNEXPECTED(error);
}
@ -71,6 +72,8 @@ Status DvppCropJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared
imageinfo.dataSize = input->SizeInBytes();
imageinfo.data = static_cast<uint8_t *>(buffer);
std::vector<uint32_t> yuv_shape_ = input->GetYuvShape();
const size_t yuv_shape_size = 4;
CHECK_FAIL_RETURN_UNEXPECTED(yuv_shape_.size() == yuv_shape_size, "yuv_shape requires 4 elements.");
imageinfo.width = yuv_shape_[0];
imageinfo.widthStride = yuv_shape_[1];
imageinfo.height = yuv_shape_[2];
@ -114,16 +117,18 @@ Status DvppCropJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared
uint32_t crop_width = CropOut->width;
uint32_t crop_widthStride = CropOut->widthStride;
const DataType dvpp_data_type(DataType::DE_UINT8);
mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output);
(*output)->SetYuvShape(crop_width, crop_widthStride, crop_height, crop_heightStride);
RETURN_IF_NOT_OK(mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output));
RETURN_IF_NOT_OK((*output)->SetYuvShape(crop_width, crop_widthStride, crop_height, crop_heightStride));
if (!((*output)->HasData())) {
std::string error = "[ERROR] Fail to get the Output result from memory!";
RETURN_STATUS_UNEXPECTED(error);
}
process.device_memory_release();
process.Release();
ret = process.device_memory_release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release device memory failed.");
ret = process.Release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release host memory failed.");
// Last part end where we transform the processed data into a tensor which can be applied in later units.
} catch (const cv::Exception &e) {
} catch (const std::exception &e) {
std::string error = "[ERROR] Fail in DvppCropJpegOp:" + std::string(e.what());
RETURN_STATUS_UNEXPECTED(error);
}
@ -147,7 +152,8 @@ Status DvppCropJpegOp::SetAscendResource(const std::shared_ptr<DeviceResource> &
if (!processor_) {
RETURN_STATUS_UNEXPECTED("Resource initialize fail, please check your env");
}
processor_->SetCropParas(crop_width_, crop_height_);
APP_ERROR ret = processor_->SetCropParas(crop_width_, crop_height_);
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "SetCropParas failed.");
return Status::OK();
}

23
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_jpeg_op.cc
View File

@ -35,21 +35,22 @@ Status DvppDecodeJpegOp::Compute(const std::shared_ptr<DeviceTensor> &input, std
CHECK_FAIL_RETURN_UNEXPECTED(input->GetDeviceBuffer() != nullptr, "The input image buffer on device is empty");
APP_ERROR ret = processor_->JPEG_D();
if (ret != APP_ERR_OK) {
processor_->Release();
ret = processor_->Release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release memory failed.");
std::string error = "Error in dvpp processing:" + std::to_string(ret);
RETURN_STATUS_UNEXPECTED(error);
}
std::shared_ptr<DvppDataInfo> DecodeOut(processor_->Get_Decode_DeviceData());
const TensorShape dvpp_shape({1, 1, 1});
const DataType dvpp_data_type(DataType::DE_UINT8);
mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output);
(*output)->SetAttributes(DecodeOut->data, DecodeOut->dataSize, DecodeOut->width, DecodeOut->widthStride,
DecodeOut->height, DecodeOut->heightStride);
RETURN_IF_NOT_OK(mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output));
RETURN_IF_NOT_OK((*output)->SetAttributes(DecodeOut->data, DecodeOut->dataSize, DecodeOut->width,
DecodeOut->widthStride, DecodeOut->height, DecodeOut->heightStride));
if (!((*output)->HasDeviceData())) {
std::string error = "[ERROR] Fail to get the Output result from memory!";
RETURN_STATUS_UNEXPECTED(error);
}
} catch (const cv::Exception &e) {
} catch (const std::exception &e) {
std::string error = "[ERROR] Fail in DvppDecodeJpegOp:" + std::string(e.what());
RETURN_STATUS_UNEXPECTED(error);
}
@ -106,16 +107,18 @@ Status DvppDecodeJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shar
const TensorShape dvpp_shape({dvpp_length, 1, 1});
const DataType dvpp_data_type(DataType::DE_UINT8);
mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output);
(*output)->SetYuvShape(decoded_width, decoded_widthStride, decoded_height, decoded_heightStride);
RETURN_IF_NOT_OK(mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output));
RETURN_IF_NOT_OK((*output)->SetYuvShape(decoded_width, decoded_widthStride, decoded_height, decoded_heightStride));
if (!((*output)->HasData())) {
std::string error = "[ERROR] Fail to get the Output result from device memory!";
RETURN_STATUS_UNEXPECTED(error);
}
process.device_memory_release();
process.Release();
ret = process.device_memory_release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release device memory failed.");
ret = process.Release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release host memory failed.");
// Last part end where we transform the processed data into a tensor which can be applied in later units.
} catch (const cv::Exception &e) {
} catch (const std::exception &e) {
std::string error = "[ERROR] Fail in DvppDecodeJpegOp:" + std::string(e.what());
RETURN_STATUS_UNEXPECTED(error);
}

21
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_png_op.cc
View File

@ -34,21 +34,22 @@ Status DvppDecodePngOp::Compute(const std::shared_ptr<DeviceTensor> &input, std:
CHECK_FAIL_RETURN_UNEXPECTED(input->GetDeviceBuffer() != nullptr, "The input image buffer on device is empty");
APP_ERROR ret = processor_->PNG_D();
if (ret != APP_ERR_OK) {
processor_->Release();
ret = processor_->Release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release memory failed.");
std::string error = "Error in dvpp processing:" + std::to_string(ret);
RETURN_STATUS_UNEXPECTED(error);
}
std::shared_ptr<DvppDataInfo> DecodeOut(processor_->Get_Decode_DeviceData());
const TensorShape dvpp_shape({1, 1, 1});
const DataType dvpp_data_type(DataType::DE_UINT8);
mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output);
(*output)->SetAttributes(DecodeOut->data, DecodeOut->dataSize, DecodeOut->width, DecodeOut->widthStride,
DecodeOut->height, DecodeOut->heightStride);
RETURN_IF_NOT_OK(mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output));
RETURN_IF_NOT_OK((*output)->SetAttributes(DecodeOut->data, DecodeOut->dataSize, DecodeOut->width,
DecodeOut->widthStride, DecodeOut->height, DecodeOut->heightStride));
if (!((*output)->HasDeviceData())) {
std::string error = "[ERROR] Fail to get the Output result from memory!";
RETURN_STATUS_UNEXPECTED(error);
}
} catch (const cv::Exception &e) {
} catch (const std::exception &e) {
std::string error = "[ERROR] Fail in DvppDecodeJpegOp:" + std::string(e.what());
RETURN_STATUS_UNEXPECTED(error);
}
@ -95,8 +96,6 @@ Status DvppDecodePngOp::Compute(const std::shared_ptr<Tensor> &input, std::share
}
// Third part end where we execute the core function of dvpp
/* 测试Device内存
*/
auto data = std::static_pointer_cast<unsigned char>(process.Get_Memory_Data());
unsigned char *ret_ptr = data.get();
std::shared_ptr<DvppDataInfo> DecodeOut(process.Get_Decode_DeviceData());
@ -109,10 +108,12 @@ Status DvppDecodePngOp::Compute(const std::shared_ptr<Tensor> &input, std::share
std::string error = "[ERROR] Fail to get the Output result from memory!";
RETURN_STATUS_UNEXPECTED(error);
}
process.device_memory_release();
process.Release();
ret = process.device_memory_release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release device memory failed.");
ret = process.Release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release host memory failed.");
// Last part end where we transform the processed data into a tensor which can be applied in later units.
} catch (const cv::Exception &e) {
} catch (const std::exception &e) {
std::string error = "[ERROR] Fail in DvppDecodePngOp:" + std::string(e.what());
RETURN_STATUS_UNEXPECTED(error);
}

27
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_resize_crop_jpeg_op.cc
View File

@ -34,7 +34,8 @@ Status DvppDecodeResizeCropJpegOp::Compute(const std::shared_ptr<DeviceTensor> &
CHECK_FAIL_RETURN_UNEXPECTED(input->GetDeviceBuffer() != nullptr, "The input image buffer on device is empty");
APP_ERROR ret = processor_->JPEG_DRC();
if (ret != APP_ERR_OK) {
processor_->Release();
ret = processor_->Release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release memory failed.");
std::string error = "Error in dvpp processing:" + std::to_string(ret);
RETURN_STATUS_UNEXPECTED(error);
}
@ -42,14 +43,14 @@ Status DvppDecodeResizeCropJpegOp::Compute(const std::shared_ptr<DeviceTensor> &
const TensorShape dvpp_shape({1, 1, 1});
const DataType dvpp_data_type(DataType::DE_UINT8);
mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output);
(*output)->SetAttributes(CropOut->data, CropOut->dataSize, CropOut->width, CropOut->widthStride, CropOut->height,
CropOut->heightStride);
RETURN_IF_NOT_OK(mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output));
RETURN_IF_NOT_OK((*output)->SetAttributes(CropOut->data, CropOut->dataSize, CropOut->width, CropOut->widthStride,
CropOut->height, CropOut->heightStride));
if (!((*output)->HasDeviceData())) {
std::string error = "[ERROR] Fail to get the Output result from memory!";
RETURN_STATUS_UNEXPECTED(error);
}
} catch (const cv::Exception &e) {
} catch (const std::exception &e) {
std::string error = "[ERROR] Fail in DvppDecodeResizeCropJpegOp:" + std::string(e.what());
RETURN_STATUS_UNEXPECTED(error);
}
@ -102,15 +103,17 @@ Status DvppDecodeResizeCropJpegOp::Compute(const std::shared_ptr<Tensor> &input,
uint32_t dvpp_length = CropOut->dataSize;
const TensorShape dvpp_shape({dvpp_length, 1, 1});
const DataType dvpp_data_type(DataType::DE_UINT8);
mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output);
RETURN_IF_NOT_OK(mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output));
if (!((*output)->HasData())) {
std::string error = "[ERROR] Fail to get the Output result from memory!";
RETURN_STATUS_UNEXPECTED(error);
}
processor.device_memory_release();
processor.Release();
ret = processor.device_memory_release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release device memory failed.");
ret = processor.Release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release host memory failed.");
// Last part end where we transform the processed data into a tensor which can be applied in later units.
} catch (const cv::Exception &e) {
} catch (const std::exception &e) {
std::string error = "[ERROR] Fail in DvppDecodeResizeCropJpegOp:" + std::string(e.what());
RETURN_STATUS_UNEXPECTED(error);
}
@ -135,8 +138,10 @@ Status DvppDecodeResizeCropJpegOp::SetAscendResource(const std::shared_ptr<Devic
if (!processor_) {
RETURN_STATUS_UNEXPECTED("Resource initialize fail, please check your env");
}
processor_->SetResizeParas(resized_width_, resized_height_);
processor_->SetCropParas(crop_width_, crop_height_);
APP_ERROR ret = processor_->SetResizeParas(resized_width_, resized_height_);
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "SetResizeParas failed.");
ret = processor_->SetCropParas(crop_width_, crop_height_);
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "SetCropParas failed.");
return Status::OK();
}

24
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_resize_jpeg_op.cc
View File

@ -34,21 +34,22 @@ Status DvppDecodeResizeJpegOp::Compute(const std::shared_ptr<DeviceTensor> &inpu
CHECK_FAIL_RETURN_UNEXPECTED(input->GetDeviceBuffer() != nullptr, "The input image buffer on device is empty");
APP_ERROR ret = processor_->JPEG_DR();
if (ret != APP_ERR_OK) {
processor_->Release();
ret = processor_->Release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release memory failed.");
std::string error = "Error in dvpp processing:" + std::to_string(ret);
RETURN_STATUS_UNEXPECTED(error);
}
std::shared_ptr<DvppDataInfo> ResizeOut(processor_->Get_Resized_DeviceData());
const TensorShape dvpp_shape({1, 1, 1});
const DataType dvpp_data_type(DataType::DE_UINT8);
mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output);
(*output)->SetAttributes(ResizeOut->data, ResizeOut->dataSize, ResizeOut->width, ResizeOut->widthStride,
ResizeOut->height, ResizeOut->heightStride);
RETURN_IF_NOT_OK(mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output));
RETURN_IF_NOT_OK((*output)->SetAttributes(ResizeOut->data, ResizeOut->dataSize, ResizeOut->width,
ResizeOut->widthStride, ResizeOut->height, ResizeOut->heightStride));
if (!((*output)->HasDeviceData())) {
std::string error = "[ERROR] Fail to get the Output result from memory!";
RETURN_STATUS_UNEXPECTED(error);
}
} catch (const cv::Exception &e) {
} catch (const std::exception &e) {
std::string error = "[ERROR] Fail in DvppDecodeResizeJpegOp:" + std::string(e.what());
RETURN_STATUS_UNEXPECTED(error);
}
@ -101,15 +102,17 @@ Status DvppDecodeResizeJpegOp::Compute(const std::shared_ptr<Tensor> &input, std
dsize_t dvpp_length = ResizeOut->dataSize;
const TensorShape dvpp_shape({dvpp_length, 1, 1});
const DataType dvpp_data_type(DataType::DE_UINT8);
mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output);
RETURN_IF_NOT_OK(mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output));
if (!((*output)->HasData())) {
std::string error = "[ERROR] Fail to get the Output result from memory!";
RETURN_STATUS_UNEXPECTED(error);
}
process.device_memory_release();
process.Release();
ret = process.device_memory_release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release device memory failed.");
ret = process.Release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release host memory failed.");
// Last part end where we transform the processed data into a tensor which can be applied in later units.
} catch (const cv::Exception &e) {
} catch (const std::exception &e) {
std::string error = "[ERROR] Fail in DvppDecodeResizeJpegOp:" + std::string(e.what());
RETURN_STATUS_UNEXPECTED(error);
}
@ -133,7 +136,8 @@ Status DvppDecodeResizeJpegOp::SetAscendResource(const std::shared_ptr<DeviceRes
if (!processor_) {
RETURN_STATUS_UNEXPECTED("Resource initialize fail, please check your env");
}
processor_->SetResizeParas(resized_width_, resized_height_);
APP_ERROR ret = processor_->SetResizeParas(resized_width_, resized_height_);
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "SetResizeParas failed.");
return Status::OK();
}

8
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_normalize_op.cc
View File

@ -22,10 +22,12 @@ namespace dataset {
Status DvppNormalizeOp::Compute(const std::shared_ptr<DeviceTensor> &input, std::shared_ptr<DeviceTensor> *output) {
const TensorShape dvpp_shape({1, 1, 1});
const DataType dvpp_data_type(DataType::DE_UINT8);
mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output);
RETURN_IF_NOT_OK(mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output));
std::vector<uint32_t> yuv_shape = input->GetYuvStrideShape();
(*output)->SetAttributes(input->GetDeviceMutableBuffer(), input->DeviceDataSize(), yuv_shape[0], yuv_shape[1],
yuv_shape[2], yuv_shape[3]);
const size_t yuv_shape_size = 4;
CHECK_FAIL_RETURN_UNEXPECTED(yuv_shape.size() == yuv_shape_size, "yuv_shape requires 4 elements.");
RETURN_IF_NOT_OK((*output)->SetAttributes(input->GetDeviceMutableBuffer(), input->DeviceDataSize(), yuv_shape[0],
yuv_shape[1], yuv_shape[2], yuv_shape[3]));
if (!((*output)->HasDeviceData())) {
std::string error = "[ERROR] Fail to get the output result from device memory!";
RETURN_STATUS_UNEXPECTED(error);

26
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_resize_jpeg_op.cc
View File

@ -39,21 +39,22 @@ Status DvppResizeJpegOp::Compute(const std::shared_ptr<DeviceTensor> &input, std
}
APP_ERROR ret = processor_->JPEG_R(last_step);
if (ret != APP_ERR_OK) {
processor_->Release();
ret = processor_->Release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release memory failed.");
std::string error = "Error in dvpp processing:" + std::to_string(ret);
RETURN_STATUS_UNEXPECTED(error);
}
std::shared_ptr<DvppDataInfo> ResizeOut(processor_->Get_Resized_DeviceData());
const TensorShape dvpp_shape({1, 1, 1});
const DataType dvpp_data_type(DataType::DE_UINT8);
mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output);
(*output)->SetAttributes(ResizeOut->data, ResizeOut->dataSize, ResizeOut->width, ResizeOut->widthStride,
ResizeOut->height, ResizeOut->heightStride);
RETURN_IF_NOT_OK(mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output));
RETURN_IF_NOT_OK((*output)->SetAttributes(ResizeOut->data, ResizeOut->dataSize, ResizeOut->width,
ResizeOut->widthStride, ResizeOut->height, ResizeOut->heightStride));
if (!((*output)->HasDeviceData())) {
std::string error = "[ERROR] Fail to get the Output result from device memory!";
RETURN_STATUS_UNEXPECTED(error);
}
} catch (const cv::Exception &e) {
} catch (const std::exception &e) {
std::string error = "[ERROR] Fail in DvppResizeJpegOp:" + std::string(e.what());
RETURN_STATUS_UNEXPECTED(error);
}
@ -115,16 +116,18 @@ Status DvppResizeJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shar
uint32_t resized_width = ResizeOut->width;
uint32_t resized_widthStride = ResizeOut->widthStride;
const DataType dvpp_data_type(DataType::DE_UINT8);
mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output);
(*output)->SetYuvShape(resized_width, resized_widthStride, resized_height, resized_heightStride);
RETURN_IF_NOT_OK(mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output));
RETURN_IF_NOT_OK((*output)->SetYuvShape(resized_width, resized_widthStride, resized_height, resized_heightStride));
if (!((*output)->HasData())) {
std::string error = "[ERROR] Fail to get the Output result from memory!";
RETURN_STATUS_UNEXPECTED(error);
}
process.device_memory_release();
process.Release();
ret = process.device_memory_release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release device memory failed.");
ret = process.Release();
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release host memory failed.");
// Last part end where we transform the processed data into a tensor which can be applied in later units.
} catch (const cv::Exception &e) {
} catch (const std::exception &e) {
std::string error = "[ERROR] Fail in DvppResizeJpegOp:" + std::string(e.what());
RETURN_STATUS_UNEXPECTED(error);
}
@ -136,7 +139,8 @@ Status DvppResizeJpegOp::SetAscendResource(const std::shared_ptr<DeviceResource>
if (!processor_) {
RETURN_STATUS_UNEXPECTED("Resource initialize fail, please check your env");
}
processor_->SetResizeParas(resized_width_, resized_height_);
APP_ERROR ret = processor_->SetResizeParas(resized_width_, resized_height_);
CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "SetResizeParas failed.");
return Status::OK();
}

28
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/utils/DvppCommon.cc
View File

@ -133,7 +133,7 @@ APP_ERROR DvppCommon::DeInit(void) {
ret = acldvppDestroyChannel(dvppChannelDesc_);
if (ret != APP_ERR_OK) {
MS_LOG(ERROR) << "Failed to destory dvpp channel, ret = " << ret << ".";
MS_LOG(ERROR) << "Failed to destroy dvpp channel, ret = " << ret << ".";
return ret;
}
@ -155,7 +155,7 @@ APP_ERROR DvppCommon::DestroyResource() {
if (vdecChannelDesc_ != nullptr) {
ret = aclvdecDestroyChannel(vdecChannelDesc_);
if (ret != APP_ERR_OK) {
MS_LOG(ERROR) << "Failed to destory dvpp channel, ret = " << ret;
MS_LOG(ERROR) << "Failed to destroy dvpp channel, ret = " << ret;
}
aclvdecDestroyChannelDesc(vdecChannelDesc_);
vdecChannelDesc_ = nullptr;
@ -1066,7 +1066,7 @@ APP_ERROR DvppCommon::CombineJpegdProcess(const RawData &imageInfo, acldvppPixel
return ret;
}
// In TransferImageH2D function, device buffer will be alloced to store the input image before decode
// In TransferImageH2D function, device buffer will be allocated to store the input image before decode
// Need to pay attention to release of the buffer
ret = TransferImageH2D(imageInfo, inputImage_);
if (ret != APP_ERR_OK) {
@ -1167,7 +1167,7 @@ APP_ERROR DvppCommon::CombinePngdProcess(const RawData &imageInfo, acldvppPixelF
return ret;
}
// In TransferImageH2D function, device buffer will be alloced to store the input image
// In TransferImageH2D function, device buffer will be allocated to store the input image
// Need to pay attention to release of the buffer
ret = TransferImageH2D(imageInfo, inputImage_);
if (ret != APP_ERR_OK) {
@ -1231,8 +1231,8 @@ APP_ERROR DvppCommon::TransferYuvDataH2D(const DvppDataInfo &imageinfo) {
RELEASE_DVPP_DATA(device_ptr);
return ret;
}
/* Important!!! decodedImage_ speifies the image in deocded format(RGB OR YUV)
* Not essentailly to be the image after decode.(Specifies the data not in RAW encode format)
/* Important!!! decodedImage_ speifies the image in decoded format(RGB OR YUV)
* Not essentially to be the image after decode.(Specifies the data not in RAW encode format)
* It can also be the image after resize(Very important)
*/
decodedImage_ = std::make_shared<DvppDataInfo>();
@ -1321,7 +1321,7 @@ APP_ERROR DvppCommon::SinkImageH2D(const RawData &imageInfo, acldvppPixelFormat
MS_LOG(ERROR) << "Failed to get size of decode output buffer, ret = " << ret << ".";
return ret;
}
// In TransferImageH2D function, device buffer will be alloced to store the input image before decode
// In TransferImageH2D function, device buffer will be allocated to store the input image before decode
// Need to pay attention to release of the buffer
ret = TransferImageH2D(imageInfo, inputImage_);
if (ret != APP_ERR_OK) {
@ -1380,7 +1380,7 @@ APP_ERROR DvppCommon::SinkImageH2D(const RawData &imageInfo) {
return ret;
}
// In TransferImageH2D function, device buffer will be alloced to store the input image
// In TransferImageH2D function, device buffer will be allocated to store the input image
// Need to pay attention to release of the buffer
ret = TransferImageH2D(imageInfo, inputImage_);
if (ret != APP_ERR_OK) {
@ -1428,7 +1428,7 @@ APP_ERROR DvppCommon::CreateStreamDesc(std::shared_ptr<DvppDataInfo> data) {
modelInBuff = nullptr;
return APP_ERR_ACL_FAILURE;
}
// Create input stream desc which need to be destoryed in vdec callback function
// Create input stream desc which need to be destroyed in vdec callback function
streamInputDesc_ = acldvppCreateStreamDesc();
if (streamInputDesc_ == nullptr) {
MS_LOG(ERROR) << "Failed to create input stream description.";
@ -1476,14 +1476,14 @@ APP_ERROR DvppCommon::CombineVdecProcess(std::shared_ptr<DvppDataInfo> data, voi
}
void *picOutBufferDev = nullptr;
// picOutBufferDev need to be destoryed in vdec callback function
// picOutBufferDev need to be destroyed in vdec callback function
ret = acldvppMalloc(&picOutBufferDev, dataSize);
if (ret != APP_ERR_OK) {
MS_LOG(ERROR) << "Failed to malloc memory with " << dataSize << " bytes, ret = " << ret << ".";
return APP_ERR_ACL_BAD_ALLOC;
}
// picOutputDesc_ will be destoryed in vdec callback function
// picOutputDesc_ will be destroyed in vdec callback function
picOutputDesc_ = acldvppCreatePicDesc();
if (picOutputDesc_ == NULL) {
return APP_ERR_ACL_BAD_ALLOC;
@ -1538,10 +1538,10 @@ APP_ERROR DvppCommon::VdecSendEosFrame() const {
return ret;
}
// destory input stream desc
// destroy input stream desc
ret = acldvppDestroyStreamDesc(eosStreamDesc);
if (ret != ACL_ERROR_NONE) {
MS_LOG(ERROR) << "Fail to destory dvpp stream desc for eos, ret = " << ret << ".";
MS_LOG(ERROR) << "Fail to destroy dvpp stream desc for eos, ret = " << ret << ".";
return ret;
}
return ret;
@ -1738,7 +1738,7 @@ APP_ERROR DvppCommon::CombineJpegeProcess(const RawData &imageInfo, uint32_t wid
inputImage_->format = format;
inputImage_->width = width;
inputImage_->height = height;
// In TransferImageH2D function, device buffer will be alloced to store the input image
// In TransferImageH2D function, device buffer will be allocated to store the input image
// Need to pay attention to release of the buffer
APP_ERROR ret = TransferImageH2D(imageInfo, inputImage_);
if (ret != APP_ERR_OK) {

4
mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc
View File

@ -1447,9 +1447,7 @@ void SVBkSb(int m, int n, int nb, LiteMat w, LiteMat u, LiteMat v, const LiteMat
int nm = std::min(m, n);
for (int i = 0; i < n; i++) {
for (int j = 0; j < nb; j++) {
dst.ptr<T>(i)[0] = 0;
}
dst.ptr<T>(i)[0] = 0;
}
for (int i = 0; i < nm; i++) {

8
mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.h
View File

@ -47,10 +47,10 @@ namespace dataset {
static_cast<int16_t>(::std::min(::std::max(static_cast<int>(X + (X >= 0.f ? 0.5f : -0.5f)), -32768), 32767));
enum PaddBorderType {
PADD_BORDER_CONSTANT = 0,
PADD_BORDER_REPLICATE = 1,
PADD_BORDER_REFLECT_101 = 4,
PADD_BORDER_DEFAULT = PADD_BORDER_REFLECT_101
PADD_BORDER_CONSTANT = 0, /**< Fills the border with constant values. */
PADD_BORDER_REPLICATE = 1, /**< Fills the border with replicate mode. */
PADD_BORDER_REFLECT_101 = 4, /**< Fills the border with reflect 101 mode. */
PADD_BORDER_DEFAULT = PADD_BORDER_REFLECT_101 /**< Default pad mode, use reflect 101 mode. */
};
struct BoxesConfig {

74
mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.h
View File

@ -127,14 +127,14 @@ using FLOAT64_C3 = Chn3<double>;
using FLOAT64_C4 = Chn4<double>;
enum LPixelType {
BGR = 0,
RGB = 1,
RGBA = 2,
RGBA2GRAY = 3,
RGBA2BGR = 4,
RGBA2RGB = 5,
NV212BGR = 6,
NV122BGR = 7,
BGR = 0, /**< Pixel in BGR type. */
RGB = 1, /**< Pixel in RGB type. */
RGBA = 2, /**< Pixel in RGBA type. */
RGBA2GRAY = 3, /**< Convert image from RGBA to GRAY. */
RGBA2BGR = 4, /**< Convert image from RGBA to BGR. */
RGBA2RGB = 5, /**< Convert image from RGBA to RGB. */
NV212BGR = 6, /**< Convert image from NV21 to BGR. */
NV122BGR = 7, /**< Convert image from NV12 to BGR. */
};
enum WARP_BORDER_MODE { WARP_BORDER_MODE_CONSTANT };
@ -142,21 +142,21 @@ enum WARP_BORDER_MODE { WARP_BORDER_MODE_CONSTANT };
class LDataType {
public:
enum Type : uint8_t {
UNKNOWN = 0,
BOOL,
INT8,
UINT8,
INT16,
UINT16,
INT32,
UINT32,
INT64,
UINT64,
FLOAT16,
FLOAT32,
FLOAT64,
DOUBLE,
NUM_OF_TYPES
UNKNOWN = 0, /**< Unknown data type. */
BOOL, /**< BOOL data type. */
INT8, /**< INT8 data type. */
UINT8, /**< UINT8 data type. */
INT16, /**< INT16 data type. */
UINT16, /**< UINT16 data type. */
INT32, /**< INT32 data type. */
UINT32, /**< UINT32 data type. */
INT64, /**< INT64 data type. */
UINT64, /**< UINT64 data type. */
FLOAT16, /**< FLOAT16 data type. */
FLOAT32, /**< FLOAT32 data type. */
FLOAT64, /**< FLOAT64 data type. */
DOUBLE, /**< DOUBLE data type. */
NUM_OF_TYPES /**< number of types. */
};
LDataType() : type_(UNKNOWN) {}
@ -179,20 +179,20 @@ class LDataType {
public:
static inline const uint8_t SIZE_IN_BYTES[] = {
0, // UNKNOWN
1, // BOOL
1, // INT8
1, // UINT8
2, // INT16
2, // UINT16
4, // INT32
4, // UINT32
8, // INT64
8, // UINT64
2, // FLOAT16
4, // FLOAT32
8, // FLOAT64
8, // DOUBLE
0, /**< Unknown size. */
1, /**< Size of BOOL. */
1, /**< Size of INT8. */
1, /**< Size of UINT8. */
2, /**< Size of INT16. */
2, /**< Size of UINT16. */
4, /**< Size of INT32. */
4, /**< Size of UINT32. */
8, /**< Size of INT64. */
8, /**< Size of UINT64. */
2, /**< Size of FLOAT16. */
4, /**< Size of FLOAT32. */
8, /**< Size of FLOAT64. */
8, /**< Size of DOUBLE. */
};
Type type_;

2
mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_and_resize_op.cc
View File

@ -65,7 +65,7 @@ Status RandomCropAndResizeOp::Compute(const TensorRow &input, TensorRow *output)
int h_in = input[i]->shape()[0];
int w_in = input[i]->shape()[1];
if (i == 0) {
(void)GetCropBox(h_in, w_in, &x, &y, &crop_height, &crop_width);
RETURN_IF_NOT_OK(GetCropBox(h_in, w_in, &x, &y, &crop_height, &crop_width));
}
RETURN_IF_NOT_OK(CropAndResize(input[i], &(*output)[i], x, y, crop_height, crop_width, target_height_,
target_width_, interpolation_));

2
mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_decode_resize_op.cc
View File

@ -50,7 +50,7 @@ Status RandomCropDecodeResizeOp::Compute(const TensorRow &input, TensorRow *outp
int w_in = 0;
RETURN_IF_NOT_OK(GetJpegImageInfo(input[i], &w_in, &h_in));
if (i == 0) {
(void)GetCropBox(h_in, w_in, &x, &y, &crop_height, &crop_width);
RETURN_IF_NOT_OK(GetCropBox(h_in, w_in, &x, &y, &crop_height, &crop_width));
}
std::shared_ptr<Tensor> decoded_tensor = nullptr;
RETURN_IF_NOT_OK(JpegCropAndDecode(input[i], &decoded_tensor, x, y, crop_width, crop_height));

7
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/affine_ir.cc
View File

@ -59,6 +59,13 @@ Status AffineOperation::ValidateParams() {
}
// Fill Value
RETURN_IF_NOT_OK(ValidateVectorFillvalue("Affine", fill_value_));
// interpolation
if (interpolation_ != InterpolationMode::kLinear && interpolation_ != InterpolationMode::kNearestNeighbour &&
interpolation_ != InterpolationMode::kCubic && interpolation_ != InterpolationMode::kArea) {
std::string err_msg = "Affine: Invalid InterpolationMode, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}

5
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/cutmix_batch_ir.cc
View File

@ -38,6 +38,11 @@ std::string CutMixBatchOperation::Name() const { return kCutMixBatchOperation; }
Status CutMixBatchOperation::ValidateParams() {
RETURN_IF_NOT_OK(ValidateFloatScalarPositive("CutMixBatch", "alpha", alpha_));
RETURN_IF_NOT_OK(ValidateProbability("CutMixBatch", prob_));
if (image_batch_format_ != ImageBatchFormat::kNHWC && image_batch_format_ != ImageBatchFormat::kNCHW) {
std::string err_msg = "CutMixBatch: Invalid ImageBatchFormat, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}

7
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/pad_ir.cc
View File

@ -41,6 +41,13 @@ Status PadOperation::ValidateParams() {
RETURN_IF_NOT_OK(ValidateVectorPadding("Pad", padding_));
// fill_value
RETURN_IF_NOT_OK(ValidateVectorFillvalue("Pad", fill_value_));
// padding_mode
if (padding_mode_ != BorderType::kConstant && padding_mode_ != BorderType::kEdge &&
padding_mode_ != BorderType::kReflect && padding_mode_ != BorderType::kSymmetric) {
std::string err_msg = "Pad: Invalid BorderType, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}

7
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_affine_ir.cc
View File

@ -121,6 +121,13 @@ Status RandomAffineOperation::ValidateParams() {
}
// Fill Value
RETURN_IF_NOT_OK(ValidateVectorFillvalue("RandomAffine", fill_value_));
// interpolation
if (interpolation_ != InterpolationMode::kLinear && interpolation_ != InterpolationMode::kNearestNeighbour &&
interpolation_ != InterpolationMode::kCubic && interpolation_ != InterpolationMode::kArea) {
std::string err_msg = "RandomAffine: Invalid InterpolationMode, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}

7
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_crop_ir.cc
View File

@ -50,6 +50,13 @@ Status RandomCropOperation::ValidateParams() {
RETURN_IF_NOT_OK(ValidateVectorPadding("RandomCrop", padding_));
// fill_value
RETURN_IF_NOT_OK(ValidateVectorFillvalue("RandomCrop", fill_value_));
// padding_mode
if (padding_mode_ != BorderType::kConstant && padding_mode_ != BorderType::kEdge &&
padding_mode_ != BorderType::kReflect && padding_mode_ != BorderType::kSymmetric) {
std::string err_msg = "RandomCrop: Invalid BorderType, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}

7
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_crop_with_bbox_ir.cc
View File

@ -50,6 +50,13 @@ Status RandomCropWithBBoxOperation::ValidateParams() {
RETURN_IF_NOT_OK(ValidateVectorPadding("RandomCropWithBBox", padding_));
// fill_value
RETURN_IF_NOT_OK(ValidateVectorFillvalue("RandomCropWithBBox", fill_value_));
// padding_mode
if (padding_mode_ != BorderType::kConstant && padding_mode_ != BorderType::kEdge &&
padding_mode_ != BorderType::kReflect && padding_mode_ != BorderType::kSymmetric) {
std::string err_msg = "RandomCropWithBBox: Invalid BorderType, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}

7
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_resized_crop_ir.cc
View File

@ -58,6 +58,13 @@ Status RandomResizedCropOperation::ValidateParams() {
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
// interpolation
if (interpolation_ != InterpolationMode::kLinear && interpolation_ != InterpolationMode::kNearestNeighbour &&
interpolation_ != InterpolationMode::kCubic && interpolation_ != InterpolationMode::kArea) {
std::string err_msg = "RandomResizedCrop: Invalid InterpolationMode, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}

7
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_resized_crop_with_bbox_ir.cc
View File

@ -53,6 +53,13 @@ Status RandomResizedCropWithBBoxOperation::ValidateParams() {
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
// interpolation
if (interpolation_ != InterpolationMode::kLinear && interpolation_ != InterpolationMode::kNearestNeighbour &&
interpolation_ != InterpolationMode::kCubic && interpolation_ != InterpolationMode::kArea) {
std::string err_msg = "RandomResizedCropWithBBox: Invalid InterpolationMode, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}

8
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_rotation_ir.cc
View File

@ -78,6 +78,14 @@ Status RandomRotationOperation::ValidateParams() {
}
// fill_value
RETURN_IF_NOT_OK(ValidateVectorFillvalue("RandomRotation", fill_value_));
// interpolation
if (interpolation_mode_ != InterpolationMode::kLinear &&
interpolation_mode_ != InterpolationMode::kNearestNeighbour && interpolation_mode_ != InterpolationMode::kCubic &&
interpolation_mode_ != InterpolationMode::kArea) {
std::string err_msg = "RandomRotation: Invalid InterpolationMode, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}

7
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/resize_ir.cc
View File

@ -34,6 +34,13 @@ std::string ResizeOperation::Name() const { return kResizeOperation; }
Status ResizeOperation::ValidateParams() {
RETURN_IF_NOT_OK(ValidateVectorSize("Resize", size_));
// interpolation
if (interpolation_ != InterpolationMode::kLinear && interpolation_ != InterpolationMode::kNearestNeighbour &&
interpolation_ != InterpolationMode::kCubic && interpolation_ != InterpolationMode::kArea) {
std::string err_msg = "Resize: Invalid InterpolationMode, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}

7
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/resize_with_bbox_ir.cc
View File

@ -37,6 +37,13 @@ std::string ResizeWithBBoxOperation::Name() const { return kResizeWithBBoxOperat
Status ResizeWithBBoxOperation::ValidateParams() {
RETURN_IF_NOT_OK(ValidateVectorSize("ResizeWithBBox", size_));
// interpolation
if (interpolation_ != InterpolationMode::kLinear && interpolation_ != InterpolationMode::kNearestNeighbour &&
interpolation_ != InterpolationMode::kCubic && interpolation_ != InterpolationMode::kArea) {
std::string err_msg = "ResizeWithBBox: Invalid InterpolationMode, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}

12
mindspore/ccsrc/minddata/dataset/liteapi/include/datasets.h
View File

@ -519,12 +519,12 @@ inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const
class MnistDataset : public Dataset {
public:
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
~MnistDataset() = default;
};

41
mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.cc
View File

@ -65,7 +65,16 @@ BasicTokenizerOperation::BasicTokenizerOperation(bool lower_case, bool keep_whit
preserve_unused_token_(preserve_unused_token),
with_offsets_(with_offsets) {}
Status BasicTokenizerOperation::ValidateParams() { return Status::OK(); }
Status BasicTokenizerOperation::ValidateParams() {
if (normalize_form_ != NormalizeForm::kNone && normalize_form_ != NormalizeForm::kNfc &&
normalize_form_ != NormalizeForm::kNfkc && normalize_form_ != NormalizeForm::kNfd &&
normalize_form_ != NormalizeForm::kNfkd) {
std::string err_msg = "BasicTokenizer: Invalid NormalizeForm, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}
std::shared_ptr<TensorOp> BasicTokenizerOperation::Build() {
std::shared_ptr<BasicTokenizerOp> tensor_op = std::make_shared<BasicTokenizerOp>(
@ -98,6 +107,14 @@ Status BertTokenizerOperation::ValidateParams() {
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (normalize_form_ != NormalizeForm::kNone && normalize_form_ != NormalizeForm::kNfc &&
normalize_form_ != NormalizeForm::kNfkc && normalize_form_ != NormalizeForm::kNfd &&
normalize_form_ != NormalizeForm::kNfkd) {
std::string err_msg = "BertTokenizer: Invalid NormalizeForm, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (max_bytes_per_token_ < 0) {
std::string err_msg = "BertTokenizer : The parameter max_bytes_per_token must be greater than or equal to 0: " +
std::to_string(max_bytes_per_token_);
@ -142,6 +159,12 @@ Status JiebaTokenizerOperation::ValidateParams() {
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (mode_ != JiebaMode::kMix && mode_ != JiebaMode::kMp && mode_ != JiebaMode::kHmm) {
std::string err_msg = "JiebaTokenizer: Invalid JiebaMode, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
RETURN_IF_NOT_OK(ValidateTokenizerDirParam("JiebaTokenizer", hmm_path_));
RETURN_IF_NOT_OK(ValidateTokenizerDirParam("JiebaTokenizer", mp_path_));
return Status::OK();
@ -264,7 +287,16 @@ std::shared_ptr<TensorOp> NgramOperation::Build() {
// NormalizeUTF8Operation
NormalizeUTF8Operation::NormalizeUTF8Operation(NormalizeForm normalize_form) : normalize_form_(normalize_form) {}
Status NormalizeUTF8Operation::ValidateParams() { return Status::OK(); }
Status NormalizeUTF8Operation::ValidateParams() {
if (normalize_form_ != NormalizeForm::kNone && normalize_form_ != NormalizeForm::kNfc &&
normalize_form_ != NormalizeForm::kNfkc && normalize_form_ != NormalizeForm::kNfd &&
normalize_form_ != NormalizeForm::kNfkd) {
std::string err_msg = "NormalizeUTF8: Invalid NormalizeForm, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}
std::shared_ptr<TensorOp> NormalizeUTF8Operation::Build() {
std::shared_ptr<NormalizeUTF8Op> tensor_op = std::make_shared<NormalizeUTF8Op>(normalize_form_);
@ -308,6 +340,11 @@ SentencePieceTokenizerOperation::SentencePieceTokenizerOperation(const std::stri
: vocab_(nullptr), vocab_path_(vocab_path), load_type_(SPieceTokenizerLoadType::kFile), out_type_(out_type) {}
Status SentencePieceTokenizerOperation::ValidateParams() {
if (out_type_ != SPieceTokenizerOutType::kString && out_type_ != SPieceTokenizerOutType::kInt) {
std::string err_msg = "SentencePieceTokenizer: Invalid SPieceTokenizerOutType, check input value of enum.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (load_type_ == SPieceTokenizerLoadType::kModel) {
if (vocab_ == nullptr) {
std::string err_msg = "SentencePieceTokenizer: vocab object type is incorrect or null.";

6
mindspore/ccsrc/minddata/dataset/text/sentence_piece_vocab.cc
View File

@ -34,6 +34,9 @@ Status SentencePieceVocab::BuildFromFile(const std::vector<std::string> &path_li
const float character_coverage, const SentencePieceModel model_type,
const std::unordered_map<std::string, std::string> &params,
std::shared_ptr<SentencePieceVocab> *vocab) {
if (vocab == nullptr) {
RETURN_STATUS_UNEXPECTED("SentencePieceVocab::BuildFromFile: input vocab can not be null");
}
std::unordered_map<std::string, std::string> unorder_map;
// the input of sentence is comma separated string
@ -85,6 +88,9 @@ Status SentencePieceVocab::BuildFromFile(const std::vector<std::string> &path_li
Status SentencePieceVocab::SaveModel(const std::shared_ptr<SentencePieceVocab> *vocab, std::string path,
std::string filename) {
if (vocab == nullptr) {
RETURN_STATUS_UNEXPECTED("SentencePieceVocab::SaveModel: input vocab can not be null");
}
auto realpath = FileUtils::GetRealPath(path.data());
if (!realpath.has_value()) {
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + path);

58
mindspore/ccsrc/minddata/dataset/text/vocab.cc
View File

@ -41,6 +41,9 @@ WordIdType Vocab::Lookup(const WordType &word) const {
#ifdef ENABLE_PYTHON
Status Vocab::BuildFromPyList(const py::list &words, const py::list &special_tokens, bool prepend_special,
std::shared_ptr<Vocab> *vocab) {
if (vocab == nullptr) {
RETURN_STATUS_UNEXPECTED("Vocab::BuildFromPyList: input vocab can not be null");
}
// check of duplication on both words and special_tokens will be performed in python
// special_tokens and words both need to be unique, and shouldn't overlap
std::unordered_map<WordType, WordIdType> word2id;
@ -62,6 +65,9 @@ Status Vocab::BuildFromPyList(const py::list &words, const py::list &special_tok
}
Status Vocab::BuildFromPyDict(const py::dict &words, std::shared_ptr<Vocab> *vocab) {
if (vocab == nullptr) {
RETURN_STATUS_UNEXPECTED("Vocab::BuildFromPyDict: input vocab can not be null");
}
std::unordered_map<WordType, WordIdType> word2id;
for (auto p : words) {
word2id[py::str(p.first)] = py::reinterpret_borrow<py::int_>(p.second);
@ -79,6 +85,9 @@ void Vocab::append_word(const std::string &word) {
Status Vocab::BuildFromUnorderedMap(const std::unordered_map<WordType, WordIdType> &words,
std::shared_ptr<Vocab> *vocab) {
if (vocab == nullptr) {
RETURN_STATUS_UNEXPECTED("Vocab::BuildFromUnorderedMap: input vocab can not be null");
}
// Validate parameters and build map
std::unordered_map<WordType, WordIdType> word2id;
for (auto p : words) {
@ -93,6 +102,9 @@ Status Vocab::BuildFromUnorderedMap(const std::unordered_map<WordType, WordIdTyp
Status Vocab::BuildFromVector(const std::vector<WordType> &words, const std::vector<WordType> &special_tokens,
bool prepend_special, std::shared_ptr<Vocab> *vocab) {
if (vocab == nullptr) {
RETURN_STATUS_UNEXPECTED("Vocab::BuildFromVector: input vocab can not be null");
}
std::unordered_map<WordType, WordIdType> word2id;
// if special is added in front, normal words id will start from number of special tokens
@ -123,18 +135,16 @@ Status Vocab::BuildFromVector(const std::vector<WordType> &words, const std::vec
Status Vocab::BuildFromFileCpp(const std::string &path, const std::string &delimiter, int32_t vocab_size,
const std::vector<WordType> &special_tokens, bool prepend_special,
std::shared_ptr<Vocab> *vocab) {
if (vocab == nullptr) {
RETURN_STATUS_UNEXPECTED("Vocab::BuildFromFileCpp: input vocab can not be null");
}
// Validate parameters
auto realpath = FileUtils::GetRealPath(path.data());
if (!realpath.has_value()) {
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + path);
}
CHECK_FAIL_RETURN_UNEXPECTED(realpath.has_value(), "Get real path failed, path=" + path);
if (vocab_size < 0 && vocab_size != -1) {
RETURN_STATUS_UNEXPECTED(
"from_file: "
"vocab_size should be either -1 or positive integer, but got " +
std::to_string(vocab_size));
}
CHECK_FAIL_RETURN_UNEXPECTED(
!(vocab_size < 0 && vocab_size != -1),
"from_file: vocab_size should be either -1 or positive integer, but got: " + std::to_string(vocab_size));
std::string duplicate_sp;
for (const WordType &sp : special_tokens) {
@ -144,12 +154,8 @@ Status Vocab::BuildFromFileCpp(const std::string &path, const std::string &delim
}
}
}
if (!duplicate_sp.empty()) {
RETURN_STATUS_UNEXPECTED(
"from_file: "
"special_tokens contains duplicate word: " +
duplicate_sp);
}
CHECK_FAIL_RETURN_UNEXPECTED(duplicate_sp.empty(),
"from_file: special_tokens contains duplicate word: " + duplicate_sp);
std::unordered_set<std::string> specials;
// used to check that words in file don't contain any special token that already exists
@ -160,24 +166,19 @@ Status Vocab::BuildFromFileCpp(const std::string &path, const std::string &delim
std::unordered_map<WordType, WordIdType> word2id;
std::fstream handle(realpath.value(), std::ios::in);
if (!handle.good() || !handle.is_open()) {
RETURN_STATUS_UNEXPECTED("from_file: fail to open: " + realpath.value());
}
CHECK_FAIL_RETURN_UNEXPECTED(handle.good() && handle.is_open(), "from_file: fail to open: " + realpath.value());
std::string word;
while (std::getline(handle, word)) {
if (!delimiter.empty()) {
// if delimiter is not found, find_first_of would return std::string::npos which is -1
word = word.substr(0, word.find_first_of(delimiter));
}
if (word2id.find(word) != word2id.end()) {
RETURN_STATUS_UNEXPECTED("from_file: word_list contains duplicate word:" + word + ".");
}
if (specials.find(word) != specials.end()) {
RETURN_STATUS_UNEXPECTED(
"from_file: "
"special_tokens and word_list contain duplicate word: " +
word);
}
CHECK_FAIL_RETURN_UNEXPECTED(word2id.find(word) == word2id.end(),
"from_file: word_list contains duplicate word:" + word);
CHECK_FAIL_RETURN_UNEXPECTED(specials.find(word) == specials.end(),
"from_file: special_tokens and word_list contain duplicate word:" + word);
word2id[word] = word_id++;
// break if enough row is read, if vocab_size is smaller than 0
if (word2id.size() == vocab_size) break;
@ -195,6 +196,9 @@ Status Vocab::BuildFromFileCpp(const std::string &path, const std::string &delim
Status Vocab::BuildFromFile(const std::string &path, const std::string &delimiter, int32_t vocab_size,
const py::list &special_tokens, bool prepend_special, std::shared_ptr<Vocab> *vocab) {
if (vocab == nullptr) {
RETURN_STATUS_UNEXPECTED("Vocab::BuildFromFile: input vocab can not be null");
}
// python validator checks special_tokens doesn't contain any duplicate words
std::unordered_set<std::string> specials;
// used to check that words in file don't contain any special token that already exists

8
mindspore/ccsrc/minddata/dataset/util/btree.h
View File

@ -316,7 +316,13 @@ class BPlusTree {
void Init() {
typename LeafNode::alloc_type alloc(alloc_);
auto *p = alloc.allocate(1);
LeafNode *p;
try {
p = alloc.allocate(1);
} catch (std::bad_alloc &e) {
p = nullptr;
return;
}
root_ = new (p) LeafNode(alloc_);
all_.Prepend(p);
leaf_nodes_.Append(p);

2
mindspore/ccsrc/minddata/mindrecord/include/common/shard_utils.h
View File

@ -181,7 +181,7 @@ bool CheckIsValidUtf8(const std::string &str);
/// \brief judge if a path is legal file
/// \param path file path
/// \return parent path
/// \return Whether the path is legal or not
bool IsLegalFile(const std::string &path);
enum DiskSizeType { kTotalSize = 0, kFreeSize };

3
mindspore/dataset/engine/validators.py
View File

@ -550,6 +550,9 @@ def check_bucket_batch_by_length(method):
if element_length_function is None and len(column_names) != 1:
raise ValueError("If element_length_function is not specified, exactly one column name should be passed.")
if element_length_function is not None and not callable(element_length_function):
raise TypeError("element_length_function object is not callable.")
# check bucket_boundaries: must be list of int, positive and strictly increasing
if not bucket_boundaries:
raise ValueError("bucket_boundaries cannot be empty.")