diff --git a/include/api/dual_abi_helper.h b/include/api/dual_abi_helper.h index 0dd4ac23f6b..862830edc6f 100644 --- a/include/api/dual_abi_helper.h +++ b/include/api/dual_abi_helper.h @@ -151,6 +151,9 @@ inline std::map PadInfoCharToString(const std::map inline void TensorMapCharToString(const std::map, T> *c, std::unordered_map *s) { + if (c == nullptr || s == nullptr) { + return; + } for (auto ch : *c) { auto key = std::string(ch.first.begin(), ch.first.end()); auto val = ch.second; diff --git a/mindspore/ccsrc/minddata/dataset/api/text.cc b/mindspore/ccsrc/minddata/dataset/api/text.cc index 838ddfcdaa4..02123a3e0c5 100644 --- a/mindspore/ccsrc/minddata/dataset/api/text.cc +++ b/mindspore/ccsrc/minddata/dataset/api/text.cc @@ -314,9 +314,9 @@ std::shared_ptr RegexTokenizer::Parse() { // SentencePieceTokenizer struct SentencePieceTokenizer::Data { Data(const std::shared_ptr &vocab, SPieceTokenizerOutType out_type) - : vocab_(vocab), out_type_(out_type) {} + : vocab_(vocab), vocab_path_(""), out_type_(out_type) {} Data(const std::vector &vocab_path, SPieceTokenizerOutType out_type) - : vocab_path_(CharToString(vocab_path)), out_type_(out_type) {} + : vocab_(nullptr), vocab_path_(CharToString(vocab_path)), out_type_(out_type) {} std::shared_ptr vocab_; std::string vocab_path_; SPieceTokenizerOutType out_type_; diff --git a/mindspore/ccsrc/minddata/dataset/api/vision.cc b/mindspore/ccsrc/minddata/dataset/api/vision.cc index 37e35f2fc8b..bca04791f51 100644 --- a/mindspore/ccsrc/minddata/dataset/api/vision.cc +++ b/mindspore/ccsrc/minddata/dataset/api/vision.cc @@ -194,8 +194,11 @@ std::shared_ptr CenterCrop::Parse(const MapTargetDevice &env) { [](int32_t i) { return (uint32_t)i; }); return std::make_shared(usize_); #endif // ENABLE_ACL + } else if (env == MapTargetDevice::kCpu) { + return std::make_shared(data_->size_); } - return std::make_shared(data_->size_); + MS_LOG(ERROR) << "Unsupported MapTargetDevice, only supported kCpu and kAscend310."; + return nullptr; } #ifndef ENABLE_ANDROID @@ -273,8 +276,11 @@ std::shared_ptr Decode::Parse(const MapTargetDevice &env) { #ifdef ENABLE_ACL return std::make_shared(); #endif // ENABLE_ACL + } else if (env == MapTargetDevice::kCpu) { + return std::make_shared(data_->rgb_); } - return std::make_shared(data_->rgb_); + MS_LOG(ERROR) << "Unsupported MapTargetDevice, only supported kCpu and kAscend310."; + return nullptr; } #ifdef ENABLE_ACL @@ -291,7 +297,11 @@ std::shared_ptr DvppDecodeResizeJpeg::Parse() { } std::shared_ptr DvppDecodeResizeJpeg::Parse(const MapTargetDevice &env) { - return std::make_shared(data_->resize_); + if (env == MapTargetDevice::kAscend310) { + return std::make_shared(data_->resize_); + } + MS_LOG(ERROR) << "Unsupported MapTargetDevice, only supported kAscend310."; + return nullptr; } // DvppDecodeResizeCrop Transform Operation. @@ -309,7 +319,11 @@ std::shared_ptr DvppDecodeResizeCropJpeg::Parse() { } std::shared_ptr DvppDecodeResizeCropJpeg::Parse(const MapTargetDevice &env) { - return std::make_shared(data_->crop_, data_->resize_); + if (env == MapTargetDevice::kAscend310) { + return std::make_shared(data_->crop_, data_->resize_); + } + MS_LOG(ERROR) << "Unsupported MapTargetDevice, only supported kAscend310."; + return nullptr; } // DvppDecodePng Transform Operation. @@ -318,7 +332,11 @@ DvppDecodePng::DvppDecodePng() {} std::shared_ptr DvppDecodePng::Parse() { return std::make_shared(); } std::shared_ptr DvppDecodePng::Parse(const MapTargetDevice &env) { - return std::make_shared(); + if (env == MapTargetDevice::kAscend310) { + return std::make_shared(); + } + MS_LOG(ERROR) << "Unsupported MapTargetDevice, only supported kAscend310."; + return nullptr; } #endif // ENABLE_ACL @@ -389,8 +407,11 @@ std::shared_ptr Normalize::Parse(const MapTargetDevice &env) { #ifdef ENABLE_ACL return std::make_shared(data_->mean_, data_->std_); #endif // ENABLE_ACL + } else if (env == MapTargetDevice::kCpu) { + return std::make_shared(data_->mean_, data_->std_); } - return std::make_shared(data_->mean_, data_->std_); + MS_LOG(ERROR) << "Unsupported MapTargetDevice, only supported kCpu and kAscend310."; + return nullptr; } #ifndef ENABLE_ANDROID @@ -828,8 +849,11 @@ std::shared_ptr Resize::Parse(const MapTargetDevice &env) { [](int32_t i) { return (uint32_t)i; }); return std::make_shared(usize_); #endif // ENABLE_ACL + } else if (env == MapTargetDevice::kCpu) { + return std::make_shared(data_->size_, data_->interpolation_); } - return std::make_shared(data_->size_, data_->interpolation_); + MS_LOG(ERROR) << "Unsupported MapTargetDevice, only supported kCpu and kAscend310."; + return nullptr; } // ResizePreserveAR Transform Operation. diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor_helpers.cc b/mindspore/ccsrc/minddata/dataset/core/tensor_helpers.cc index c358e24dd1d..b8bc9d6d562 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor_helpers.cc +++ b/mindspore/ccsrc/minddata/dataset/core/tensor_helpers.cc @@ -34,6 +34,10 @@ void IndexGeneratorHelper(int8_t depth, std::vector *numbers, int8_t new_depth = depth - 1; // depth is always less than or equal to numbers->size() (based on the caller functions) size_t curr_ind = static_cast(numbers->size() - static_cast(depth)); + if (curr_ind >= slice_list.size()) { + MS_LOG(ERROR) << "The index is out of range in slice_list."; + return; + } if (slice_list[curr_ind].slice_.valid()) { dsize_t increment = slice_list[curr_ind].slice_.step_; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc index 7b922dc6003..2c1cf0b874b 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc @@ -100,6 +100,7 @@ Status DatasetOp::RemoveChild(std::shared_ptr child) { } Status DatasetOp::InsertAsParent(std::shared_ptr to_add) { + RETURN_UNEXPECTED_IF_NULL(to_add); for (auto &prev_parent : this->parent_) { RETURN_IF_NOT_OK(prev_parent->RemoveChild(shared_from_this())); RETURN_IF_NOT_OK(prev_parent->AddChild(to_add)); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc index e6bc3b1b513..9b1082ff5fc 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc @@ -270,6 +270,9 @@ Status CocoOp::ParseAnnotationIds() { } std::ifstream in(realpath.value()); + if (!in.is_open()) { + RETURN_STATUS_UNEXPECTED("Invalid file, failed to open annotation file: " + annotation_path_); + } in >> js; } catch (const std::exception &err) { RETURN_STATUS_UNEXPECTED("Invalid file, failed to open JSON file: " + annotation_path_ + "."); diff --git a/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc b/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc index 7c3f5034fc1..24433f63907 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc @@ -33,6 +33,8 @@ namespace dataset { // Constructor ExecutionTree::ExecutionTree() : id_count_(0), tree_state_(kDeTStateInit) { tg_ = std::make_unique(); + root_ = nullptr; + prepare_flags_ = 0; #ifndef ENABLE_SECURITY profiling_manager_ = std::make_unique(this); #endif diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/build_sentence_piece_vocab_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/build_sentence_piece_vocab_node.cc index 1fcb9aa2d09..1b56cf7bad5 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/build_sentence_piece_vocab_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/build_sentence_piece_vocab_node.cc @@ -70,6 +70,13 @@ Status BuildSentenceVocabNode::ValidateParams() { RETURN_STATUS_SYNTAX_ERROR(err_msg); } + if (model_type_ != SentencePieceModel::kUnigram && model_type_ != SentencePieceModel::kBpe && + model_type_ != SentencePieceModel::kChar && model_type_ != SentencePieceModel::kWord) { + std::string err_msg = "BuildSentenceVocabNode: Invalid SentencePieceModel, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + if (vocab_size_ <= 0) { std::string err_msg = "BuildSentenceVocabNode: vocab_size should be positive, but got: " + std::to_string(vocab_size_); diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/clue_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/clue_node.cc index 4426455e319..c4638b4a043 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/clue_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/clue_node.cc @@ -60,6 +60,12 @@ Status CLUENode::ValidateParams() { RETURN_IF_NOT_OK(ValidateStringValue("CLUENode", usage_, {"train", "test", "eval"})); + if (shuffle_ != ShuffleMode::kFalse && shuffle_ != ShuffleMode::kFiles && shuffle_ != ShuffleMode::kGlobal) { + std::string err_msg = "CLUENode: Invalid ShuffleMode, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + if (num_samples_ < 0) { std::string err_msg = "CLUENode: Invalid number of samples: " + std::to_string(num_samples_); MS_LOG(ERROR) << err_msg; diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/coco_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/coco_node.cc index 8da109391f0..41e16722e5e 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/coco_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/coco_node.cc @@ -63,6 +63,11 @@ Status CocoNode::ValidateParams() { MS_LOG(ERROR) << err_msg; RETURN_STATUS_SYNTAX_ERROR(err_msg); } + if (access(annotation_file_.c_str(), R_OK) == -1) { + std::string err_msg = "CocoNode: No access to specified annotation file: " + annotation_file_; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } RETURN_IF_NOT_OK(ValidateStringValue("CocoNode", task_, {"Detection", "Stuff", "Panoptic", "Keypoint"})); diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/csv_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/csv_node.cc index 83693e14d3b..b648cc6643c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/csv_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/csv_node.cc @@ -70,6 +70,12 @@ Status CSVNode::ValidateParams() { RETURN_STATUS_SYNTAX_ERROR(err_msg); } + if (shuffle_ != ShuffleMode::kFalse && shuffle_ != ShuffleMode::kFiles && shuffle_ != ShuffleMode::kGlobal) { + std::string err_msg = "CSVNode: Invalid ShuffleMode, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + if (num_samples_ < 0) { std::string err_msg = "CSVNode: Invalid number of samples: " + std::to_string(num_samples_); MS_LOG(ERROR) << err_msg; diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/manifest_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/manifest_node.cc index b205bb646b2..576f658c4f9 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/manifest_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/manifest_node.cc @@ -77,6 +77,11 @@ Status ManifestNode::ValidateParams() { MS_LOG(ERROR) << err_msg; RETURN_STATUS_SYNTAX_ERROR(err_msg); } + if (access(dataset_file_.c_str(), R_OK) == -1) { + std::string err_msg = "ManifestNode: No access to specified annotation file: " + dataset_file_; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } RETURN_IF_NOT_OK(ValidateDatasetSampler("ManifestNode", sampler_)); diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/text_file_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/text_file_node.cc index 0e049f61ea3..50afa26283d 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/text_file_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/text_file_node.cc @@ -58,6 +58,12 @@ Status TextFileNode::ValidateParams() { RETURN_IF_NOT_OK(DatasetNode::ValidateParams()); RETURN_IF_NOT_OK(ValidateDatasetFilesParam("TextFileNode", dataset_files_)); + if (shuffle_ != ShuffleMode::kFalse && shuffle_ != ShuffleMode::kFiles && shuffle_ != ShuffleMode::kGlobal) { + std::string err_msg = "TextFileNode: Invalid ShuffleMode, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + if (num_samples_ < 0) { std::string err_msg = "TextFileNode: Invalid number of samples: " + std::to_string(num_samples_); MS_LOG(ERROR) << err_msg; diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc index beb91aea86c..2e64bd0ac94 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc @@ -52,6 +52,13 @@ void TFRecordNode::Print(std::ostream &out) const { // Validator for TFRecordNode Status TFRecordNode::ValidateParams() { RETURN_IF_NOT_OK(DatasetNode::ValidateParams()); + + if (shuffle_ != ShuffleMode::kFalse && shuffle_ != ShuffleMode::kFiles && shuffle_ != ShuffleMode::kGlobal) { + std::string err_msg = "TFRecordNode: Invalid ShuffleMode, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + if (dataset_files_.empty()) { std::string err_msg = "TFRecordNode: dataset_files is not specified."; MS_LOG(ERROR) << err_msg; diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/data_helper.h b/mindspore/ccsrc/minddata/dataset/include/dataset/data_helper.h index 80aec6ba97f..61116044924 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/data_helper.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/data_helper.h @@ -45,6 +45,7 @@ class DataHelper { /// Creates the output directory if doesn't exist /// \param[in] in_dir Image folder directory that takes in images /// \param[in] out_dir Directory containing output json files + /// \return Status The status code returned Status CreateAlbum(const std::string &in_dir, const std::string &out_dir) { return CreateAlbumIF(StringToChar(in_dir), StringToChar(out_dir)); } @@ -339,6 +340,7 @@ class DataHelper { } /// \brief Write pointer to bin, use pointer to avoid memcpy + /// \note The value of `length`` must be equal to the length of `data` /// \param[in] in_file File name to write to /// \param[in] data Pointer to data /// \param[in] length Length of values to write from pointer @@ -346,16 +348,22 @@ class DataHelper { template Status WriteBinFile(const std::string &in_file, T *data, size_t length) { try { + if (data == nullptr) { + return Status(kMDUnexpectedError, "input data can not be null"); + } std::ofstream o(in_file, std::ios::binary | std::ios::out); if (!o.is_open()) { return Status(kMDUnexpectedError, "Error opening Bin file to write"); } o.write(reinterpret_cast(data), std::streamsize(length * sizeof(T))); + if (!o.good()) { + return Status(kMDUnexpectedError, "Error writing Bin file"); + } o.close(); } // Catch any exception and convert to Status return code catch (const std::exception &err) { - return Status(kMDUnexpectedError, "Write bin file failed "); + return Status(kMDUnexpectedError, "Write bin file failed"); } return Status::OK(); } @@ -370,7 +378,7 @@ class DataHelper { size_t DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr, const size_t &buffer_size); /// \brief Helper function to delete key in json file - /// note This function will return okay even if key not found + /// \note This function will return okay even if key not found /// \param[in] in_file Json file to remove key from /// \param[in] key The key to remove /// \return Status The status code returned @@ -383,9 +391,9 @@ class DataHelper { void Print(std::ostream &out) const; /// \brief << Stream output operator overload - /// \notes This allows you to write the debug print info using stream operators + /// \note This allows you to write the debug print info using stream operators /// \param out Reference to the output stream being overloaded - /// \param ds Reference to the DataSchema to display + /// \param dh Reference to the DataSchema to display /// \return The output stream must be returned friend std::ostream &operator<<(std::ostream &out, const DataHelper &dh) { dh.Print(out); diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h b/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h index ccbc7bff6f7..b8fe7cab656 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h @@ -932,9 +932,9 @@ class CelebADataset : public Dataset { /// \param[in] decode Decode the images after reading (default=false). /// \param[in] extensions Set of file extensions to be included in the dataset (default={}). /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit CelebADataset(const std::vector &dataset_dir, const std::vector &usage, - const std::shared_ptr &sampler, bool decode, - const std::set> &extensions, const std::shared_ptr &cache); + CelebADataset(const std::vector &dataset_dir, const std::vector &usage, + const std::shared_ptr &sampler, bool decode, const std::set> &extensions, + const std::shared_ptr &cache); /// \brief Constructor of CelebADataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset. @@ -943,9 +943,8 @@ class CelebADataset : public Dataset { /// \param[in] decode Decode the images after reading (default=false). /// \param[in] extensions Set of file extensions to be included in the dataset (default={}). /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit CelebADataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, - bool decode, const std::set> &extensions, - const std::shared_ptr &cache); + CelebADataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, + bool decode, const std::set> &extensions, const std::shared_ptr &cache); /// \brief Constructor of CelebADataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset. @@ -954,9 +953,9 @@ class CelebADataset : public Dataset { /// \param[in] decode Decode the images after reading (default=false). /// \param[in] extensions Set of file extensions to be included in the dataset (default={}). /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit CelebADataset(const std::vector &dataset_dir, const std::vector &usage, - const std::reference_wrapper sampler, bool decode, - const std::set> &extensions, const std::shared_ptr &cache); + CelebADataset(const std::vector &dataset_dir, const std::vector &usage, + const std::reference_wrapper sampler, bool decode, + const std::set> &extensions, const std::shared_ptr &cache); /// \brief Destructor of CelebADataset. ~CelebADataset() = default; @@ -1027,24 +1026,24 @@ class Cifar10Dataset : public Dataset { /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not /// given, a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()). /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit Cifar10Dataset(const std::vector &dataset_dir, const std::vector &usage, - const std::shared_ptr &sampler, const std::shared_ptr &cache); + Cifar10Dataset(const std::vector &dataset_dir, const std::vector &usage, + const std::shared_ptr &sampler, const std::shared_ptr &cache); /// \brief Constructor of Cifar10Dataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset. /// \param[in] usage Part of dataset of CIFAR10, can be "train", "test" or "all". /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit Cifar10Dataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, - const std::shared_ptr &cache); + Cifar10Dataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, + const std::shared_ptr &cache); /// \brief Constructor of Cifar10Dataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset. /// \param[in] usage Part of dataset of CIFAR10, can be "train", "test" or "all". /// \param[in] sampler Sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit Cifar10Dataset(const std::vector &dataset_dir, const std::vector &usage, - const std::reference_wrapper sampler, const std::shared_ptr &cache); + Cifar10Dataset(const std::vector &dataset_dir, const std::vector &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache); /// \brief Destructor of Cifar10Dataset. ~Cifar10Dataset() = default; @@ -1101,24 +1100,24 @@ class Cifar100Dataset : public Dataset { /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not /// given, a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()). /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit Cifar100Dataset(const std::vector &dataset_dir, const std::vector &usage, - const std::shared_ptr &sampler, const std::shared_ptr &cache); + Cifar100Dataset(const std::vector &dataset_dir, const std::vector &usage, + const std::shared_ptr &sampler, const std::shared_ptr &cache); /// \brief Constructor of Cifar100Dataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset. /// \param[in] usage Part of dataset of CIFAR100, can be "train", "test" or "all". /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit Cifar100Dataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, - const std::shared_ptr &cache); + Cifar100Dataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, + const std::shared_ptr &cache); /// \brief Constructor of Cifar100Dataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset. /// \param[in] usage Part of dataset of CIFAR100, can be "train", "test" or "all". /// \param[in] sampler Sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit Cifar100Dataset(const std::vector &dataset_dir, const std::vector &usage, - const std::reference_wrapper sampler, const std::shared_ptr &cache); + Cifar100Dataset(const std::vector &dataset_dir, const std::vector &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache); /// \brief Destructor of Cifar100Dataset. ~Cifar100Dataset() = default; @@ -1181,9 +1180,9 @@ class CityscapesDataset : public Dataset { /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not /// given, a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()). /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). - explicit CityscapesDataset(const std::vector &dataset_dir, const std::vector &usage, - const std::vector &quality_mode, const std::vector &task, bool decode, - const std::shared_ptr &sampler, const std::shared_ptr &cache); + CityscapesDataset(const std::vector &dataset_dir, const std::vector &usage, + const std::vector &quality_mode, const std::vector &task, bool decode, + const std::shared_ptr &sampler, const std::shared_ptr &cache); /// \brief Constructor of CityscapesDataset. /// \param[in] dataset_dir The dataset dir to be read. @@ -1196,9 +1195,9 @@ class CityscapesDataset : public Dataset { /// \param[in] decode Decode the images after reading. /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). - explicit CityscapesDataset(const std::vector &dataset_dir, const std::vector &usage, - const std::vector &quality_mode, const std::vector &task, bool decode, - const Sampler *sampler, const std::shared_ptr &cache); + CityscapesDataset(const std::vector &dataset_dir, const std::vector &usage, + const std::vector &quality_mode, const std::vector &task, bool decode, + const Sampler *sampler, const std::shared_ptr &cache); /// \brief Constructor of CityscapesDataset. /// \param[in] dataset_dir The dataset dir to be read. @@ -1211,9 +1210,9 @@ class CityscapesDataset : public Dataset { /// \param[in] decode Decode the images after reading. /// \param[in] sampler Sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). - explicit CityscapesDataset(const std::vector &dataset_dir, const std::vector &usage, - const std::vector &quality_mode, const std::vector &task, bool decode, - const std::reference_wrapper sampler, const std::shared_ptr &cache); + CityscapesDataset(const std::vector &dataset_dir, const std::vector &usage, + const std::vector &quality_mode, const std::vector &task, bool decode, + const std::reference_wrapper sampler, const std::shared_ptr &cache); /// \brief Destructor of CityscapesDataset. ~CityscapesDataset() = default; @@ -1303,9 +1302,9 @@ class CLUEDataset : public Dataset { /// \param[in] shard_id The shard ID within num_shards. This argument should be /// specified only when num_shards is also specified (Default = 0). /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit CLUEDataset(const std::vector> &dataset_files, const std::vector &task, - const std::vector &usage, int64_t num_samples, ShuffleMode shuffle, int32_t num_shards, - int32_t shard_id, const std::shared_ptr &cache); + CLUEDataset(const std::vector> &dataset_files, const std::vector &task, + const std::vector &usage, int64_t num_samples, ShuffleMode shuffle, int32_t num_shards, + int32_t shard_id, const std::shared_ptr &cache); /// \brief Destructor of CLUEDataset. ~CLUEDataset() = default; @@ -1484,10 +1483,10 @@ class CSVDataset : public Dataset { /// \param[in] shard_id The shard ID within num_shards. This argument should be /// specified only when num_shards is also specified (Default = 0). /// \param[in] cache Tensor cache to use.(default=nullptr which means no cache is used). - explicit CSVDataset(const std::vector> &dataset_files, char field_delim, - const std::vector> &column_defaults, - const std::vector> &column_names, int64_t num_samples, ShuffleMode shuffle, - int32_t num_shards, int32_t shard_id, const std::shared_ptr &cache); + CSVDataset(const std::vector> &dataset_files, char field_delim, + const std::vector> &column_defaults, + const std::vector> &column_names, int64_t num_samples, ShuffleMode shuffle, + int32_t num_shards, int32_t shard_id, const std::shared_ptr &cache); /// \brief Destructor of CSVDataset. ~CSVDataset() = default; @@ -1538,9 +1537,9 @@ class DIV2KDataset : public Dataset { /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not /// given, a `RandomSampler` will be used to randomly iterate the entire dataset. /// \param[in] cache Tensor cache to use. - explicit DIV2KDataset(const std::vector &dataset_dir, const std::vector &usage, - const std::vector &downgrade, int32_t scale, bool decode, - const std::shared_ptr &sampler, const std::shared_ptr &cache); + DIV2KDataset(const std::vector &dataset_dir, const std::vector &usage, const std::vector &downgrade, + int32_t scale, bool decode, const std::shared_ptr &sampler, + const std::shared_ptr &cache); /// \brief Constructor of DIV2KDataset. /// \param[in] dataset_dir The dataset dir to be read. @@ -1551,9 +1550,8 @@ class DIV2KDataset : public Dataset { /// \param[in] decode Decode the images after reading. /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use. - explicit DIV2KDataset(const std::vector &dataset_dir, const std::vector &usage, - const std::vector &downgrade, int32_t scale, bool decode, const Sampler *sampler, - const std::shared_ptr &cache); + DIV2KDataset(const std::vector &dataset_dir, const std::vector &usage, const std::vector &downgrade, + int32_t scale, bool decode, const Sampler *sampler, const std::shared_ptr &cache); /// \brief Constructor of DIV2KDataset. /// \param[in] dataset_dir The dataset dir to be read. @@ -1564,9 +1562,9 @@ class DIV2KDataset : public Dataset { /// \param[in] decode Decode the images after reading. /// \param[in] sampler Sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use. - explicit DIV2KDataset(const std::vector &dataset_dir, const std::vector &usage, - const std::vector &downgrade, int32_t scale, bool decode, - const std::reference_wrapper sampler, const std::shared_ptr &cache); + DIV2KDataset(const std::vector &dataset_dir, const std::vector &usage, const std::vector &downgrade, + int32_t scale, bool decode, const std::reference_wrapper sampler, + const std::shared_ptr &cache); /// \brief Destructor of DIV2KDataset. ~DIV2KDataset() = default; @@ -1641,8 +1639,8 @@ class FlickrDataset : public Dataset { /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not /// given, a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()). /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). - explicit FlickrDataset(const std::vector &dataset_dir, const std::vector &annotation_file, bool decode, - const std::shared_ptr &sampler, const std::shared_ptr &cache); + FlickrDataset(const std::vector &dataset_dir, const std::vector &annotation_file, bool decode, + const std::shared_ptr &sampler, const std::shared_ptr &cache); /// \brief Constructor of FlickrDataset. /// \param[in] dataset_dir The dataset dir to be read @@ -1650,8 +1648,8 @@ class FlickrDataset : public Dataset { /// \param[in] decode Decode the images after reading. /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). - explicit FlickrDataset(const std::vector &dataset_dir, const std::vector &annotation_file, bool decode, - const Sampler *sampler, const std::shared_ptr &cache); + FlickrDataset(const std::vector &dataset_dir, const std::vector &annotation_file, bool decode, + const Sampler *sampler, const std::shared_ptr &cache); /// \brief Constructor of FlickrDataset. /// \param[in] dataset_dir The dataset dir to be read @@ -1659,8 +1657,8 @@ class FlickrDataset : public Dataset { /// \param[in] decode Decode the images after reading. /// \param[in] sampler Sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). - explicit FlickrDataset(const std::vector &dataset_dir, const std::vector &annotation_file, bool decode, - const std::reference_wrapper sampler, const std::shared_ptr &cache); + FlickrDataset(const std::vector &dataset_dir, const std::vector &annotation_file, bool decode, + const std::reference_wrapper sampler, const std::shared_ptr &cache); /// \brief Destructor of FlickrDataset. ~FlickrDataset() = default; @@ -1725,10 +1723,10 @@ class ImageFolderDataset : public Dataset { /// \param[in] extensions File extensions to be read. /// \param[in] class_indexing a class name to label map. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit ImageFolderDataset(const std::vector &dataset_dir, bool decode, - const std::shared_ptr &sampler, const std::set> &extensions, - const std::map, int32_t> &class_indexing, - const std::shared_ptr &cache); + ImageFolderDataset(const std::vector &dataset_dir, bool decode, const std::shared_ptr &sampler, + const std::set> &extensions, + const std::map, int32_t> &class_indexing, + const std::shared_ptr &cache); /// \brief Constructor of ImageFolderDataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset. @@ -1737,10 +1735,10 @@ class ImageFolderDataset : public Dataset { /// \param[in] extensions File extensions to be read. /// \param[in] class_indexing a class name to label map. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit ImageFolderDataset(const std::vector &dataset_dir, bool decode, const Sampler *sampler, - const std::set> &extensions, - const std::map, int32_t> &class_indexing, - const std::shared_ptr &cache); + ImageFolderDataset(const std::vector &dataset_dir, bool decode, const Sampler *sampler, + const std::set> &extensions, + const std::map, int32_t> &class_indexing, + const std::shared_ptr &cache); /// \brief Constructor of ImageFolderDataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset. @@ -1749,11 +1747,10 @@ class ImageFolderDataset : public Dataset { /// \param[in] extensions File extensions to be read. /// \param[in] class_indexing a class name to label map. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit ImageFolderDataset(const std::vector &dataset_dir, bool decode, - const std::reference_wrapper sampler, - const std::set> &extensions, - const std::map, int32_t> &class_indexing, - const std::shared_ptr &cache); + ImageFolderDataset(const std::vector &dataset_dir, bool decode, const std::reference_wrapper sampler, + const std::set> &extensions, + const std::map, int32_t> &class_indexing, + const std::shared_ptr &cache); /// \brief Destructor of ImageFolderDataset. ~ImageFolderDataset() = default; @@ -1833,10 +1830,9 @@ class ManifestDataset : public Dataset { /// names will be sorted alphabetically and each class will be given a unique index starting from 0). /// \param[in] decode Decode the images after reading (default=false). /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit ManifestDataset(const std::vector &dataset_file, const std::vector &usage, - const std::shared_ptr &sampler, - const std::map, int32_t> &class_indexing, bool decode, - const std::shared_ptr &cache); + ManifestDataset(const std::vector &dataset_file, const std::vector &usage, + const std::shared_ptr &sampler, const std::map, int32_t> &class_indexing, + bool decode, const std::shared_ptr &cache); /// \brief Constructor of ManifestDataset. /// \param[in] dataset_file The dataset file to be read. @@ -1846,9 +1842,9 @@ class ManifestDataset : public Dataset { /// names will be sorted alphabetically and each class will be given a unique index starting from 0). /// \param[in] decode Decode the images after reading (default=false). /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit ManifestDataset(const std::vector &dataset_file, const std::vector &usage, - const Sampler *sampler, const std::map, int32_t> &class_indexing, - bool decode, const std::shared_ptr &cache); + ManifestDataset(const std::vector &dataset_file, const std::vector &usage, const Sampler *sampler, + const std::map, int32_t> &class_indexing, bool decode, + const std::shared_ptr &cache); /// \brief Constructor of ManifestDataset. /// \param[in] dataset_file The dataset file to be read. @@ -1858,10 +1854,10 @@ class ManifestDataset : public Dataset { /// names will be sorted alphabetically and each class will be given a unique index starting from 0). /// \param[in] decode Decode the images after reading (default=false). /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit ManifestDataset(const std::vector &dataset_file, const std::vector &usage, - const std::reference_wrapper sampler, - const std::map, int32_t> &class_indexing, bool decode, - const std::shared_ptr &cache); + ManifestDataset(const std::vector &dataset_file, const std::vector &usage, + const std::reference_wrapper sampler, + const std::map, int32_t> &class_indexing, bool decode, + const std::shared_ptr &cache); /// \brief Destructor of ManifestDataset. ~ManifestDataset() = default; @@ -1945,10 +1941,10 @@ class MindDataDataset : public Dataset { /// ShuffleMode::kGlobal - Shuffle both the files and samples. /// ShuffleMode::kInfile - Shuffle samples in file. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit MindDataDataset(const std::vector &dataset_file, const std::vector> &columns_list, - const std::shared_ptr &sampler, const nlohmann::json *padded_sample, - int64_t num_padded, ShuffleMode shuffle_mode = ShuffleMode::kGlobal, - const std::shared_ptr &cache = nullptr); + MindDataDataset(const std::vector &dataset_file, const std::vector> &columns_list, + const std::shared_ptr &sampler, const nlohmann::json *padded_sample, int64_t num_padded, + ShuffleMode shuffle_mode = ShuffleMode::kGlobal, + const std::shared_ptr &cache = nullptr); /// \brief Constructor of MindDataDataset. /// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source @@ -1965,10 +1961,10 @@ class MindDataDataset : public Dataset { /// ShuffleMode::kGlobal - Shuffle both the files and samples. /// ShuffleMode::kInfile - Shuffle samples in file. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit MindDataDataset(const std::vector &dataset_file, const std::vector> &columns_list, - const Sampler *sampler, const nlohmann::json *padded_sample, int64_t num_padded, - ShuffleMode shuffle_mode = ShuffleMode::kGlobal, - const std::shared_ptr &cache = nullptr); + MindDataDataset(const std::vector &dataset_file, const std::vector> &columns_list, + const Sampler *sampler, const nlohmann::json *padded_sample, int64_t num_padded, + ShuffleMode shuffle_mode = ShuffleMode::kGlobal, + const std::shared_ptr &cache = nullptr); /// \brief Constructor of MindDataDataset. /// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source @@ -1985,10 +1981,10 @@ class MindDataDataset : public Dataset { /// ShuffleMode::kGlobal - Shuffle both the files and samples. /// ShuffleMode::kInfile - Shuffle samples in file. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit MindDataDataset(const std::vector &dataset_file, const std::vector> &columns_list, - const std::reference_wrapper sampler, const nlohmann::json *padded_sample, - int64_t num_padded, ShuffleMode shuffle_mode = ShuffleMode::kGlobal, - const std::shared_ptr &cache = nullptr); + MindDataDataset(const std::vector &dataset_file, const std::vector> &columns_list, + const std::reference_wrapper sampler, const nlohmann::json *padded_sample, + int64_t num_padded, ShuffleMode shuffle_mode = ShuffleMode::kGlobal, + const std::shared_ptr &cache = nullptr); /// \brief Constructor of MindDataDataset. /// \param[in] dataset_files List of dataset files to be read directly. @@ -2004,11 +2000,11 @@ class MindDataDataset : public Dataset { /// ShuffleMode::kGlobal - Shuffle both the files and samples. /// ShuffleMode::kInfile - Shuffle data within each file. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit MindDataDataset(const std::vector> &dataset_files, - const std::vector> &columns_list, const std::shared_ptr &sampler, - const nlohmann::json *padded_sample, int64_t num_padded, - ShuffleMode shuffle_mode = ShuffleMode::kGlobal, - const std::shared_ptr &cache = nullptr); + MindDataDataset(const std::vector> &dataset_files, + const std::vector> &columns_list, const std::shared_ptr &sampler, + const nlohmann::json *padded_sample, int64_t num_padded, + ShuffleMode shuffle_mode = ShuffleMode::kGlobal, + const std::shared_ptr &cache = nullptr); /// \brief Constructor of MindDataDataset. /// \param[in] dataset_files List of dataset files to be read directly. @@ -2024,11 +2020,11 @@ class MindDataDataset : public Dataset { /// ShuffleMode::kGlobal - Shuffle both the files and samples. /// ShuffleMode::kInfile - Shuffle data within each file. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit MindDataDataset(const std::vector> &dataset_files, - const std::vector> &columns_list, const Sampler *sampler, - const nlohmann::json *padded_sample, int64_t num_padded, - ShuffleMode shuffle_mode = ShuffleMode::kGlobal, - const std::shared_ptr &cache = nullptr); + MindDataDataset(const std::vector> &dataset_files, + const std::vector> &columns_list, const Sampler *sampler, + const nlohmann::json *padded_sample, int64_t num_padded, + ShuffleMode shuffle_mode = ShuffleMode::kGlobal, + const std::shared_ptr &cache = nullptr); /// \brief Constructor of MindDataDataset. /// \param[in] dataset_files List of dataset files to be read directly. @@ -2044,11 +2040,11 @@ class MindDataDataset : public Dataset { /// ShuffleMode::kGlobal - Shuffle both the files and samples. /// ShuffleMode::kInfile - Shuffle samples in file. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit MindDataDataset(const std::vector> &dataset_files, - const std::vector> &columns_list, - const std::reference_wrapper sampler, const nlohmann::json *padded_sample, - int64_t num_padded, ShuffleMode shuffle_mode = ShuffleMode::kGlobal, - const std::shared_ptr &cache = nullptr); + MindDataDataset(const std::vector> &dataset_files, + const std::vector> &columns_list, const std::reference_wrapper sampler, + const nlohmann::json *padded_sample, int64_t num_padded, + ShuffleMode shuffle_mode = ShuffleMode::kGlobal, + const std::shared_ptr &cache = nullptr); /// \brief Destructor of MindDataDataset. ~MindDataDataset() = default; @@ -2214,24 +2210,24 @@ class MnistDataset : public Dataset { /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not /// given, a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()). /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit MnistDataset(const std::vector &dataset_dir, const std::vector &usage, - const std::shared_ptr &sampler, const std::shared_ptr &cache); + MnistDataset(const std::vector &dataset_dir, const std::vector &usage, + const std::shared_ptr &sampler, const std::shared_ptr &cache); /// \brief Constructor of MnistDataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset. /// \param[in] usage Part of dataset of MNIST, can be "train", "test" or "all". /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit MnistDataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, - const std::shared_ptr &cache); + MnistDataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, + const std::shared_ptr &cache); /// \brief Constructor of MnistDataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset. /// \param[in] usage Part of dataset of MNIST, can be "train", "test" or "all". /// \param[in] sampler Sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit MnistDataset(const std::vector &dataset_dir, const std::vector &usage, - const std::reference_wrapper sampler, const std::shared_ptr &cache); + MnistDataset(const std::vector &dataset_dir, const std::vector &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache); /// Destructor of MnistDataset. ~MnistDataset() = default; @@ -2342,24 +2338,24 @@ class SBUDataset : public Dataset { /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not /// given, a `RandomSampler` will be used to randomly iterate the entire dataset. /// \param[in] cache Tensor cache to use. - explicit SBUDataset(const std::vector &dataset_dir, bool decode, const std::shared_ptr &sampler, - const std::shared_ptr &cache); + SBUDataset(const std::vector &dataset_dir, bool decode, const std::shared_ptr &sampler, + const std::shared_ptr &cache); /// \brief Constructor of SBUDataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset. /// \param[in] decode Decode the images after reading. /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use. - explicit SBUDataset(const std::vector &dataset_dir, bool decode, const Sampler *sampler, - const std::shared_ptr &cache); + SBUDataset(const std::vector &dataset_dir, bool decode, const Sampler *sampler, + const std::shared_ptr &cache); /// \brief Constructor of SBUDataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset. /// \param[in] decode Decode the images after reading. /// \param[in] sampler Sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use. - explicit SBUDataset(const std::vector &dataset_dir, bool decode, const std::reference_wrapper sampler, - const std::shared_ptr &cache); + SBUDataset(const std::vector &dataset_dir, bool decode, const std::reference_wrapper sampler, + const std::shared_ptr &cache); /// Destructor of SBUDataset. ~SBUDataset() = default; @@ -2422,9 +2418,8 @@ class TextFileDataset : public Dataset { /// \param[in] shard_id The shard ID within num_shards. This argument should be /// specified only when num_shards is also specified (Default = 0). /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit TextFileDataset(const std::vector> &dataset_files, int64_t num_samples, - ShuffleMode shuffle, int32_t num_shards, int32_t shard_id, - const std::shared_ptr &cache); + TextFileDataset(const std::vector> &dataset_files, int64_t num_samples, ShuffleMode shuffle, + int32_t num_shards, int32_t shard_id, const std::shared_ptr &cache); /// Destructor of TextFileDataset. ~TextFileDataset() = default; @@ -2582,9 +2577,8 @@ class USPSDataset : public Dataset { /// \param[in] shard_id The shard ID within num_shards. This argument should be /// specified only when num_shards is also specified (Default = 0). /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). - explicit USPSDataset(const std::vector &dataset_dir, const std::vector &usage, int64_t num_samples, - ShuffleMode shuffle, int32_t num_shards, int32_t shard_id, - const std::shared_ptr &cache); + USPSDataset(const std::vector &dataset_dir, const std::vector &usage, int64_t num_samples, + ShuffleMode shuffle, int32_t num_shards, int32_t shard_id, const std::shared_ptr &cache); /// Destructor of USPSDataset. ~USPSDataset() = default; @@ -2628,10 +2622,9 @@ class VOCDataset : public Dataset { /// given, a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()). /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). /// \param[in] extra_metadata Flag to add extra meta-data to row (default=false). - explicit VOCDataset(const std::vector &dataset_dir, const std::vector &task, - const std::vector &usage, const std::map, int32_t> &class_indexing, - bool decode, const std::shared_ptr &sampler, const std::shared_ptr &cache, - bool extra_metadata); + VOCDataset(const std::vector &dataset_dir, const std::vector &task, const std::vector &usage, + const std::map, int32_t> &class_indexing, bool decode, + const std::shared_ptr &sampler, const std::shared_ptr &cache, bool extra_metadata); /// \brief Constructor of VOCDataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset. @@ -2642,10 +2635,9 @@ class VOCDataset : public Dataset { /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). /// \param[in] extra_metadata Flag to add extra meta-data to row (default=false). - explicit VOCDataset(const std::vector &dataset_dir, const std::vector &task, - const std::vector &usage, const std::map, int32_t> &class_indexing, - bool decode, const Sampler *sampler, const std::shared_ptr &cache, - bool extra_metadata); + VOCDataset(const std::vector &dataset_dir, const std::vector &task, const std::vector &usage, + const std::map, int32_t> &class_indexing, bool decode, const Sampler *sampler, + const std::shared_ptr &cache, bool extra_metadata); /// \brief Constructor of VOCDataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset. @@ -2656,10 +2648,10 @@ class VOCDataset : public Dataset { /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). /// \param[in] extra_metadata Flag to add extra meta-data to row (default=false). - explicit VOCDataset(const std::vector &dataset_dir, const std::vector &task, - const std::vector &usage, const std::map, int32_t> &class_indexing, - bool decode, const std::reference_wrapper sampler, - const std::shared_ptr &cache, bool extra_metadata); + VOCDataset(const std::vector &dataset_dir, const std::vector &task, const std::vector &usage, + const std::map, int32_t> &class_indexing, bool decode, + const std::reference_wrapper sampler, const std::shared_ptr &cache, + bool extra_metadata); /// Destructor of VOCDataset. ~VOCDataset() = default; diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/text.h b/mindspore/ccsrc/minddata/dataset/include/dataset/text.h index c35de781284..3ee0957d910 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/text.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/text.h @@ -113,10 +113,10 @@ class BertTokenizer final : public TensorTransform { /// \param[in] preserve_unused_token If true, do not split special tokens like '[CLS]', '[SEP]', '[UNK]', '[PAD]' and /// '[MASK]' (default=true). /// \param[in] with_offsets Whether to output offsets of tokens (default=false). - explicit BertTokenizer(const std::shared_ptr &vocab, const std::vector &suffix_indicator, - int32_t max_bytes_per_token, const std::vector &unknown_token, bool lower_case, - bool keep_whitespace, const NormalizeForm normalize_form, bool preserve_unused_token, - bool with_offsets); + BertTokenizer(const std::shared_ptr &vocab, const std::vector &suffix_indicator, + int32_t max_bytes_per_token, const std::vector &unknown_token, bool lower_case, + bool keep_whitespace, const NormalizeForm normalize_form, bool preserve_unused_token, + bool with_offsets); /// \brief Destructor ~BertTokenizer() = default; @@ -162,8 +162,8 @@ class JiebaTokenizer final : public TensorTransform { /// - JiebaMode.kHMM, tokenizes with Hidden Markov Model Segment algorithm. /// - JiebaMode.kMIX, tokenizes with a mix of MPSegment and HMMSegment algorithms. /// \param[in] with_offsets Whether to output offsets of tokens (default=false). - explicit JiebaTokenizer(const std::string &hmm_path, const std::string &mp_path, - const JiebaMode &mode = JiebaMode::kMix, bool with_offsets = false) + JiebaTokenizer(const std::string &hmm_path, const std::string &mp_path, const JiebaMode &mode = JiebaMode::kMix, + bool with_offsets = false) : JiebaTokenizer(StringToChar(hmm_path), StringToChar(mp_path), mode, with_offsets) {} /// \brief Constructor. @@ -177,8 +177,8 @@ class JiebaTokenizer final : public TensorTransform { /// - JiebaMode.kHMM, tokenizes with Hidden Markov Model Segment algorithm. /// - JiebaMode.kMIX, tokenizes with a mix of MPSegment and HMMSegment algorithms. /// \param[in] with_offsets Whether to output offsets of tokens (default=false). - explicit JiebaTokenizer(const std::vector &hmm_path, const std::vector &mp_path, const JiebaMode &mode, - bool with_offsets); + JiebaTokenizer(const std::vector &hmm_path, const std::vector &mp_path, const JiebaMode &mode, + bool with_offsets); /// \brief Destructor ~JiebaTokenizer() = default; @@ -255,8 +255,8 @@ class Lookup final : public TensorTransform { /// runtime error will be thrown (default={}, means no unknown_token is specified). /// \param[in] data_type mindspore::DataType of the tensor after lookup; must be numeric, including bool. /// (default=mindspore::DataType::kNumberTypeInt32). - explicit Lookup(const std::shared_ptr &vocab, const std::optional> &unknown_token, - mindspore::DataType data_type = mindspore::DataType::kNumberTypeInt32); + Lookup(const std::shared_ptr &vocab, const std::optional> &unknown_token, + mindspore::DataType data_type = mindspore::DataType::kNumberTypeInt32); /// \brief Destructor ~Lookup() = default; @@ -296,8 +296,8 @@ class Ngram final : public TensorTransform { /// \param[in] right_pad {"pad_token", pad_width}. Padding performed on right side of the sequence.pad_width will /// be capped at n-1. right_pad=("-",2) would pad the right side of the sequence with "--" (default={"", 0}}). /// \param[in] separator Symbol used to join strings together (default=" "). - explicit Ngram(const std::vector &ngrams, const std::pair, int32_t> &left_pad, - const std::pair, int32_t> &right_pad, const std::vector &separator); + Ngram(const std::vector &ngrams, const std::pair, int32_t> &left_pad, + const std::pair, int32_t> &right_pad, const std::vector &separator); /// \brief Destructor ~Ngram() = default; @@ -348,7 +348,7 @@ class RegexReplace final : public TensorTransform { /// \param[in] replace The string to replace the matched element. /// \param[in] replace_all Confirm whether to replace all. If false, only replace the first matched element; /// if true, replace all matched elements (default=true). - explicit RegexReplace(std::string pattern, std::string replace, bool replace_all = true) + RegexReplace(std::string pattern, std::string replace, bool replace_all = true) : RegexReplace(StringToChar(pattern), StringToChar(replace), replace_all) {} /// \brief Constructor. @@ -356,7 +356,7 @@ class RegexReplace final : public TensorTransform { /// \param[in] replace The string to replace the matched element. /// \param[in] replace_all Confirm whether to replace all. If false, only replace the first matched element; /// if true, replace all matched elements (default=true). - explicit RegexReplace(const std::vector &pattern, const std::vector &replace, bool replace_all); + RegexReplace(const std::vector &pattern, const std::vector &replace, bool replace_all); /// \brief Destructor ~RegexReplace() = default; diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/vision.h b/mindspore/ccsrc/minddata/dataset/include/dataset/vision.h index ba9ae786437..b5eee10143a 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/vision.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/vision.h @@ -269,11 +269,10 @@ class NormalizePad final : public TensorTransform { /// The standard deviation values must be in range (0.0, 255.0]. /// \param[in] dtype The output datatype of Tensor. /// The standard deviation values must be "float32" or "float16"(default = "float32"). - explicit NormalizePad(const std::vector &mean, const std::vector &std, - const std::string &dtype = "float32") + NormalizePad(const std::vector &mean, const std::vector &std, const std::string &dtype = "float32") : NormalizePad(mean, std, StringToChar(dtype)) {} - explicit NormalizePad(const std::vector &mean, const std::vector &std, const std::vector &dtype); + NormalizePad(const std::vector &mean, const std::vector &std, const std::vector &dtype); /// \brief Destructor. ~NormalizePad() = default; @@ -332,7 +331,7 @@ class RandomColor final : public TensorTransform { /// \brief Constructor. /// \param[in] t_lb Lower bound random weights. /// \param[in] t_ub Upper bound random weights. - explicit RandomColor(float t_lb, float t_ub); + RandomColor(float t_lb, float t_ub); /// \brief Destructor. ~RandomColor() = default; diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/vision_ascend.h b/mindspore/ccsrc/minddata/dataset/include/dataset/vision_ascend.h index b3918a30043..22909d7869a 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/vision_ascend.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/vision_ascend.h @@ -64,7 +64,7 @@ class DvppDecodeResizeCropJpeg final : public TensorTransform { /// \brief Constructor. /// \param[in] crop Parameter vector of two integers for each dimension after final crop, with respect to H,W order. /// \param[in] resize Parameter vector of two integers for each dimension after resize, with respect to H,W order. - explicit DvppDecodeResizeCropJpeg(std::vector crop, std::vector resize); + DvppDecodeResizeCropJpeg(std::vector crop, std::vector resize); /// \brief Destructor. ~DvppDecodeResizeCropJpeg() = default; diff --git a/mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.cc b/mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.cc index 3856c5179ce..70fa333da3b 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.cc @@ -354,10 +354,11 @@ Status ToFloat16(const std::shared_ptr &input, std::shared_ptr * RETURN_IF_NOT_OK(Tensor::CreateEmpty(input->shape(), new_type, output)); auto in_itr = input->begin(); + auto in_end = input->end(); auto out_itr = (*output)->begin(); auto out_end = (*output)->end(); - for (; out_itr != out_end; ++in_itr, ++out_itr) { + for (; (in_itr != in_end) && (out_itr != out_end); ++in_itr, ++out_itr) { float element = *in_itr; float float16_max = static_cast(std::numeric_limits::max()); float float16_min = static_cast(std::numeric_limits::lowest()); diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_crop_jpeg_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_crop_jpeg_op.cc index 30ce6e7fd9d..661cc937590 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_crop_jpeg_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_crop_jpeg_op.cc @@ -38,21 +38,22 @@ Status DvppCropJpegOp::Compute(const std::shared_ptr &input, std:: } APP_ERROR ret = processor_->JPEG_C(last_step); if (ret != APP_ERR_OK) { - processor_->Release(); + ret = processor_->Release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release memory failed."); std::string error = "Error in dvpp crop processing:" + std::to_string(ret); RETURN_STATUS_UNEXPECTED(error); } std::shared_ptr CropOut(processor_->Get_Croped_DeviceData()); const TensorShape dvpp_shape({1, 1, 1}); const DataType dvpp_data_type(DataType::DE_UINT8); - mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output); - (*output)->SetAttributes(CropOut->data, CropOut->dataSize, CropOut->width, CropOut->widthStride, CropOut->height, - CropOut->heightStride); + RETURN_IF_NOT_OK(mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output)); + RETURN_IF_NOT_OK((*output)->SetAttributes(CropOut->data, CropOut->dataSize, CropOut->width, CropOut->widthStride, + CropOut->height, CropOut->heightStride)); if (!((*output)->HasDeviceData())) { std::string error = "[ERROR] Fail to get the Output result from device memory!"; RETURN_STATUS_UNEXPECTED(error); } - } catch (const cv::Exception &e) { + } catch (const std::exception &e) { std::string error = "[ERROR] Fail in DvppCropJpegOp:" + std::string(e.what()); RETURN_STATUS_UNEXPECTED(error); } @@ -71,6 +72,8 @@ Status DvppCropJpegOp::Compute(const std::shared_ptr &input, std::shared imageinfo.dataSize = input->SizeInBytes(); imageinfo.data = static_cast(buffer); std::vector yuv_shape_ = input->GetYuvShape(); + const size_t yuv_shape_size = 4; + CHECK_FAIL_RETURN_UNEXPECTED(yuv_shape_.size() == yuv_shape_size, "yuv_shape requires 4 elements."); imageinfo.width = yuv_shape_[0]; imageinfo.widthStride = yuv_shape_[1]; imageinfo.height = yuv_shape_[2]; @@ -114,16 +117,18 @@ Status DvppCropJpegOp::Compute(const std::shared_ptr &input, std::shared uint32_t crop_width = CropOut->width; uint32_t crop_widthStride = CropOut->widthStride; const DataType dvpp_data_type(DataType::DE_UINT8); - mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output); - (*output)->SetYuvShape(crop_width, crop_widthStride, crop_height, crop_heightStride); + RETURN_IF_NOT_OK(mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output)); + RETURN_IF_NOT_OK((*output)->SetYuvShape(crop_width, crop_widthStride, crop_height, crop_heightStride)); if (!((*output)->HasData())) { std::string error = "[ERROR] Fail to get the Output result from memory!"; RETURN_STATUS_UNEXPECTED(error); } - process.device_memory_release(); - process.Release(); + ret = process.device_memory_release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release device memory failed."); + ret = process.Release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release host memory failed."); // Last part end where we transform the processed data into a tensor which can be applied in later units. - } catch (const cv::Exception &e) { + } catch (const std::exception &e) { std::string error = "[ERROR] Fail in DvppCropJpegOp:" + std::string(e.what()); RETURN_STATUS_UNEXPECTED(error); } @@ -147,7 +152,8 @@ Status DvppCropJpegOp::SetAscendResource(const std::shared_ptr & if (!processor_) { RETURN_STATUS_UNEXPECTED("Resource initialize fail, please check your env"); } - processor_->SetCropParas(crop_width_, crop_height_); + APP_ERROR ret = processor_->SetCropParas(crop_width_, crop_height_); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "SetCropParas failed."); return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_jpeg_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_jpeg_op.cc index 7b902e049f1..0e0ca95801f 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_jpeg_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_jpeg_op.cc @@ -35,21 +35,22 @@ Status DvppDecodeJpegOp::Compute(const std::shared_ptr &input, std CHECK_FAIL_RETURN_UNEXPECTED(input->GetDeviceBuffer() != nullptr, "The input image buffer on device is empty"); APP_ERROR ret = processor_->JPEG_D(); if (ret != APP_ERR_OK) { - processor_->Release(); + ret = processor_->Release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release memory failed."); std::string error = "Error in dvpp processing:" + std::to_string(ret); RETURN_STATUS_UNEXPECTED(error); } std::shared_ptr DecodeOut(processor_->Get_Decode_DeviceData()); const TensorShape dvpp_shape({1, 1, 1}); const DataType dvpp_data_type(DataType::DE_UINT8); - mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output); - (*output)->SetAttributes(DecodeOut->data, DecodeOut->dataSize, DecodeOut->width, DecodeOut->widthStride, - DecodeOut->height, DecodeOut->heightStride); + RETURN_IF_NOT_OK(mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output)); + RETURN_IF_NOT_OK((*output)->SetAttributes(DecodeOut->data, DecodeOut->dataSize, DecodeOut->width, + DecodeOut->widthStride, DecodeOut->height, DecodeOut->heightStride)); if (!((*output)->HasDeviceData())) { std::string error = "[ERROR] Fail to get the Output result from memory!"; RETURN_STATUS_UNEXPECTED(error); } - } catch (const cv::Exception &e) { + } catch (const std::exception &e) { std::string error = "[ERROR] Fail in DvppDecodeJpegOp:" + std::string(e.what()); RETURN_STATUS_UNEXPECTED(error); } @@ -106,16 +107,18 @@ Status DvppDecodeJpegOp::Compute(const std::shared_ptr &input, std::shar const TensorShape dvpp_shape({dvpp_length, 1, 1}); const DataType dvpp_data_type(DataType::DE_UINT8); - mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output); - (*output)->SetYuvShape(decoded_width, decoded_widthStride, decoded_height, decoded_heightStride); + RETURN_IF_NOT_OK(mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output)); + RETURN_IF_NOT_OK((*output)->SetYuvShape(decoded_width, decoded_widthStride, decoded_height, decoded_heightStride)); if (!((*output)->HasData())) { std::string error = "[ERROR] Fail to get the Output result from device memory!"; RETURN_STATUS_UNEXPECTED(error); } - process.device_memory_release(); - process.Release(); + ret = process.device_memory_release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release device memory failed."); + ret = process.Release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release host memory failed."); // Last part end where we transform the processed data into a tensor which can be applied in later units. - } catch (const cv::Exception &e) { + } catch (const std::exception &e) { std::string error = "[ERROR] Fail in DvppDecodeJpegOp:" + std::string(e.what()); RETURN_STATUS_UNEXPECTED(error); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_png_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_png_op.cc index c8406cee967..783fab464f2 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_png_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_png_op.cc @@ -34,21 +34,22 @@ Status DvppDecodePngOp::Compute(const std::shared_ptr &input, std: CHECK_FAIL_RETURN_UNEXPECTED(input->GetDeviceBuffer() != nullptr, "The input image buffer on device is empty"); APP_ERROR ret = processor_->PNG_D(); if (ret != APP_ERR_OK) { - processor_->Release(); + ret = processor_->Release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release memory failed."); std::string error = "Error in dvpp processing:" + std::to_string(ret); RETURN_STATUS_UNEXPECTED(error); } std::shared_ptr DecodeOut(processor_->Get_Decode_DeviceData()); const TensorShape dvpp_shape({1, 1, 1}); const DataType dvpp_data_type(DataType::DE_UINT8); - mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output); - (*output)->SetAttributes(DecodeOut->data, DecodeOut->dataSize, DecodeOut->width, DecodeOut->widthStride, - DecodeOut->height, DecodeOut->heightStride); + RETURN_IF_NOT_OK(mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output)); + RETURN_IF_NOT_OK((*output)->SetAttributes(DecodeOut->data, DecodeOut->dataSize, DecodeOut->width, + DecodeOut->widthStride, DecodeOut->height, DecodeOut->heightStride)); if (!((*output)->HasDeviceData())) { std::string error = "[ERROR] Fail to get the Output result from memory!"; RETURN_STATUS_UNEXPECTED(error); } - } catch (const cv::Exception &e) { + } catch (const std::exception &e) { std::string error = "[ERROR] Fail in DvppDecodeJpegOp:" + std::string(e.what()); RETURN_STATUS_UNEXPECTED(error); } @@ -95,8 +96,6 @@ Status DvppDecodePngOp::Compute(const std::shared_ptr &input, std::share } // Third part end where we execute the core function of dvpp - /* 测试Device内存 - */ auto data = std::static_pointer_cast(process.Get_Memory_Data()); unsigned char *ret_ptr = data.get(); std::shared_ptr DecodeOut(process.Get_Decode_DeviceData()); @@ -109,10 +108,12 @@ Status DvppDecodePngOp::Compute(const std::shared_ptr &input, std::share std::string error = "[ERROR] Fail to get the Output result from memory!"; RETURN_STATUS_UNEXPECTED(error); } - process.device_memory_release(); - process.Release(); + ret = process.device_memory_release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release device memory failed."); + ret = process.Release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release host memory failed."); // Last part end where we transform the processed data into a tensor which can be applied in later units. - } catch (const cv::Exception &e) { + } catch (const std::exception &e) { std::string error = "[ERROR] Fail in DvppDecodePngOp:" + std::string(e.what()); RETURN_STATUS_UNEXPECTED(error); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_resize_crop_jpeg_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_resize_crop_jpeg_op.cc index 153a9818e9b..245a12d0701 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_resize_crop_jpeg_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_resize_crop_jpeg_op.cc @@ -34,7 +34,8 @@ Status DvppDecodeResizeCropJpegOp::Compute(const std::shared_ptr & CHECK_FAIL_RETURN_UNEXPECTED(input->GetDeviceBuffer() != nullptr, "The input image buffer on device is empty"); APP_ERROR ret = processor_->JPEG_DRC(); if (ret != APP_ERR_OK) { - processor_->Release(); + ret = processor_->Release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release memory failed."); std::string error = "Error in dvpp processing:" + std::to_string(ret); RETURN_STATUS_UNEXPECTED(error); } @@ -42,14 +43,14 @@ Status DvppDecodeResizeCropJpegOp::Compute(const std::shared_ptr & const TensorShape dvpp_shape({1, 1, 1}); const DataType dvpp_data_type(DataType::DE_UINT8); - mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output); - (*output)->SetAttributes(CropOut->data, CropOut->dataSize, CropOut->width, CropOut->widthStride, CropOut->height, - CropOut->heightStride); + RETURN_IF_NOT_OK(mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output)); + RETURN_IF_NOT_OK((*output)->SetAttributes(CropOut->data, CropOut->dataSize, CropOut->width, CropOut->widthStride, + CropOut->height, CropOut->heightStride)); if (!((*output)->HasDeviceData())) { std::string error = "[ERROR] Fail to get the Output result from memory!"; RETURN_STATUS_UNEXPECTED(error); } - } catch (const cv::Exception &e) { + } catch (const std::exception &e) { std::string error = "[ERROR] Fail in DvppDecodeResizeCropJpegOp:" + std::string(e.what()); RETURN_STATUS_UNEXPECTED(error); } @@ -102,15 +103,17 @@ Status DvppDecodeResizeCropJpegOp::Compute(const std::shared_ptr &input, uint32_t dvpp_length = CropOut->dataSize; const TensorShape dvpp_shape({dvpp_length, 1, 1}); const DataType dvpp_data_type(DataType::DE_UINT8); - mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output); + RETURN_IF_NOT_OK(mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output)); if (!((*output)->HasData())) { std::string error = "[ERROR] Fail to get the Output result from memory!"; RETURN_STATUS_UNEXPECTED(error); } - processor.device_memory_release(); - processor.Release(); + ret = processor.device_memory_release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release device memory failed."); + ret = processor.Release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release host memory failed."); // Last part end where we transform the processed data into a tensor which can be applied in later units. - } catch (const cv::Exception &e) { + } catch (const std::exception &e) { std::string error = "[ERROR] Fail in DvppDecodeResizeCropJpegOp:" + std::string(e.what()); RETURN_STATUS_UNEXPECTED(error); } @@ -135,8 +138,10 @@ Status DvppDecodeResizeCropJpegOp::SetAscendResource(const std::shared_ptrSetResizeParas(resized_width_, resized_height_); - processor_->SetCropParas(crop_width_, crop_height_); + APP_ERROR ret = processor_->SetResizeParas(resized_width_, resized_height_); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "SetResizeParas failed."); + ret = processor_->SetCropParas(crop_width_, crop_height_); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "SetCropParas failed."); return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_resize_jpeg_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_resize_jpeg_op.cc index ffe61bf2508..4adf7f24cc3 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_resize_jpeg_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_resize_jpeg_op.cc @@ -34,21 +34,22 @@ Status DvppDecodeResizeJpegOp::Compute(const std::shared_ptr &inpu CHECK_FAIL_RETURN_UNEXPECTED(input->GetDeviceBuffer() != nullptr, "The input image buffer on device is empty"); APP_ERROR ret = processor_->JPEG_DR(); if (ret != APP_ERR_OK) { - processor_->Release(); + ret = processor_->Release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release memory failed."); std::string error = "Error in dvpp processing:" + std::to_string(ret); RETURN_STATUS_UNEXPECTED(error); } std::shared_ptr ResizeOut(processor_->Get_Resized_DeviceData()); const TensorShape dvpp_shape({1, 1, 1}); const DataType dvpp_data_type(DataType::DE_UINT8); - mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output); - (*output)->SetAttributes(ResizeOut->data, ResizeOut->dataSize, ResizeOut->width, ResizeOut->widthStride, - ResizeOut->height, ResizeOut->heightStride); + RETURN_IF_NOT_OK(mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output)); + RETURN_IF_NOT_OK((*output)->SetAttributes(ResizeOut->data, ResizeOut->dataSize, ResizeOut->width, + ResizeOut->widthStride, ResizeOut->height, ResizeOut->heightStride)); if (!((*output)->HasDeviceData())) { std::string error = "[ERROR] Fail to get the Output result from memory!"; RETURN_STATUS_UNEXPECTED(error); } - } catch (const cv::Exception &e) { + } catch (const std::exception &e) { std::string error = "[ERROR] Fail in DvppDecodeResizeJpegOp:" + std::string(e.what()); RETURN_STATUS_UNEXPECTED(error); } @@ -101,15 +102,17 @@ Status DvppDecodeResizeJpegOp::Compute(const std::shared_ptr &input, std dsize_t dvpp_length = ResizeOut->dataSize; const TensorShape dvpp_shape({dvpp_length, 1, 1}); const DataType dvpp_data_type(DataType::DE_UINT8); - mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output); + RETURN_IF_NOT_OK(mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output)); if (!((*output)->HasData())) { std::string error = "[ERROR] Fail to get the Output result from memory!"; RETURN_STATUS_UNEXPECTED(error); } - process.device_memory_release(); - process.Release(); + ret = process.device_memory_release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release device memory failed."); + ret = process.Release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release host memory failed."); // Last part end where we transform the processed data into a tensor which can be applied in later units. - } catch (const cv::Exception &e) { + } catch (const std::exception &e) { std::string error = "[ERROR] Fail in DvppDecodeResizeJpegOp:" + std::string(e.what()); RETURN_STATUS_UNEXPECTED(error); } @@ -133,7 +136,8 @@ Status DvppDecodeResizeJpegOp::SetAscendResource(const std::shared_ptrSetResizeParas(resized_width_, resized_height_); + APP_ERROR ret = processor_->SetResizeParas(resized_width_, resized_height_); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "SetResizeParas failed."); return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_normalize_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_normalize_op.cc index 4e18d705169..093f5a2b34a 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_normalize_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_normalize_op.cc @@ -22,10 +22,12 @@ namespace dataset { Status DvppNormalizeOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { const TensorShape dvpp_shape({1, 1, 1}); const DataType dvpp_data_type(DataType::DE_UINT8); - mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output); + RETURN_IF_NOT_OK(mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output)); std::vector yuv_shape = input->GetYuvStrideShape(); - (*output)->SetAttributes(input->GetDeviceMutableBuffer(), input->DeviceDataSize(), yuv_shape[0], yuv_shape[1], - yuv_shape[2], yuv_shape[3]); + const size_t yuv_shape_size = 4; + CHECK_FAIL_RETURN_UNEXPECTED(yuv_shape.size() == yuv_shape_size, "yuv_shape requires 4 elements."); + RETURN_IF_NOT_OK((*output)->SetAttributes(input->GetDeviceMutableBuffer(), input->DeviceDataSize(), yuv_shape[0], + yuv_shape[1], yuv_shape[2], yuv_shape[3])); if (!((*output)->HasDeviceData())) { std::string error = "[ERROR] Fail to get the output result from device memory!"; RETURN_STATUS_UNEXPECTED(error); diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_resize_jpeg_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_resize_jpeg_op.cc index ebbacc19568..e87283c2a65 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_resize_jpeg_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_resize_jpeg_op.cc @@ -39,21 +39,22 @@ Status DvppResizeJpegOp::Compute(const std::shared_ptr &input, std } APP_ERROR ret = processor_->JPEG_R(last_step); if (ret != APP_ERR_OK) { - processor_->Release(); + ret = processor_->Release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release memory failed."); std::string error = "Error in dvpp processing:" + std::to_string(ret); RETURN_STATUS_UNEXPECTED(error); } std::shared_ptr ResizeOut(processor_->Get_Resized_DeviceData()); const TensorShape dvpp_shape({1, 1, 1}); const DataType dvpp_data_type(DataType::DE_UINT8); - mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output); - (*output)->SetAttributes(ResizeOut->data, ResizeOut->dataSize, ResizeOut->width, ResizeOut->widthStride, - ResizeOut->height, ResizeOut->heightStride); + RETURN_IF_NOT_OK(mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output)); + RETURN_IF_NOT_OK((*output)->SetAttributes(ResizeOut->data, ResizeOut->dataSize, ResizeOut->width, + ResizeOut->widthStride, ResizeOut->height, ResizeOut->heightStride)); if (!((*output)->HasDeviceData())) { std::string error = "[ERROR] Fail to get the Output result from device memory!"; RETURN_STATUS_UNEXPECTED(error); } - } catch (const cv::Exception &e) { + } catch (const std::exception &e) { std::string error = "[ERROR] Fail in DvppResizeJpegOp:" + std::string(e.what()); RETURN_STATUS_UNEXPECTED(error); } @@ -115,16 +116,18 @@ Status DvppResizeJpegOp::Compute(const std::shared_ptr &input, std::shar uint32_t resized_width = ResizeOut->width; uint32_t resized_widthStride = ResizeOut->widthStride; const DataType dvpp_data_type(DataType::DE_UINT8); - mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output); - (*output)->SetYuvShape(resized_width, resized_widthStride, resized_height, resized_heightStride); + RETURN_IF_NOT_OK(mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, output)); + RETURN_IF_NOT_OK((*output)->SetYuvShape(resized_width, resized_widthStride, resized_height, resized_heightStride)); if (!((*output)->HasData())) { std::string error = "[ERROR] Fail to get the Output result from memory!"; RETURN_STATUS_UNEXPECTED(error); } - process.device_memory_release(); - process.Release(); + ret = process.device_memory_release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release device memory failed."); + ret = process.Release(); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "Release host memory failed."); // Last part end where we transform the processed data into a tensor which can be applied in later units. - } catch (const cv::Exception &e) { + } catch (const std::exception &e) { std::string error = "[ERROR] Fail in DvppResizeJpegOp:" + std::string(e.what()); RETURN_STATUS_UNEXPECTED(error); } @@ -136,7 +139,8 @@ Status DvppResizeJpegOp::SetAscendResource(const std::shared_ptr if (!processor_) { RETURN_STATUS_UNEXPECTED("Resource initialize fail, please check your env"); } - processor_->SetResizeParas(resized_width_, resized_height_); + APP_ERROR ret = processor_->SetResizeParas(resized_width_, resized_height_); + CHECK_FAIL_RETURN_UNEXPECTED(ret == APP_ERR_OK, "SetResizeParas failed."); return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/utils/DvppCommon.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/utils/DvppCommon.cc index 0661badbc44..0a399f932a7 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/utils/DvppCommon.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/utils/DvppCommon.cc @@ -133,7 +133,7 @@ APP_ERROR DvppCommon::DeInit(void) { ret = acldvppDestroyChannel(dvppChannelDesc_); if (ret != APP_ERR_OK) { - MS_LOG(ERROR) << "Failed to destory dvpp channel, ret = " << ret << "."; + MS_LOG(ERROR) << "Failed to destroy dvpp channel, ret = " << ret << "."; return ret; } @@ -155,7 +155,7 @@ APP_ERROR DvppCommon::DestroyResource() { if (vdecChannelDesc_ != nullptr) { ret = aclvdecDestroyChannel(vdecChannelDesc_); if (ret != APP_ERR_OK) { - MS_LOG(ERROR) << "Failed to destory dvpp channel, ret = " << ret; + MS_LOG(ERROR) << "Failed to destroy dvpp channel, ret = " << ret; } aclvdecDestroyChannelDesc(vdecChannelDesc_); vdecChannelDesc_ = nullptr; @@ -1066,7 +1066,7 @@ APP_ERROR DvppCommon::CombineJpegdProcess(const RawData &imageInfo, acldvppPixel return ret; } - // In TransferImageH2D function, device buffer will be alloced to store the input image before decode + // In TransferImageH2D function, device buffer will be allocated to store the input image before decode // Need to pay attention to release of the buffer ret = TransferImageH2D(imageInfo, inputImage_); if (ret != APP_ERR_OK) { @@ -1167,7 +1167,7 @@ APP_ERROR DvppCommon::CombinePngdProcess(const RawData &imageInfo, acldvppPixelF return ret; } - // In TransferImageH2D function, device buffer will be alloced to store the input image + // In TransferImageH2D function, device buffer will be allocated to store the input image // Need to pay attention to release of the buffer ret = TransferImageH2D(imageInfo, inputImage_); if (ret != APP_ERR_OK) { @@ -1231,8 +1231,8 @@ APP_ERROR DvppCommon::TransferYuvDataH2D(const DvppDataInfo &imageinfo) { RELEASE_DVPP_DATA(device_ptr); return ret; } - /* Important!!! decodedImage_ speifies the image in deocded format(RGB OR YUV) - * Not essentailly to be the image after decode.(Specifies the data not in RAW encode format) + /* Important!!! decodedImage_ speifies the image in decoded format(RGB OR YUV) + * Not essentially to be the image after decode.(Specifies the data not in RAW encode format) * It can also be the image after resize(Very important) */ decodedImage_ = std::make_shared(); @@ -1321,7 +1321,7 @@ APP_ERROR DvppCommon::SinkImageH2D(const RawData &imageInfo, acldvppPixelFormat MS_LOG(ERROR) << "Failed to get size of decode output buffer, ret = " << ret << "."; return ret; } - // In TransferImageH2D function, device buffer will be alloced to store the input image before decode + // In TransferImageH2D function, device buffer will be allocated to store the input image before decode // Need to pay attention to release of the buffer ret = TransferImageH2D(imageInfo, inputImage_); if (ret != APP_ERR_OK) { @@ -1380,7 +1380,7 @@ APP_ERROR DvppCommon::SinkImageH2D(const RawData &imageInfo) { return ret; } - // In TransferImageH2D function, device buffer will be alloced to store the input image + // In TransferImageH2D function, device buffer will be allocated to store the input image // Need to pay attention to release of the buffer ret = TransferImageH2D(imageInfo, inputImage_); if (ret != APP_ERR_OK) { @@ -1428,7 +1428,7 @@ APP_ERROR DvppCommon::CreateStreamDesc(std::shared_ptr data) { modelInBuff = nullptr; return APP_ERR_ACL_FAILURE; } - // Create input stream desc which need to be destoryed in vdec callback function + // Create input stream desc which need to be destroyed in vdec callback function streamInputDesc_ = acldvppCreateStreamDesc(); if (streamInputDesc_ == nullptr) { MS_LOG(ERROR) << "Failed to create input stream description."; @@ -1476,14 +1476,14 @@ APP_ERROR DvppCommon::CombineVdecProcess(std::shared_ptr data, voi } void *picOutBufferDev = nullptr; - // picOutBufferDev need to be destoryed in vdec callback function + // picOutBufferDev need to be destroyed in vdec callback function ret = acldvppMalloc(&picOutBufferDev, dataSize); if (ret != APP_ERR_OK) { MS_LOG(ERROR) << "Failed to malloc memory with " << dataSize << " bytes, ret = " << ret << "."; return APP_ERR_ACL_BAD_ALLOC; } - // picOutputDesc_ will be destoryed in vdec callback function + // picOutputDesc_ will be destroyed in vdec callback function picOutputDesc_ = acldvppCreatePicDesc(); if (picOutputDesc_ == NULL) { return APP_ERR_ACL_BAD_ALLOC; @@ -1538,10 +1538,10 @@ APP_ERROR DvppCommon::VdecSendEosFrame() const { return ret; } - // destory input stream desc + // destroy input stream desc ret = acldvppDestroyStreamDesc(eosStreamDesc); if (ret != ACL_ERROR_NONE) { - MS_LOG(ERROR) << "Fail to destory dvpp stream desc for eos, ret = " << ret << "."; + MS_LOG(ERROR) << "Fail to destroy dvpp stream desc for eos, ret = " << ret << "."; return ret; } return ret; @@ -1738,7 +1738,7 @@ APP_ERROR DvppCommon::CombineJpegeProcess(const RawData &imageInfo, uint32_t wid inputImage_->format = format; inputImage_->width = width; inputImage_->height = height; - // In TransferImageH2D function, device buffer will be alloced to store the input image + // In TransferImageH2D function, device buffer will be allocated to store the input image // Need to pay attention to release of the buffer APP_ERROR ret = TransferImageH2D(imageInfo, inputImage_); if (ret != APP_ERR_OK) { diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc index 95a690da65d..7b6cb1f172f 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc @@ -1447,9 +1447,7 @@ void SVBkSb(int m, int n, int nb, LiteMat w, LiteMat u, LiteMat v, const LiteMat int nm = std::min(m, n); for (int i = 0; i < n; i++) { - for (int j = 0; j < nb; j++) { - dst.ptr(i)[0] = 0; - } + dst.ptr(i)[0] = 0; } for (int i = 0; i < nm; i++) { diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.h b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.h index dd1d741d552..087dbb57e2a 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.h +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.h @@ -47,10 +47,10 @@ namespace dataset { static_cast(::std::min(::std::max(static_cast(X + (X >= 0.f ? 0.5f : -0.5f)), -32768), 32767)); enum PaddBorderType { - PADD_BORDER_CONSTANT = 0, - PADD_BORDER_REPLICATE = 1, - PADD_BORDER_REFLECT_101 = 4, - PADD_BORDER_DEFAULT = PADD_BORDER_REFLECT_101 + PADD_BORDER_CONSTANT = 0, /**< Fills the border with constant values. */ + PADD_BORDER_REPLICATE = 1, /**< Fills the border with replicate mode. */ + PADD_BORDER_REFLECT_101 = 4, /**< Fills the border with reflect 101 mode. */ + PADD_BORDER_DEFAULT = PADD_BORDER_REFLECT_101 /**< Default pad mode, use reflect 101 mode. */ }; struct BoxesConfig { diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.h b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.h index a689db34787..c7bee9c1b83 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.h +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.h @@ -127,14 +127,14 @@ using FLOAT64_C3 = Chn3; using FLOAT64_C4 = Chn4; enum LPixelType { - BGR = 0, - RGB = 1, - RGBA = 2, - RGBA2GRAY = 3, - RGBA2BGR = 4, - RGBA2RGB = 5, - NV212BGR = 6, - NV122BGR = 7, + BGR = 0, /**< Pixel in BGR type. */ + RGB = 1, /**< Pixel in RGB type. */ + RGBA = 2, /**< Pixel in RGBA type. */ + RGBA2GRAY = 3, /**< Convert image from RGBA to GRAY. */ + RGBA2BGR = 4, /**< Convert image from RGBA to BGR. */ + RGBA2RGB = 5, /**< Convert image from RGBA to RGB. */ + NV212BGR = 6, /**< Convert image from NV21 to BGR. */ + NV122BGR = 7, /**< Convert image from NV12 to BGR. */ }; enum WARP_BORDER_MODE { WARP_BORDER_MODE_CONSTANT }; @@ -142,21 +142,21 @@ enum WARP_BORDER_MODE { WARP_BORDER_MODE_CONSTANT }; class LDataType { public: enum Type : uint8_t { - UNKNOWN = 0, - BOOL, - INT8, - UINT8, - INT16, - UINT16, - INT32, - UINT32, - INT64, - UINT64, - FLOAT16, - FLOAT32, - FLOAT64, - DOUBLE, - NUM_OF_TYPES + UNKNOWN = 0, /**< Unknown data type. */ + BOOL, /**< BOOL data type. */ + INT8, /**< INT8 data type. */ + UINT8, /**< UINT8 data type. */ + INT16, /**< INT16 data type. */ + UINT16, /**< UINT16 data type. */ + INT32, /**< INT32 data type. */ + UINT32, /**< UINT32 data type. */ + INT64, /**< INT64 data type. */ + UINT64, /**< UINT64 data type. */ + FLOAT16, /**< FLOAT16 data type. */ + FLOAT32, /**< FLOAT32 data type. */ + FLOAT64, /**< FLOAT64 data type. */ + DOUBLE, /**< DOUBLE data type. */ + NUM_OF_TYPES /**< number of types. */ }; LDataType() : type_(UNKNOWN) {} @@ -179,20 +179,20 @@ class LDataType { public: static inline const uint8_t SIZE_IN_BYTES[] = { - 0, // UNKNOWN - 1, // BOOL - 1, // INT8 - 1, // UINT8 - 2, // INT16 - 2, // UINT16 - 4, // INT32 - 4, // UINT32 - 8, // INT64 - 8, // UINT64 - 2, // FLOAT16 - 4, // FLOAT32 - 8, // FLOAT64 - 8, // DOUBLE + 0, /**< Unknown size. */ + 1, /**< Size of BOOL. */ + 1, /**< Size of INT8. */ + 1, /**< Size of UINT8. */ + 2, /**< Size of INT16. */ + 2, /**< Size of UINT16. */ + 4, /**< Size of INT32. */ + 4, /**< Size of UINT32. */ + 8, /**< Size of INT64. */ + 8, /**< Size of UINT64. */ + 2, /**< Size of FLOAT16. */ + 4, /**< Size of FLOAT32. */ + 8, /**< Size of FLOAT64. */ + 8, /**< Size of DOUBLE. */ }; Type type_; diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_and_resize_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_and_resize_op.cc index 1e58a9e09c6..3b9107dac5d 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_and_resize_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_and_resize_op.cc @@ -65,7 +65,7 @@ Status RandomCropAndResizeOp::Compute(const TensorRow &input, TensorRow *output) int h_in = input[i]->shape()[0]; int w_in = input[i]->shape()[1]; if (i == 0) { - (void)GetCropBox(h_in, w_in, &x, &y, &crop_height, &crop_width); + RETURN_IF_NOT_OK(GetCropBox(h_in, w_in, &x, &y, &crop_height, &crop_width)); } RETURN_IF_NOT_OK(CropAndResize(input[i], &(*output)[i], x, y, crop_height, crop_width, target_height_, target_width_, interpolation_)); diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_decode_resize_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_decode_resize_op.cc index 2521bfed36f..5e4782f49de 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_decode_resize_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_decode_resize_op.cc @@ -50,7 +50,7 @@ Status RandomCropDecodeResizeOp::Compute(const TensorRow &input, TensorRow *outp int w_in = 0; RETURN_IF_NOT_OK(GetJpegImageInfo(input[i], &w_in, &h_in)); if (i == 0) { - (void)GetCropBox(h_in, w_in, &x, &y, &crop_height, &crop_width); + RETURN_IF_NOT_OK(GetCropBox(h_in, w_in, &x, &y, &crop_height, &crop_width)); } std::shared_ptr decoded_tensor = nullptr; RETURN_IF_NOT_OK(JpegCropAndDecode(input[i], &decoded_tensor, x, y, crop_width, crop_height)); diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/affine_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/affine_ir.cc index cc05c637bb3..f8b1d6a6d32 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/affine_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/affine_ir.cc @@ -59,6 +59,13 @@ Status AffineOperation::ValidateParams() { } // Fill Value RETURN_IF_NOT_OK(ValidateVectorFillvalue("Affine", fill_value_)); + // interpolation + if (interpolation_ != InterpolationMode::kLinear && interpolation_ != InterpolationMode::kNearestNeighbour && + interpolation_ != InterpolationMode::kCubic && interpolation_ != InterpolationMode::kArea) { + std::string err_msg = "Affine: Invalid InterpolationMode, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/cutmix_batch_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/cutmix_batch_ir.cc index 49df9682d66..fab2c82a579 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/cutmix_batch_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/cutmix_batch_ir.cc @@ -38,6 +38,11 @@ std::string CutMixBatchOperation::Name() const { return kCutMixBatchOperation; } Status CutMixBatchOperation::ValidateParams() { RETURN_IF_NOT_OK(ValidateFloatScalarPositive("CutMixBatch", "alpha", alpha_)); RETURN_IF_NOT_OK(ValidateProbability("CutMixBatch", prob_)); + if (image_batch_format_ != ImageBatchFormat::kNHWC && image_batch_format_ != ImageBatchFormat::kNCHW) { + std::string err_msg = "CutMixBatch: Invalid ImageBatchFormat, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/pad_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/pad_ir.cc index 3e5499b41db..dbcedf3871d 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/pad_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/pad_ir.cc @@ -41,6 +41,13 @@ Status PadOperation::ValidateParams() { RETURN_IF_NOT_OK(ValidateVectorPadding("Pad", padding_)); // fill_value RETURN_IF_NOT_OK(ValidateVectorFillvalue("Pad", fill_value_)); + // padding_mode + if (padding_mode_ != BorderType::kConstant && padding_mode_ != BorderType::kEdge && + padding_mode_ != BorderType::kReflect && padding_mode_ != BorderType::kSymmetric) { + std::string err_msg = "Pad: Invalid BorderType, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_affine_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_affine_ir.cc index 2c4fc91eedb..247957ab7cb 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_affine_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_affine_ir.cc @@ -121,6 +121,13 @@ Status RandomAffineOperation::ValidateParams() { } // Fill Value RETURN_IF_NOT_OK(ValidateVectorFillvalue("RandomAffine", fill_value_)); + // interpolation + if (interpolation_ != InterpolationMode::kLinear && interpolation_ != InterpolationMode::kNearestNeighbour && + interpolation_ != InterpolationMode::kCubic && interpolation_ != InterpolationMode::kArea) { + std::string err_msg = "RandomAffine: Invalid InterpolationMode, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_crop_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_crop_ir.cc index 19611028949..62a8b1d0ad1 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_crop_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_crop_ir.cc @@ -50,6 +50,13 @@ Status RandomCropOperation::ValidateParams() { RETURN_IF_NOT_OK(ValidateVectorPadding("RandomCrop", padding_)); // fill_value RETURN_IF_NOT_OK(ValidateVectorFillvalue("RandomCrop", fill_value_)); + // padding_mode + if (padding_mode_ != BorderType::kConstant && padding_mode_ != BorderType::kEdge && + padding_mode_ != BorderType::kReflect && padding_mode_ != BorderType::kSymmetric) { + std::string err_msg = "RandomCrop: Invalid BorderType, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_crop_with_bbox_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_crop_with_bbox_ir.cc index 2329dffae52..ff4351a6e09 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_crop_with_bbox_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_crop_with_bbox_ir.cc @@ -50,6 +50,13 @@ Status RandomCropWithBBoxOperation::ValidateParams() { RETURN_IF_NOT_OK(ValidateVectorPadding("RandomCropWithBBox", padding_)); // fill_value RETURN_IF_NOT_OK(ValidateVectorFillvalue("RandomCropWithBBox", fill_value_)); + // padding_mode + if (padding_mode_ != BorderType::kConstant && padding_mode_ != BorderType::kEdge && + padding_mode_ != BorderType::kReflect && padding_mode_ != BorderType::kSymmetric) { + std::string err_msg = "RandomCropWithBBox: Invalid BorderType, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_resized_crop_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_resized_crop_ir.cc index 535537851d0..83220ea6d60 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_resized_crop_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_resized_crop_ir.cc @@ -58,6 +58,13 @@ Status RandomResizedCropOperation::ValidateParams() { MS_LOG(ERROR) << err_msg; RETURN_STATUS_SYNTAX_ERROR(err_msg); } + // interpolation + if (interpolation_ != InterpolationMode::kLinear && interpolation_ != InterpolationMode::kNearestNeighbour && + interpolation_ != InterpolationMode::kCubic && interpolation_ != InterpolationMode::kArea) { + std::string err_msg = "RandomResizedCrop: Invalid InterpolationMode, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_resized_crop_with_bbox_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_resized_crop_with_bbox_ir.cc index e33d4dfc02c..7c90d5275f0 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_resized_crop_with_bbox_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_resized_crop_with_bbox_ir.cc @@ -53,6 +53,13 @@ Status RandomResizedCropWithBBoxOperation::ValidateParams() { MS_LOG(ERROR) << err_msg; RETURN_STATUS_SYNTAX_ERROR(err_msg); } + // interpolation + if (interpolation_ != InterpolationMode::kLinear && interpolation_ != InterpolationMode::kNearestNeighbour && + interpolation_ != InterpolationMode::kCubic && interpolation_ != InterpolationMode::kArea) { + std::string err_msg = "RandomResizedCropWithBBox: Invalid InterpolationMode, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_rotation_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_rotation_ir.cc index 4926d3ab574..47d67413e2d 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_rotation_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_rotation_ir.cc @@ -78,6 +78,14 @@ Status RandomRotationOperation::ValidateParams() { } // fill_value RETURN_IF_NOT_OK(ValidateVectorFillvalue("RandomRotation", fill_value_)); + // interpolation + if (interpolation_mode_ != InterpolationMode::kLinear && + interpolation_mode_ != InterpolationMode::kNearestNeighbour && interpolation_mode_ != InterpolationMode::kCubic && + interpolation_mode_ != InterpolationMode::kArea) { + std::string err_msg = "RandomRotation: Invalid InterpolationMode, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/resize_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/resize_ir.cc index 50d328745bb..8bd4d3d2c84 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/resize_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/resize_ir.cc @@ -34,6 +34,13 @@ std::string ResizeOperation::Name() const { return kResizeOperation; } Status ResizeOperation::ValidateParams() { RETURN_IF_NOT_OK(ValidateVectorSize("Resize", size_)); + // interpolation + if (interpolation_ != InterpolationMode::kLinear && interpolation_ != InterpolationMode::kNearestNeighbour && + interpolation_ != InterpolationMode::kCubic && interpolation_ != InterpolationMode::kArea) { + std::string err_msg = "Resize: Invalid InterpolationMode, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/resize_with_bbox_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/resize_with_bbox_ir.cc index 05503c348e3..b7aa7403469 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/resize_with_bbox_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/resize_with_bbox_ir.cc @@ -37,6 +37,13 @@ std::string ResizeWithBBoxOperation::Name() const { return kResizeWithBBoxOperat Status ResizeWithBBoxOperation::ValidateParams() { RETURN_IF_NOT_OK(ValidateVectorSize("ResizeWithBBox", size_)); + // interpolation + if (interpolation_ != InterpolationMode::kLinear && interpolation_ != InterpolationMode::kNearestNeighbour && + interpolation_ != InterpolationMode::kCubic && interpolation_ != InterpolationMode::kArea) { + std::string err_msg = "ResizeWithBBox: Invalid InterpolationMode, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/liteapi/include/datasets.h b/mindspore/ccsrc/minddata/dataset/liteapi/include/datasets.h index 2823bd85453..8905ebf4628 100644 --- a/mindspore/ccsrc/minddata/dataset/liteapi/include/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/liteapi/include/datasets.h @@ -519,12 +519,12 @@ inline std::shared_ptr Album(const std::string &dataset_dir, const class MnistDataset : public Dataset { public: - explicit MnistDataset(const std::vector &dataset_dir, const std::vector &usage, - const std::shared_ptr &sampler, const std::shared_ptr &cache); - explicit MnistDataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, - const std::shared_ptr &cache); - explicit MnistDataset(const std::vector &dataset_dir, const std::vector &usage, - const std::reference_wrapper sampler, const std::shared_ptr &cache); + MnistDataset(const std::vector &dataset_dir, const std::vector &usage, + const std::shared_ptr &sampler, const std::shared_ptr &cache); + MnistDataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, + const std::shared_ptr &cache); + MnistDataset(const std::vector &dataset_dir, const std::vector &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache); ~MnistDataset() = default; }; diff --git a/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.cc b/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.cc index 96bdf55ae2e..eedc8233228 100644 --- a/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.cc @@ -65,7 +65,16 @@ BasicTokenizerOperation::BasicTokenizerOperation(bool lower_case, bool keep_whit preserve_unused_token_(preserve_unused_token), with_offsets_(with_offsets) {} -Status BasicTokenizerOperation::ValidateParams() { return Status::OK(); } +Status BasicTokenizerOperation::ValidateParams() { + if (normalize_form_ != NormalizeForm::kNone && normalize_form_ != NormalizeForm::kNfc && + normalize_form_ != NormalizeForm::kNfkc && normalize_form_ != NormalizeForm::kNfd && + normalize_form_ != NormalizeForm::kNfkd) { + std::string err_msg = "BasicTokenizer: Invalid NormalizeForm, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + return Status::OK(); +} std::shared_ptr BasicTokenizerOperation::Build() { std::shared_ptr tensor_op = std::make_shared( @@ -98,6 +107,14 @@ Status BertTokenizerOperation::ValidateParams() { RETURN_STATUS_SYNTAX_ERROR(err_msg); } + if (normalize_form_ != NormalizeForm::kNone && normalize_form_ != NormalizeForm::kNfc && + normalize_form_ != NormalizeForm::kNfkc && normalize_form_ != NormalizeForm::kNfd && + normalize_form_ != NormalizeForm::kNfkd) { + std::string err_msg = "BertTokenizer: Invalid NormalizeForm, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + if (max_bytes_per_token_ < 0) { std::string err_msg = "BertTokenizer : The parameter max_bytes_per_token must be greater than or equal to 0: " + std::to_string(max_bytes_per_token_); @@ -142,6 +159,12 @@ Status JiebaTokenizerOperation::ValidateParams() { RETURN_STATUS_SYNTAX_ERROR(err_msg); } + if (mode_ != JiebaMode::kMix && mode_ != JiebaMode::kMp && mode_ != JiebaMode::kHmm) { + std::string err_msg = "JiebaTokenizer: Invalid JiebaMode, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + RETURN_IF_NOT_OK(ValidateTokenizerDirParam("JiebaTokenizer", hmm_path_)); RETURN_IF_NOT_OK(ValidateTokenizerDirParam("JiebaTokenizer", mp_path_)); return Status::OK(); @@ -264,7 +287,16 @@ std::shared_ptr NgramOperation::Build() { // NormalizeUTF8Operation NormalizeUTF8Operation::NormalizeUTF8Operation(NormalizeForm normalize_form) : normalize_form_(normalize_form) {} -Status NormalizeUTF8Operation::ValidateParams() { return Status::OK(); } +Status NormalizeUTF8Operation::ValidateParams() { + if (normalize_form_ != NormalizeForm::kNone && normalize_form_ != NormalizeForm::kNfc && + normalize_form_ != NormalizeForm::kNfkc && normalize_form_ != NormalizeForm::kNfd && + normalize_form_ != NormalizeForm::kNfkd) { + std::string err_msg = "NormalizeUTF8: Invalid NormalizeForm, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + return Status::OK(); +} std::shared_ptr NormalizeUTF8Operation::Build() { std::shared_ptr tensor_op = std::make_shared(normalize_form_); @@ -308,6 +340,11 @@ SentencePieceTokenizerOperation::SentencePieceTokenizerOperation(const std::stri : vocab_(nullptr), vocab_path_(vocab_path), load_type_(SPieceTokenizerLoadType::kFile), out_type_(out_type) {} Status SentencePieceTokenizerOperation::ValidateParams() { + if (out_type_ != SPieceTokenizerOutType::kString && out_type_ != SPieceTokenizerOutType::kInt) { + std::string err_msg = "SentencePieceTokenizer: Invalid SPieceTokenizerOutType, check input value of enum."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } if (load_type_ == SPieceTokenizerLoadType::kModel) { if (vocab_ == nullptr) { std::string err_msg = "SentencePieceTokenizer: vocab object type is incorrect or null."; diff --git a/mindspore/ccsrc/minddata/dataset/text/sentence_piece_vocab.cc b/mindspore/ccsrc/minddata/dataset/text/sentence_piece_vocab.cc index 7c2634620ee..0e75c9b9a3f 100644 --- a/mindspore/ccsrc/minddata/dataset/text/sentence_piece_vocab.cc +++ b/mindspore/ccsrc/minddata/dataset/text/sentence_piece_vocab.cc @@ -34,6 +34,9 @@ Status SentencePieceVocab::BuildFromFile(const std::vector &path_li const float character_coverage, const SentencePieceModel model_type, const std::unordered_map ¶ms, std::shared_ptr *vocab) { + if (vocab == nullptr) { + RETURN_STATUS_UNEXPECTED("SentencePieceVocab::BuildFromFile: input vocab can not be null"); + } std::unordered_map unorder_map; // the input of sentence is comma separated string @@ -85,6 +88,9 @@ Status SentencePieceVocab::BuildFromFile(const std::vector &path_li Status SentencePieceVocab::SaveModel(const std::shared_ptr *vocab, std::string path, std::string filename) { + if (vocab == nullptr) { + RETURN_STATUS_UNEXPECTED("SentencePieceVocab::SaveModel: input vocab can not be null"); + } auto realpath = FileUtils::GetRealPath(path.data()); if (!realpath.has_value()) { RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + path); diff --git a/mindspore/ccsrc/minddata/dataset/text/vocab.cc b/mindspore/ccsrc/minddata/dataset/text/vocab.cc index 49db608c38b..aaa84465b55 100644 --- a/mindspore/ccsrc/minddata/dataset/text/vocab.cc +++ b/mindspore/ccsrc/minddata/dataset/text/vocab.cc @@ -41,6 +41,9 @@ WordIdType Vocab::Lookup(const WordType &word) const { #ifdef ENABLE_PYTHON Status Vocab::BuildFromPyList(const py::list &words, const py::list &special_tokens, bool prepend_special, std::shared_ptr *vocab) { + if (vocab == nullptr) { + RETURN_STATUS_UNEXPECTED("Vocab::BuildFromPyList: input vocab can not be null"); + } // check of duplication on both words and special_tokens will be performed in python // special_tokens and words both need to be unique, and shouldn't overlap std::unordered_map word2id; @@ -62,6 +65,9 @@ Status Vocab::BuildFromPyList(const py::list &words, const py::list &special_tok } Status Vocab::BuildFromPyDict(const py::dict &words, std::shared_ptr *vocab) { + if (vocab == nullptr) { + RETURN_STATUS_UNEXPECTED("Vocab::BuildFromPyDict: input vocab can not be null"); + } std::unordered_map word2id; for (auto p : words) { word2id[py::str(p.first)] = py::reinterpret_borrow(p.second); @@ -79,6 +85,9 @@ void Vocab::append_word(const std::string &word) { Status Vocab::BuildFromUnorderedMap(const std::unordered_map &words, std::shared_ptr *vocab) { + if (vocab == nullptr) { + RETURN_STATUS_UNEXPECTED("Vocab::BuildFromUnorderedMap: input vocab can not be null"); + } // Validate parameters and build map std::unordered_map word2id; for (auto p : words) { @@ -93,6 +102,9 @@ Status Vocab::BuildFromUnorderedMap(const std::unordered_map &words, const std::vector &special_tokens, bool prepend_special, std::shared_ptr *vocab) { + if (vocab == nullptr) { + RETURN_STATUS_UNEXPECTED("Vocab::BuildFromVector: input vocab can not be null"); + } std::unordered_map word2id; // if special is added in front, normal words id will start from number of special tokens @@ -123,18 +135,16 @@ Status Vocab::BuildFromVector(const std::vector &words, const std::vec Status Vocab::BuildFromFileCpp(const std::string &path, const std::string &delimiter, int32_t vocab_size, const std::vector &special_tokens, bool prepend_special, std::shared_ptr *vocab) { + if (vocab == nullptr) { + RETURN_STATUS_UNEXPECTED("Vocab::BuildFromFileCpp: input vocab can not be null"); + } // Validate parameters auto realpath = FileUtils::GetRealPath(path.data()); - if (!realpath.has_value()) { - RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + path); - } + CHECK_FAIL_RETURN_UNEXPECTED(realpath.has_value(), "Get real path failed, path=" + path); - if (vocab_size < 0 && vocab_size != -1) { - RETURN_STATUS_UNEXPECTED( - "from_file: " - "vocab_size should be either -1 or positive integer, but got " + - std::to_string(vocab_size)); - } + CHECK_FAIL_RETURN_UNEXPECTED( + !(vocab_size < 0 && vocab_size != -1), + "from_file: vocab_size should be either -1 or positive integer, but got: " + std::to_string(vocab_size)); std::string duplicate_sp; for (const WordType &sp : special_tokens) { @@ -144,12 +154,8 @@ Status Vocab::BuildFromFileCpp(const std::string &path, const std::string &delim } } } - if (!duplicate_sp.empty()) { - RETURN_STATUS_UNEXPECTED( - "from_file: " - "special_tokens contains duplicate word: " + - duplicate_sp); - } + CHECK_FAIL_RETURN_UNEXPECTED(duplicate_sp.empty(), + "from_file: special_tokens contains duplicate word: " + duplicate_sp); std::unordered_set specials; // used to check that words in file don't contain any special token that already exists @@ -160,24 +166,19 @@ Status Vocab::BuildFromFileCpp(const std::string &path, const std::string &delim std::unordered_map word2id; std::fstream handle(realpath.value(), std::ios::in); - if (!handle.good() || !handle.is_open()) { - RETURN_STATUS_UNEXPECTED("from_file: fail to open: " + realpath.value()); - } + CHECK_FAIL_RETURN_UNEXPECTED(handle.good() && handle.is_open(), "from_file: fail to open: " + realpath.value()); + std::string word; while (std::getline(handle, word)) { if (!delimiter.empty()) { // if delimiter is not found, find_first_of would return std::string::npos which is -1 word = word.substr(0, word.find_first_of(delimiter)); } - if (word2id.find(word) != word2id.end()) { - RETURN_STATUS_UNEXPECTED("from_file: word_list contains duplicate word:" + word + "."); - } - if (specials.find(word) != specials.end()) { - RETURN_STATUS_UNEXPECTED( - "from_file: " - "special_tokens and word_list contain duplicate word: " + - word); - } + CHECK_FAIL_RETURN_UNEXPECTED(word2id.find(word) == word2id.end(), + "from_file: word_list contains duplicate word:" + word); + CHECK_FAIL_RETURN_UNEXPECTED(specials.find(word) == specials.end(), + "from_file: special_tokens and word_list contain duplicate word:" + word); + word2id[word] = word_id++; // break if enough row is read, if vocab_size is smaller than 0 if (word2id.size() == vocab_size) break; @@ -195,6 +196,9 @@ Status Vocab::BuildFromFileCpp(const std::string &path, const std::string &delim Status Vocab::BuildFromFile(const std::string &path, const std::string &delimiter, int32_t vocab_size, const py::list &special_tokens, bool prepend_special, std::shared_ptr *vocab) { + if (vocab == nullptr) { + RETURN_STATUS_UNEXPECTED("Vocab::BuildFromFile: input vocab can not be null"); + } // python validator checks special_tokens doesn't contain any duplicate words std::unordered_set specials; // used to check that words in file don't contain any special token that already exists diff --git a/mindspore/ccsrc/minddata/dataset/util/btree.h b/mindspore/ccsrc/minddata/dataset/util/btree.h index 9e99fbcd5a3..b097266817a 100644 --- a/mindspore/ccsrc/minddata/dataset/util/btree.h +++ b/mindspore/ccsrc/minddata/dataset/util/btree.h @@ -316,7 +316,13 @@ class BPlusTree { void Init() { typename LeafNode::alloc_type alloc(alloc_); - auto *p = alloc.allocate(1); + LeafNode *p; + try { + p = alloc.allocate(1); + } catch (std::bad_alloc &e) { + p = nullptr; + return; + } root_ = new (p) LeafNode(alloc_); all_.Prepend(p); leaf_nodes_.Append(p); diff --git a/mindspore/ccsrc/minddata/mindrecord/include/common/shard_utils.h b/mindspore/ccsrc/minddata/mindrecord/include/common/shard_utils.h index 01e81b6be91..1a83d62ef74 100644 --- a/mindspore/ccsrc/minddata/mindrecord/include/common/shard_utils.h +++ b/mindspore/ccsrc/minddata/mindrecord/include/common/shard_utils.h @@ -181,7 +181,7 @@ bool CheckIsValidUtf8(const std::string &str); /// \brief judge if a path is legal file /// \param path file path -/// \return parent path +/// \return Whether the path is legal or not bool IsLegalFile(const std::string &path); enum DiskSizeType { kTotalSize = 0, kFreeSize }; diff --git a/mindspore/dataset/engine/validators.py b/mindspore/dataset/engine/validators.py index 578bb7c8c4d..ec0c2b2d7aa 100644 --- a/mindspore/dataset/engine/validators.py +++ b/mindspore/dataset/engine/validators.py @@ -550,6 +550,9 @@ def check_bucket_batch_by_length(method): if element_length_function is None and len(column_names) != 1: raise ValueError("If element_length_function is not specified, exactly one column name should be passed.") + if element_length_function is not None and not callable(element_length_function): + raise TypeError("element_length_function object is not callable.") + # check bucket_boundaries: must be list of int, positive and strictly increasing if not bucket_boundaries: raise ValueError("bucket_boundaries cannot be empty.")