xxq250
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update cpp api & doc

This commit is contained in:
YangLuo 2021-06-15 19:11:15 +08:00 committed by luoyang
parent 4982424ccf
commit 4e9f6d54f7
18 changed files with 653 additions and 595 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
mindspore/ccsrc/minddata/dataset/api/execute.cc
View File

@ -55,9 +55,9 @@ struct Execute::ExtraInfo {
};
// FIXME - Temporarily overload Execute to support both TensorOperation and TensorTransform
Execute::Execute(std::shared_ptr<TensorOperation> op, MapTargetDevice deviceType, uint32_t device_id) {
Execute::Execute(std::shared_ptr<TensorOperation> op, MapTargetDevice device_type, uint32_t device_id) {
ops_.emplace_back(std::move(op));
device_type_ = deviceType;
device_type_ = device_type;
info_ = std::make_shared<ExtraInfo>();
#ifdef ENABLE_ACL
if (device_type_ == MapTargetDevice::kAscend310) {
@ -65,72 +65,72 @@ Execute::Execute(std::shared_ptr<TensorOperation> op, MapTargetDevice deviceType
Status rc = device_resource_->InitResource(device_id);
if (!rc.IsOk()) {
device_resource_ = nullptr;
MS_LOG(ERROR) << "Initialize Ascend310 resource fail";
MS_LOG(ERROR) << "Initialize Ascend310 resource fail.";
}
}
#endif
}
Execute::Execute(std::shared_ptr<TensorTransform> op, MapTargetDevice deviceType, uint32_t device_id) {
Execute::Execute(std::shared_ptr<TensorTransform> op, MapTargetDevice device_type, uint32_t device_id) {
// Initialize the op and other context
transforms_.emplace_back(op);
info_ = std::make_shared<ExtraInfo>();
device_type_ = deviceType;
device_type_ = device_type;
#ifdef ENABLE_ACL
if (device_type_ == MapTargetDevice::kAscend310) {
device_resource_ = std::make_shared<AscendResource>();
Status rc = device_resource_->InitResource(device_id);
if (!rc.IsOk()) {
device_resource_ = nullptr;
MS_LOG(ERROR) << "Initialize Ascend310 resource fail";
MS_LOG(ERROR) << "Initialize Ascend310 resource fail.";
}
}
#endif
}
Execute::Execute(std::reference_wrapper<TensorTransform> op, MapTargetDevice deviceType, uint32_t device_id) {
Execute::Execute(std::reference_wrapper<TensorTransform> op, MapTargetDevice device_type, uint32_t device_id) {
// Initialize the transforms_ and other context
std::shared_ptr<TensorOperation> operation = op.get().Parse();
ops_.emplace_back(std::move(operation));
info_ = std::make_shared<ExtraInfo>();
info_->init_with_shared_ptr_ = false;
device_type_ = deviceType;
device_type_ = device_type;
#ifdef ENABLE_ACL
if (device_type_ == MapTargetDevice::kAscend310) {
device_resource_ = std::make_shared<AscendResource>();
Status rc = device_resource_->InitResource(device_id);
if (!rc.IsOk()) {
device_resource_ = nullptr;
MS_LOG(ERROR) << "Initialize Ascend310 resource fail";
MS_LOG(ERROR) << "Initialize Ascend310 resource fail.";
}
}
#endif
}
// Execute function for the example case: auto decode(new vision::Decode());
Execute::Execute(TensorTransform *op, MapTargetDevice deviceType, uint32_t device_id) {
Execute::Execute(TensorTransform *op, MapTargetDevice device_type, uint32_t device_id) {
// Initialize the transforms_ and other context
std::shared_ptr<TensorTransform> smart_ptr_op(op);
transforms_.emplace_back(smart_ptr_op);
info_ = std::make_shared<ExtraInfo>();
device_type_ = deviceType;
device_type_ = device_type;
#ifdef ENABLE_ACL
if (device_type_ == MapTargetDevice::kAscend310) {
device_resource_ = std::make_shared<AscendResource>();
Status rc = device_resource_->InitResource(device_id);
if (!rc.IsOk()) {
device_resource_ = nullptr;
MS_LOG(ERROR) << "Initialize Ascend310 resource fail";
MS_LOG(ERROR) << "Initialize Ascend310 resource fail.";
}
}
#endif
}
Execute::Execute(std::vector<std::shared_ptr<TensorOperation>> ops, MapTargetDevice deviceType, uint32_t device_id)
: ops_(std::move(ops)), device_type_(deviceType) {
Execute::Execute(std::vector<std::shared_ptr<TensorOperation>> ops, MapTargetDevice device_type, uint32_t device_id)
: ops_(std::move(ops)), device_type_(device_type) {
info_ = std::make_shared<ExtraInfo>();
#ifdef ENABLE_ACL
if (device_type_ == MapTargetDevice::kAscend310) {
@ -138,46 +138,46 @@ Execute::Execute(std::vector<std::shared_ptr<TensorOperation>> ops, MapTargetDev
Status rc = device_resource_->InitResource(device_id);
if (!rc.IsOk()) {
device_resource_ = nullptr;
MS_LOG(ERROR) << "Initialize Ascend310 resource fail";
MS_LOG(ERROR) << "Initialize Ascend310 resource fail.";
}
}
#endif
}
Execute::Execute(std::vector<std::shared_ptr<TensorTransform>> ops, MapTargetDevice deviceType, uint32_t device_id) {
Execute::Execute(std::vector<std::shared_ptr<TensorTransform>> ops, MapTargetDevice device_type, uint32_t device_id) {
// Initialize the transforms_ and other context
transforms_ = ops;
info_ = std::make_shared<ExtraInfo>();
device_type_ = deviceType;
device_type_ = device_type;
#ifdef ENABLE_ACL
if (device_type_ == MapTargetDevice::kAscend310) {
device_resource_ = std::make_shared<AscendResource>();
Status rc = device_resource_->InitResource(device_id);
if (!rc.IsOk()) {
device_resource_ = nullptr;
MS_LOG(ERROR) << "Initialize Ascend310 resource fail";
MS_LOG(ERROR) << "Initialize Ascend310 resource fail.";
}
}
#endif
}
Execute::Execute(const std::vector<std::reference_wrapper<TensorTransform>> ops, MapTargetDevice deviceType,
Execute::Execute(const std::vector<std::reference_wrapper<TensorTransform>> ops, MapTargetDevice device_type,
uint32_t device_id) {
// Initialize the transforms_ and other context
if (deviceType == MapTargetDevice::kCpu) {
if (device_type == MapTargetDevice::kCpu) {
(void)std::transform(
ops.begin(), ops.end(), std::back_inserter(ops_),
[](TensorTransform &operation) -> std::shared_ptr<TensorOperation> { return operation.Parse(); });
} else {
for (auto &op : ops) {
ops_.emplace_back(op.get().Parse(deviceType));
ops_.emplace_back(op.get().Parse(device_type));
}
}
info_ = std::make_shared<ExtraInfo>();
info_->init_with_shared_ptr_ = false;
device_type_ = deviceType;
device_type_ = device_type;
#ifdef ENABLE_ACL
if (device_type_ == MapTargetDevice::kAscend310) {
device_resource_ = std::make_shared<AscendResource>();
@ -191,7 +191,7 @@ Execute::Execute(const std::vector<std::reference_wrapper<TensorTransform>> ops,
}
// Execute function for the example vector case: auto decode(new vision::Decode());
Execute::Execute(const std::vector<TensorTransform *> &ops, MapTargetDevice deviceType, uint32_t device_id) {
Execute::Execute(const std::vector<TensorTransform *> &ops, MapTargetDevice device_type, uint32_t device_id) {
// Initialize the transforms_ and other context
for (auto &op : ops) {
std::shared_ptr<TensorTransform> smart_ptr_op(op);
@ -199,7 +199,7 @@ Execute::Execute(const std::vector<TensorTransform *> &ops, MapTargetDevice devi
}
info_ = std::make_shared<ExtraInfo>();
device_type_ = deviceType;
device_type_ = device_type;
#ifdef ENABLE_ACL
if (device_type_ == MapTargetDevice::kAscend310) {
device_resource_ = std::make_shared<AscendResource>();
@ -226,15 +226,15 @@ Execute::~Execute() {
Status Execute::operator()(const mindspore::MSTensor &input, mindspore::MSTensor *output) {
// Validate input tensor
CHECK_FAIL_RETURN_UNEXPECTED(input.DataSize() > 0, "Input Tensor has no data");
CHECK_FAIL_RETURN_UNEXPECTED(validate_device_(), "Device Type should be 'Ascend310' or 'CPU'");
CHECK_FAIL_RETURN_UNEXPECTED(input.DataSize() > 0, "Input Tensor has no data.");
CHECK_FAIL_RETURN_UNEXPECTED(ValidateDevice(), "Device Type should be 'Ascend310' or 'CPU'.");
// Parse TensorTransform transforms_ into TensorOperation ops_
if (info_->init_with_shared_ptr_) {
RETURN_IF_NOT_OK(ParseTransforms_());
RETURN_IF_NOT_OK(ParseTransforms());
info_->init_with_shared_ptr_ = false;
}
CHECK_FAIL_RETURN_UNEXPECTED(!ops_.empty(), "Input TensorOperation should be provided");
CHECK_FAIL_RETURN_UNEXPECTED(!ops_.empty(), "Input TensorOperation should be provided.");
// Validate and build runtime ops
std::vector<std::shared_ptr<TensorOp>> transforms; // record the transformations
@ -244,10 +244,11 @@ Status Execute::operator()(const mindspore::MSTensor &input, mindspore::MSTensor
for (int32_t i = 0; i < ops_.size(); i++) {
if (ops_[i] == nullptr) {
MS_LOG(ERROR) << "Input TensorOperation["
<< std::to_string(i) + "] is unsupported on your input device:" << env_list.at(device_type_);
std::string err_msg = "Input TensorOperation[" + std::to_string(i) +
"] is unsupported on your input device:" + env_list.at(device_type_);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_UNEXPECTED(err_msg);
}
CHECK_FAIL_RETURN_UNEXPECTED(ops_[i] != nullptr, "Input TensorOperation[" + std::to_string(i) + "] is null");
RETURN_IF_NOT_OK(ops_[i]->ValidateParams());
transforms.emplace_back(ops_[i]->Build());
}
@ -285,7 +286,7 @@ Status Execute::operator()(const mindspore::MSTensor &input, mindspore::MSTensor
*output = mindspore::MSTensor(std::make_shared<DETensor>(de_tensor));
} else { // Ascend310 case, where we must set Ascend resource on each operators
#ifdef ENABLE_ACL
CHECK_FAIL_RETURN_UNEXPECTED(device_resource_, "Device resource is nullptr which is illegal under case Ascend310");
CHECK_FAIL_RETURN_UNEXPECTED(device_resource_, "Device resource is nullptr which is illegal under case Ascend310.");
// Sink data from host into device
std::shared_ptr<mindspore::dataset::DeviceTensor> device_input;
RETURN_IF_NOT_OK(device_resource_->Sink(input, &device_input));
@ -300,7 +301,7 @@ Status Execute::operator()(const mindspore::MSTensor &input, mindspore::MSTensor
// For next transform
device_input = std::move(device_output);
}
CHECK_FAIL_RETURN_UNEXPECTED(device_input->HasDeviceData(), "Apply transform failed, output tensor has no data");
CHECK_FAIL_RETURN_UNEXPECTED(device_input->HasDeviceData(), "Apply transform failed, output tensor has no data.");
*output = mindspore::MSTensor(std::make_shared<DETensor>(device_input, true));
#endif
@ -310,18 +311,18 @@ Status Execute::operator()(const mindspore::MSTensor &input, mindspore::MSTensor
Status Execute::operator()(const std::vector<MSTensor> &input_tensor_list, std::vector<MSTensor> *output_tensor_list) {
// Validate input tensor
CHECK_FAIL_RETURN_UNEXPECTED(!input_tensor_list.empty(), "Input Tensor is not valid");
CHECK_FAIL_RETURN_UNEXPECTED(!input_tensor_list.empty(), "Input Tensor is not valid.");
for (auto &tensor : input_tensor_list) {
CHECK_FAIL_RETURN_UNEXPECTED(tensor.DataSize() > 0, "Input Tensor has no data");
CHECK_FAIL_RETURN_UNEXPECTED(tensor.DataSize() > 0, "Input Tensor has no data.");
}
CHECK_FAIL_RETURN_UNEXPECTED(validate_device_(), "Device Type should be 'Ascend310' or 'CPU'");
CHECK_FAIL_RETURN_UNEXPECTED(ValidateDevice(), "Device Type should be 'Ascend310' or 'CPU'.");
// Parse TensorTransform transforms_ into TensorOperation ops_
if (info_->init_with_shared_ptr_) {
RETURN_IF_NOT_OK(ParseTransforms_());
RETURN_IF_NOT_OK(ParseTransforms());
info_->init_with_shared_ptr_ = false;
}
CHECK_FAIL_RETURN_UNEXPECTED(!ops_.empty(), "Input TensorOperation should be provided");
CHECK_FAIL_RETURN_UNEXPECTED(!ops_.empty(), "Input TensorOperation should be provided.");
std::map<MapTargetDevice, std::string> env_list = {
{MapTargetDevice::kCpu, "kCpu"}, {MapTargetDevice::kGpu, "kGpu"}, {MapTargetDevice::kAscend310, "kAscend310"}};
@ -330,10 +331,11 @@ Status Execute::operator()(const std::vector<MSTensor> &input_tensor_list, std::
std::vector<std::shared_ptr<TensorOp>> transforms;
for (int32_t i = 0; i < ops_.size(); i++) {
if (ops_[i] == nullptr) {
MS_LOG(ERROR) << "Input TensorOperation["
<< std::to_string(i) + "] is unsupported on your input device:" << env_list.at(device_type_);
std::string err_msg = "Input TensorOperation[" + std::to_string(i) +
"] is unsupported on your input device:" + env_list.at(device_type_);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_UNEXPECTED(err_msg);
}
CHECK_FAIL_RETURN_UNEXPECTED(ops_[i] != nullptr, "Input TensorOperation[" + std::to_string(i) + "] is null");
RETURN_IF_NOT_OK(ops_[i]->ValidateParams());
transforms.emplace_back(ops_[i]->Build());
}
@ -369,10 +371,10 @@ Status Execute::operator()(const std::vector<MSTensor> &input_tensor_list, std::
output_tensor_list->emplace_back(ms_tensor);
++idx;
}
CHECK_FAIL_RETURN_UNEXPECTED(!output_tensor_list->empty(), "Output Tensor is not valid");
CHECK_FAIL_RETURN_UNEXPECTED(!output_tensor_list->empty(), "Output Tensor is not valid.");
} else { // Case Ascend310
#ifdef ENABLE_ACL
CHECK_FAIL_RETURN_UNEXPECTED(device_resource_, "Device resource is nullptr which is illegal under case Ascend310");
CHECK_FAIL_RETURN_UNEXPECTED(device_resource_, "Device resource is nullptr which is illegal under case Ascend310.");
for (auto &input_tensor : input_tensor_list) {
// Sink each data from host into device
std::shared_ptr<dataset::DeviceTensor> device_input;
@ -398,7 +400,7 @@ Status Execute::operator()(const std::vector<MSTensor> &input_tensor_list, std::
// Release the data on the device because we have copied one piece onto host
RETURN_IF_NOT_OK(device_resource_->DeviceDataRelease());
}
CHECK_FAIL_RETURN_UNEXPECTED(!output_tensor_list->empty(), "Output Tensor vector is empty");
CHECK_FAIL_RETURN_UNEXPECTED(!output_tensor_list->empty(), "Output Tensor vector is empty.");
#endif
}
return Status::OK();
@ -411,7 +413,7 @@ std::vector<uint32_t> AippSizeFilter(const std::vector<uint32_t> &resize_para, c
if ((resize_para.size() == 0 || resize_para.size() == 1) && crop_para.size() == 0) {
aipp_size = {0, 0};
MS_LOG(WARNING) << "Dynamic input shape is not supported, incomplete aipp config file will be generated. Please "
"checkout your TensorTransform input, both src_image_size_h and src_image_size will be 0";
"checkout your TensorTransform input, both src_image_size_h and src_image_size will be 0.";
return aipp_size;
}
@ -456,7 +458,7 @@ std::vector<float> AippStdFilter(const std::vector<uint32_t> &normalize_para) {
std::transform(normalize_para.begin() + 3, normalize_para.end(), std::back_inserter(aipp_std),
[](uint32_t i) { return 10000 / static_cast<float>(i); });
} else { // If 0 occurs in std vector
MS_LOG(WARNING) << "Detect 0 in std vector, please verify your input";
MS_LOG(WARNING) << "Detect 0 in std vector, please verify your input.";
aipp_std = {1.0, 1.0, 1.0};
}
} else {
@ -514,7 +516,7 @@ std::string Execute::AippCfgGenerator() {
std::string config_location = "./aipp.cfg";
#ifdef ENABLE_ACL
if (info_->init_with_shared_ptr_) {
ParseTransforms_();
ParseTransforms();
info_->init_with_shared_ptr_ = false;
}
std::vector<uint32_t> paras; // Record the parameters value of each Ascend operators
@ -522,7 +524,7 @@ std::string Execute::AippCfgGenerator() {
// Validate operator ir
json ir_info;
if (ops_[i] == nullptr) {
MS_LOG(ERROR) << "Input TensorOperation[" + std::to_string(i) + "] is null";
MS_LOG(ERROR) << "Input TensorOperation[" + std::to_string(i) + "] is null.";
return "";
}
@ -541,8 +543,8 @@ std::string Execute::AippCfgGenerator() {
outfile.open(config_location, std::ofstream::out);
if (!outfile.is_open()) {
MS_LOG(ERROR) << "Fail to open Aipp config file, please verify your system config(including authority)"
<< "We will return empty string which represent the location of Aipp config file in this case";
MS_LOG(ERROR) << "Fail to open Aipp config file, please verify your system config(including authority)."
<< "We will return empty string which represent the location of Aipp config file in this case.";
std::string except = "";
return except;
}
@ -611,10 +613,10 @@ std::string Execute::AippCfgGenerator() {
bool IsEmptyPtr(std::shared_ptr<TensorTransform> api_ptr) { return api_ptr == nullptr; }
Status Execute::ParseTransforms_() {
Status Execute::ParseTransforms() {
auto iter = std::find_if(transforms_.begin(), transforms_.end(), IsEmptyPtr);
if (iter != transforms_.end()) {
std::string err_msg = "Your input TensorTransforms contain at least one nullptr, please check your input";
std::string err_msg = "Your input TensorTransforms contain at least one nullptr, please check your input.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_UNEXPECTED(err_msg);
}
@ -633,10 +635,10 @@ Status Execute::ParseTransforms_() {
return Status::OK();
}
Status Execute::validate_device_() {
Status Execute::ValidateDevice() {
if (device_type_ != MapTargetDevice::kCpu && device_type_ != MapTargetDevice::kAscend310 &&
device_type_ != MapTargetDevice::kGpu) {
std::string err_msg = "Your input device is not supported. (Option: CPU or GPU or Ascend310)";
std::string err_msg = "Your input device is not supported. (Option: CPU or GPU or Ascend310).";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_UNEXPECTED(err_msg);
}
@ -644,7 +646,7 @@ Status Execute::validate_device_() {
}
Status Execute::DeviceMemoryRelease() {
CHECK_FAIL_RETURN_UNEXPECTED(device_resource_, "Device resource is nullptr which is illegal under case Ascend310");
CHECK_FAIL_RETURN_UNEXPECTED(device_resource_, "Device resource is nullptr which is illegal under case Ascend310.");
Status rc = device_resource_->DeviceDataRelease();
if (rc.IsError()) {
std::string err_msg = "Error in device data release";

53
mindspore/ccsrc/minddata/dataset/include/dataset/config.h
View File

@ -28,55 +28,56 @@ namespace dataset {
// Config operations for setting and getting the configuration.
namespace config {
/// \brief Function to set the seed to be used in any random generator. This is used to produce deterministic results.
/// \param[in] seed the default seed to use.
/// \brief A function to set the seed to be used in any random generator. This is used to produce deterministic results.
/// \param[in] seed The default seed to be used.
bool set_seed(int32_t seed);
/// \brief Function to get the seed.
/// \return the seed set in the configuration.
/// \brief A function to get the seed.
/// \return The seed set in the configuration.
uint32_t get_seed();
/// \brief Function to set the number of rows to be prefetched.
/// \param[in] prefetch_size total number of rows to be prefetched.
/// \brief A function to set the number of rows to be prefetched.
/// \param[in] prefetch_size Total number of rows to be prefetched.
bool set_prefetch_size(int32_t prefetch_size);
/// \brief Function to get the prefetch size in number of rows.
/// \return total number of rows to be prefetched.
/// \brief A function to get the prefetch size in number of rows.
/// \return Total number of rows to be prefetched.
int32_t get_prefetch_size();
/// \brief Function to set the default number of parallel workers.
/// \param[in] num_parallel_workers number of parallel workers to be used as a default for each operation.
/// \brief A function to set the default number of parallel workers.
/// \param[in] num_parallel_workers Number of parallel workers to be used as the default for each operation.
bool set_num_parallel_workers(int32_t num_parallel_workers);
/// \brief Function to get the default number of parallel workers.
/// \return number of parallel workers to be used as a default for each operation.
/// \brief A function to get the default number of parallel workers.
/// \return Number of parallel workers to be used as the default for each operation.
int32_t get_num_parallel_workers();
/// \brief Function to set the default interval (in milliseconds) for monitor sampling.
/// \param[in] interval interval (in milliseconds) to be used for performance monitor sampling.
/// \brief A function to set the default interval (in milliseconds) for monitor sampling.
/// \param[in] interval Interval (in milliseconds) to be used for performance monitor sampling.
bool set_monitor_sampling_interval(int32_t interval);
/// \brief Function to get the default interval of performance monitor sampling.
/// \return interval (in milliseconds) for performance monitor sampling.
/// \brief A function to get the default interval of performance monitor sampling.
/// \return Interval (in milliseconds) for performance monitor sampling.
int32_t get_monitor_sampling_interval();
/// \brief Function to set the default timeout (in seconds) for DSWaitedCallback. In case of a deadlock, the wait
/// \brief A function to set the default timeout (in seconds) for DSWaitedCallback. In case of a deadlock, the wait
/// function will exit after the timeout period.
/// \param[in] timeout timeout (in seconds) to be used to end the wait in DSWaitedCallback in case of a deadlock.
/// \param[in] timeout Timeout (in seconds) to be used to end the wait in DSWaitedCallback in case of a deadlock.
bool set_callback_timeout(int32_t timeout);
/// \brief Function to get the default timeout for DSWaitedCallback. In case of a deadback, the wait function will exit
/// after the timeout period.
/// \return the duration in seconds.
/// \brief A function to get the default timeout for DSWaitedCallback. In case of a deadback, the wait function
/// will exit after the timeout period.
/// \return The duration in seconds.
int32_t get_callback_timeout();
/// \brief Function to load configuration from a file.
/// \param[in] file path of the configuration file to be loaded.
/// \note This API exists because std::string will constrained by ABI compile option while char don't.
/// \brief A function to load the configuration from a file.
/// \param[in] file Path of the configuration file to be loaded.
/// \note The reason for using this API is that std::string will be constrained by the
/// compiler option '_GLIBCXX_USE_CXX11_ABI' while char is free of this restriction.
bool load(const std::vector<char> &file);
/// \brief Function to load configuration from a file.
/// \param[in] file path of the configuration file to be loaded.
/// \brief A function to load the configuration from a file.
/// \param[in] file Path of the configuration file to be loaded.
inline bool load(std::string file) { return load(StringToChar(file)); }
} // namespace config

10
mindspore/ccsrc/minddata/dataset/include/dataset/constants.h
View File

@ -51,10 +51,10 @@ enum class ShuffleMode {
/// \brief The method of padding.
enum class BorderType {
kConstant = 0, ///< Fills the border with constant values.
kEdge = 1, ///< Fills the border with the last value on the edge.
kReflect = 2, ///< Reflects the values on the edge omitting the last value of edge.
kSymmetric = 3 ///< Reflects the values on the edge repeating the last value of edge.
kConstant = 0, ///< Fill the border with constant values.
kEdge = 1, ///< Fill the border with the last value on the edge.
kReflect = 2, ///< Reflect the values on the edge omitting the last value of edge.
kSymmetric = 3 ///< Reflect the values on the edge repeating the last value of edge.
};
/// \brief Possible options for Image format types in a batch.
@ -108,7 +108,7 @@ enum class SentencePieceModel {
/// \brief Possible options to specify a specific normalize mode.
enum class NormalizeForm {
kNone = 0, ///< Do nothing for input string tensor.
kNone = 0, ///< Keep the input string tensor unchanged.
kNfc, ///< Normalize with Normalization Form C.
kNfkc, ///< Normalize with Normalization Form KC.
kNfd, ///< Normalize with Normalization Form D.

30
mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h
View File

@ -95,55 +95,55 @@ class Dataset : public std::enable_shared_from_this<Dataset> {
/// \brief Destructor
~Dataset() = default;
/// \brief Gets the dataset size
/// \brief Get the dataset size
/// \param[in] estimate This is only supported by some of the ops and it's used to speed up the process of getting
/// dataset size at the expense of accuracy.
/// \return Dataset size. If failed, return -1.
int64_t GetDatasetSize(bool estimate = false);
/// \brief Gets the output type
/// \brief Get the output type
/// \return A vector contains output DataType of dataset. If failed, return an empty vector.
std::vector<mindspore::DataType> GetOutputTypes();
/// \brief Gets the output shape
/// \brief Get the output shape
/// \return A vector contains output TensorShape of dataset. If failed, return an empty vector.
std::vector<std::vector<int64_t>> GetOutputShapes();
/// \brief Gets the batch size
/// \brief Get the batch size
/// \return Batch size configuration of dataset.
int64_t GetBatchSize();
/// \brief Gets the repeat count
/// \brief Get the repeat count
/// \return Repeat count configuration of dataset.
int64_t GetRepeatCount();
/// \brief Gets the number of classes
/// \brief Get the number of classes
/// \return Number of classes of dataset. If failed, return -1.
int64_t GetNumClasses();
/// \brief Gets the column names
/// \brief Get the column names
/// \return A vector contains all column names of dataset. If failed, return an empty vector.
std::vector<std::string> GetColumnNames() { return VectorCharToString(GetColumnNamesCharIF()); }
/// \brief Gets the class indexing
/// \brief Get the class indexing
/// \return A map of ClassIndexing of dataset. If failed, return an empty map.
std::vector<std::pair<std::string, std::vector<int32_t>>> GetClassIndexing() {
return ClassIndexCharToString(GetClassIndexingCharIF());
}
/// \brief Setter function for runtime number of workers.
/// \brief Function to set runtime number of workers.
/// \param[in] num_workers The number of threads in this operator.
/// \return Shared pointer to the original object.
std::shared_ptr<Dataset> SetNumWorkers(int32_t num_workers);
/// \brief Function to create an PullBasedIterator over the Dataset.
/// \brief A Function to create an PullBasedIterator over the Dataset.
/// \param[in] columns List of columns to be used to specify the order of columns.
/// \return Shared pointer to the Iterator.
std::shared_ptr<PullIterator> CreatePullBasedIterator(std::vector<std::vector<char>> columns = {});
/// \brief Function to create an Iterator over the Dataset pipeline.
/// \param[in] columns List of columns to be used to specify the order of columns.
/// \param[in] num_epochs Number of epochs to run through the pipeline, default -1 which means infinite epochs.
/// \param[in] num_epochs Number of epochs to run through the pipeline (default=-1, which means infinite epochs).
/// An empty row is returned at the end of each epoch.
/// \return Shared pointer to the Iterator.
std::shared_ptr<Iterator> CreateIterator(std::vector<std::string> columns = {}, int32_t num_epochs = -1) {
@ -674,7 +674,8 @@ class ZipDataset : public Dataset {
/// \brief Function to create a SchemaObj.
/// \param[in] schema_file Path of schema file.
/// \note This API exists because std::string will constrained by ABI compile option while char don't.
/// \note The reason for using this API is that std::string will be constrained by the
/// compiler option '_GLIBCXX_USE_CXX11_ABI' while char is free of this restriction.
/// \return Shared pointer to the current schema.
std::shared_ptr<SchemaObj> SchemaCharIF(const std::vector<char> &schema_file);
@ -1012,7 +1013,7 @@ inline std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const s
/// \param[in] annotation_file Path to the annotation json.
/// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint'.
/// \param[in] decode Decode the images after reading.
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset..
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
/// \param[in] extra_metadata Flag to add extra meta-data to row. (default=false)
/// \return Shared pointer to the CocoDataset.
@ -1111,7 +1112,7 @@ class ImageFolderDataset : public Dataset {
/// \brief Function to create an ImageFolderDataset.
/// \note A source dataset that reads images from a tree of directories.
/// All images within one folder have the same label.
/// The generated dataset has two columns ["image", "label"]
/// The generated dataset has two columns ["image", "label"].
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] decode A flag to decode in ImageFolder.
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
@ -1366,7 +1367,6 @@ inline std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file
/// ShuffleMode::kGlobal - Shuffle both the files and samples.
/// ShuffleMode::kInfile - Shuffle samples in file.
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
/// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used).
/// \return Shared pointer to the MindDataDataset.
inline std::shared_ptr<MindDataDataset> MindData(
const std::vector<std::string> &dataset_files, const std::vector<std::string> &columns_list = {},

80
mindspore/ccsrc/minddata/dataset/include/dataset/execute.h
View File

@ -33,58 +33,62 @@ class DeviceResource;
class Execute {
public:
/// \brief Constructor.
/// \param[in] op TensorOperation to be applied in Eager mode, it accepts op in type of shared pointer.
/// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0).
explicit Execute(std::shared_ptr<TensorOperation> op, MapTargetDevice deviceType = MapTargetDevice::kCpu,
/// \param[in] op TensorOperation to be applied in Eager mode, it accepts operation in type of shared pointer.
/// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0).
explicit Execute(std::shared_ptr<TensorOperation> op, MapTargetDevice device_type = MapTargetDevice::kCpu,
uint32_t device_id = 0);
/// \brief Constructor.
/// \param[in] op TensorTransform to be applied in Eager mode, it accepts op in type of shared pointer.
/// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0).
explicit Execute(std::shared_ptr<TensorTransform> op, MapTargetDevice deviceType = MapTargetDevice::kCpu,
/// \param[in] op TensorTransform to be applied in Eager mode, it accepts operation in type of shared pointer.
/// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0).
explicit Execute(std::shared_ptr<TensorTransform> op, MapTargetDevice device_type = MapTargetDevice::kCpu,
uint32_t device_id = 0);
/// \brief Constructor.
/// \param[in] op TensorTransform to be applied in Eager mode, it accepts op in type of reference.
/// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0).
explicit Execute(std::reference_wrapper<TensorTransform> op, MapTargetDevice deviceType = MapTargetDevice::kCpu,
/// \param[in] op TensorTransform to be applied in Eager mode, it accepts operation in type of reference.
/// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0).
explicit Execute(std::reference_wrapper<TensorTransform> op, MapTargetDevice device_type = MapTargetDevice::kCpu,
uint32_t device_id = 0);
/// \brief Constructor.
/// \param[in] op TensorTransform to be applied in Eager mode, it accepts op in type of raw pointer.
/// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0).
explicit Execute(TensorTransform *op, MapTargetDevice deviceType = MapTargetDevice::kCpu, uint32_t device_id = 0);
/// \param[in] op TensorTransform to be applied in Eager mode, it accepts operation in type of raw pointer.
/// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0).
explicit Execute(TensorTransform *op, MapTargetDevice device_type = MapTargetDevice::kCpu, uint32_t device_id = 0);
/// \brief Constructor.
/// \param[in] ops A vector of TensorOperations to be applied in Eager mode, it accepts op in type of shared pointer.
/// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0).
/// \param[in] ops A vector of TensorOperations to be applied in Eager mode, it accepts operation
/// in type of shared pointer.
/// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0).
explicit Execute(std::vector<std::shared_ptr<TensorOperation>> ops,
MapTargetDevice deviceType = MapTargetDevice::kCpu, uint32_t device_id = 0);
MapTargetDevice device_type = MapTargetDevice::kCpu, uint32_t device_id = 0);
/// \brief Constructor.
/// \param[in] ops A vector of TensorTransforms to be applied in Eager mode, it accepts op in type of shared pointer.
/// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0).
/// \param[in] ops A vector of TensorTransforms to be applied in Eager mode, it accepts operation
/// in type of shared pointer.
/// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0).
explicit Execute(std::vector<std::shared_ptr<TensorTransform>> ops,
MapTargetDevice deviceType = MapTargetDevice::kCpu, uint32_t device_id = 0);
MapTargetDevice device_type = MapTargetDevice::kCpu, uint32_t device_id = 0);
/// \brief Constructor.
/// \param[in] ops A vector of TensorTransforms to be applied in Eager mode, it accepts op in type of raw pointer.
/// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0).
/// \param[in] ops A vector of TensorTransforms to be applied in Eager mode, it accepts operation
/// in type of raw pointer.
/// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0).
explicit Execute(const std::vector<std::reference_wrapper<TensorTransform>> ops,
MapTargetDevice deviceType = MapTargetDevice::kCpu, uint32_t device_id = 0);
MapTargetDevice device_type = MapTargetDevice::kCpu, uint32_t device_id = 0);
/// \brief Constructor.
/// \param[in] ops A vector of TensorTransforms to be applied in Eager mode, it accepts op in type of raw pointer.
/// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0).
explicit Execute(const std::vector<TensorTransform *> &ops, MapTargetDevice deviceType = MapTargetDevice::kCpu,
/// \param[in] ops A vector of TensorTransforms to be applied in Eager mode, it accepts operation
/// in type of raw pointer.
/// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU).
/// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0).
explicit Execute(const std::vector<TensorTransform *> &ops, MapTargetDevice device_type = MapTargetDevice::kCpu,
uint32_t device_id = 0);
/// \brief Destructor.
@ -102,23 +106,23 @@ class Execute {
/// \return Status error code, returns OK if no error encountered.
Status operator()(const std::vector<mindspore::MSTensor> &input_tensor_list, std::vector<mindspore::MSTensor> *out);
/// \brief The function to release device memory on Ascend310.
Status DeviceMemoryRelease();
/// \brief The function to generate AIPP configuration.
std::string AippCfgGenerator();
private:
Status ParseTransforms_();
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
Status ParseTransforms();
Status validate_device_();
/// \brief The function to validate target device setting is valid or not.
Status ValidateDevice();
std::vector<std::shared_ptr<TensorTransform>> transforms_;
std::vector<std::shared_ptr<TensorOperation>> ops_;
MapTargetDevice device_type_;
std::shared_ptr<DeviceResource> device_resource_;
struct ExtraInfo;
std::shared_ptr<ExtraInfo> info_;
};

27
mindspore/ccsrc/minddata/dataset/include/dataset/iterator.h
View File

@ -47,21 +47,21 @@ using MSTensorVec = std::vector<mindspore::MSTensor>;
// Abstract class for iterating over the dataset.
class Iterator {
public:
/// \brief Constructor
/// \brief Constructor.
Iterator();
/// \brief Destructor
/// \brief Destructor.
~Iterator();
/// \brief Method for building and launching the pipeline.
/// \param[in] ops - a vector of DatasetOp in the data pipeline.
/// \param[in] num_epochs Number of epochs passed down to EpochCtrlNode, default -1, infinite epochs
/// \param[in] ds The last DatasetOp in the dataset pipeline.
/// \param[in] num_epochs Number of epochs passed down to EpochCtrlNode (default=-1, which means infinite epochs).
/// \return Status error code, returns OK if no error encountered.
Status BuildAndLaunchTree(std::shared_ptr<Dataset> ds, int32_t num_epochs);
/// \brief Function to get the next row from the data pipeline.
/// \note Type of return data is a map(with column name).
/// \param[out] row - the output tensor row.
/// \note Type of return data is a unordered_map(with column name).
/// \param[out] row The output tensor row.
/// \return Status error code, returns OK if no error encountered.
Status GetNextRow(MSTensorMap *row) {
MSTensorMapChar row_;
@ -72,13 +72,14 @@ class Iterator {
return s;
}
// Char interface(CharIF) of GetNextRow
// This This API exists because std::string will constrained by ABI compile option while char don't.
/// \brief Char interface(CharIF) of GetNextRow.
/// \note The reason for using this API is that std::string will be constrained by the
/// compiler option '_GLIBCXX_USE_CXX11_ABI' while char is free of this restriction.
Status GetNextRowCharIF(MSTensorMapChar *row);
/// \brief Function to get the next row from the data pipeline.
/// \note Type of return data is a vector(without column name).
/// \param[out] row - the output tensor row.
/// \param[out] row The output tensor row.
/// \return Status error code, returns OK if no error encountered.
virtual Status GetNextRow(MSTensorVec *row);
@ -119,10 +120,10 @@ class Iterator {
class PullIterator : public Iterator {
public:
/// \brief Constructor
/// \brief Constructor.
PullIterator();
/// \brief Destructor
/// \brief Destructor.
~PullIterator() = default;
/// \brief Function to get next row from the data pipeline.
@ -133,7 +134,7 @@ class PullIterator : public Iterator {
/// \brief Function to get specified rows from the data pipeline.
/// \note Type of return data is a vector(without column name).
/// \note This behavior is subject to change
/// \note This behavior is subject to change.
/// \param[in] num_rows The number of rows to fetch.
/// \param[out] row The output tensor row.
/// \return Status error code, returns OK if no error encountered else false.
@ -141,7 +142,7 @@ class PullIterator : public Iterator {
/// \brief Method for building and launching the pipeline.
/// \note Consider making this function protected.
/// \param[in] ds - The root node that calls the function
/// \param[in] ds The root node that calls the function.
/// \return Status error code, returns OK if no error encountered.
Status BuildAndLaunchTree(std::shared_ptr<Dataset> ds);

14
mindspore/ccsrc/minddata/dataset/include/dataset/samplers.h
View File

@ -87,7 +87,7 @@ class DistributedSampler final : public Sampler {
~DistributedSampler() = default;
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \brief The function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() const override;
@ -118,7 +118,7 @@ class PKSampler final : public Sampler {
~PKSampler() = default;
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \brief The function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() const override;
@ -143,7 +143,7 @@ class RandomSampler final : public Sampler {
~RandomSampler() = default;
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \brief The function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() const override;
@ -167,7 +167,7 @@ class SequentialSampler final : public Sampler {
~SequentialSampler() = default;
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \brief The function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() const override;
@ -191,7 +191,7 @@ class SubsetSampler : public Sampler {
~SubsetSampler() = default;
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \brief The function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() const override;
@ -214,7 +214,7 @@ class SubsetRandomSampler final : public SubsetSampler {
~SubsetRandomSampler() = default;
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \brief The function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() const override;
};
@ -236,7 +236,7 @@ class WeightedRandomSampler final : public Sampler {
~WeightedRandomSampler() = default;
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \brief The function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() const override;

236
mindspore/ccsrc/minddata/dataset/include/dataset/text.h
View File

@ -40,19 +40,19 @@ namespace text {
#ifndef _WIN32
/// \brief Tokenize a scalar tensor of UTF-8 string by specific rules.
/// \note BasicTokenizer is not supported on Windows platform yet.
/// \note BasicTokenizer is not supported on the Windows platform yet.
class BasicTokenizer final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] lower_case If true, apply CaseFold, NormalizeUTF8 (NFD mode), RegexReplace operation on input text to
/// fold the text to lower case and strip accents characters. If false, only apply
/// NormalizeUTF8('normalization_form' mode) operation on input text (default=false).
/// \param[in] keep_whitespace If true, the whitespace will be kept in out tokens (default=false).
/// \param[in] normalize_form Used to specify a specific normalize mode. This is only effective when 'lower_case' is
/// false. See NormalizeUTF8 for details (default=NormalizeForm::kNone).
/// \param[in] preserve_unused_token If true, do not split special tokens like '[CLS]', '[SEP]', '[UNK]', '[PAD]',
/// '[MASK]' (default=true).
/// \param[in] with_offsets Whether or not output offsets of tokens (default=false).
/// \param[in] lower_case If true, apply CaseFold, NormalizeUTF8 (NFD mode) and RegexReplace operations to
/// the input text to fold the text to lower case and strip accents characters. If false, only apply
/// the NormalizeUTF8('normalization_form' mode) operation to the input text (default=false).
/// \param[in] keep_whitespace If true, the whitespace will be kept in output tokens (default=false).
/// \param[in] normalize_form This parameter is used to specify a specific normalize mode. This is only effective
/// when 'lower_case' is false. See NormalizeUTF8 for details (default=NormalizeForm::kNone).
/// \param[in] preserve_unused_token If true, do not split special tokens like '[CLS]', '[SEP]', '[UNK]', '[PAD]' and
/// '[MASK]' (default=true).
/// \param[in] with_offsets Whether to output offsets of tokens (default=false).
explicit BasicTokenizer(bool lower_case = false, bool keep_whitespace = false,
const NormalizeForm normalize_form = NormalizeForm::kNone, bool preserve_unused_token = true,
bool with_offsets = false);
@ -61,8 +61,8 @@ class BasicTokenizer final : public TensorTransform {
~BasicTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -70,25 +70,26 @@ class BasicTokenizer final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Tokenizer used for Bert text process.
/// \note BertTokenizer is not supported on Windows platform yet.
/// \brief A tokenizer used for Bert text process.
/// \note BertTokenizer is not supported on the Windows platform yet.
class BertTokenizer final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] vocab A Vocab object.
/// \param[in] suffix_indicator Used to show that the subword is the last part of a word (default='##').
/// \param[in] suffix_indicator This parameter is used to show that the sub-word
/// is the last part of a word (default='##').
/// \param[in] max_bytes_per_token Tokens exceeding this length will not be further split (default=100).
/// \param[in] unknown_token When a token cannot be found, return the token directly if 'unknown_token' is an empty
/// string, else return the string specified(default='[UNK]').
/// \param[in] lower_case If true, apply CaseFold, NormalizeUTF8 (NFD mode), RegexReplace operation on input text to
/// fold the text to lower case and strip accents characters. If false, only apply
/// NormalizeUTF8('normalization_form' mode) operation on input text (default=false).
/// \param[in] keep_whitespace If true, the whitespace will be kept in out tokens (default=false).
/// \param[in] normalize_form Used to specify a specific normalize mode. This is only effective when 'lower_case' is
/// false. See NormalizeUTF8 for details (default=NormalizeForm::kNone).
/// \param[in] preserve_unused_token If true, do not split special tokens like '[CLS]', '[SEP]', '[UNK]', '[PAD]',
/// string, else return the specified string (default='[UNK]').
/// \param[in] lower_case If true, apply CaseFold, NormalizeUTF8 (NFD mode) and RegexReplace operations to
/// the input text to fold the text to lower case and strip accents characters. If false, only apply
/// the NormalizeUTF8('normalization_form' mode) operation to the input text (default=false).
/// \param[in] keep_whitespace If true, the whitespace will be kept in output tokens (default=false).
/// \param[in] normalize_form This parameter is used to specify a specific normalize mode. This is only effective
/// when 'lower_case' is false. See NormalizeUTF8 for details (default=NormalizeForm::kNone).
/// \param[in] preserve_unused_token If true, do not split special tokens like '[CLS]', '[SEP]', '[UNK]', '[PAD]' and
/// '[MASK]' (default=true).
/// \param[in] with_offsets Whether or not output offsets of tokens (default=false).
/// \param[in] with_offsets Whether to output offsets of tokens (default=false).
explicit BertTokenizer(const std::shared_ptr<Vocab> &vocab, const std::string &suffix_indicator = "##",
int32_t max_bytes_per_token = 100, const std::string &unknown_token = "[UNK]",
bool lower_case = false, bool keep_whitespace = false,
@ -106,8 +107,8 @@ class BertTokenizer final : public TensorTransform {
~BertTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -115,7 +116,7 @@ class BertTokenizer final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Apply case fold operation on UTF-8 string tensor.
/// \brief Apply case fold operation on UTF-8 string tensors.
/// \return Shared pointer to the current TensorOperation.
class CaseFold final : public TensorTransform {
public:
@ -126,26 +127,26 @@ class CaseFold final : public TensorTransform {
~CaseFold() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
#endif
/// \brief Tokenize Chinese string into words based on dictionary.
/// \note The integrity of the HMMSEgment algorithm and MPSegment algorithm files must be confirmed.
/// \brief Tokenize a Chinese string into words based on the dictionary.
/// \note The integrity of the HMMSegment algorithm and MPSegment algorithm files must be confirmed.
class JiebaTokenizer final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] hmm_path Dictionary file is used by HMMSegment algorithm. The dictionary can be obtained on the
/// official website of cppjieba.
/// \param[in] mp_path Dictionary file is used by MPSegment algorithm. The dictionary can be obtained on the
/// official website of cppjieba.
/// \param[in] mode Valid values can be any of [JiebaMode.MP, JiebaMode.HMM, JiebaMode.MIX](default=JiebaMode.MIX).
/// - JiebaMode.kMP, tokenize with MPSegment algorithm.
/// - JiebaMode.kHMM, tokenize with Hiddel Markov Model Segment algorithm.
/// - JiebaMode.kMIX, tokenize with a mix of MPSegment and HMMSegment algorithm.
/// \param[in] with_offsets Whether or not output offsets of tokens (default=false).
/// \param[in] hmm_path Dictionary file is used by the HMMSegment algorithm. The dictionary can be obtained on the
/// official website of cppjieba (https://github.com/yanyiwu/cppjieba).
/// \param[in] mp_path Dictionary file is used by the MPSegment algorithm. The dictionary can be obtained on the
/// official website of cppjieba (https://github.com/yanyiwu/cppjieba).
/// \param[in] mode Valid values can be any of JiebaMode.MP, JiebaMode.HMM and JiebaMode.MIX (default=JiebaMode.MIX).
/// - JiebaMode.kMP, tokenizes with MPSegment algorithm.
/// - JiebaMode.kHMM, tokenizes with Hidden Markov Model Segment algorithm.
/// - JiebaMode.kMIX, tokenizes with a mix of MPSegment and HMMSegment algorithms.
/// \param[in] with_offsets Whether to output offsets of tokens (default=false).
explicit JiebaTokenizer(const std::string &hmm_path, const std::string &mp_path,
const JiebaMode &mode = JiebaMode::kMix, bool with_offsets = false)
: JiebaTokenizer(StringToChar(hmm_path), StringToChar(mp_path), mode, with_offsets) {}
@ -156,46 +157,46 @@ class JiebaTokenizer final : public TensorTransform {
/// \brief Destructor
~JiebaTokenizer() = default;
/// \brief Add user defined word to JiebaTokenizer's dictionary.
/// \brief Add a user defined word to the JiebaTokenizer's dictionary.
/// \param[in] word The word to be added to the JiebaTokenizer instance.
/// The added word will not be written into the built-in dictionary on disk.
/// \param[in] freq The frequency of the word to be added. The higher the frequency,
/// the better chance the word will be tokenized (default=None, use default frequency).
/// \return Status error code, returns OK if no error encountered.
/// \return Status error code, returns OK if no error is encountered.
Status AddWord(const std::string &word, int64_t freq = 0) { return AddWordChar(StringToChar(word), freq); }
/// \brief Add user defined dictionary of word-freq pairs to JiebaTokenizer's dictionary.
/// \param[in] user_dict Vector of word-freq pairs to be added to JiebaTokenizer's dictionary.
/// \return Status error code, returns OK if no error encountered.
/// \brief Add a user defined dictionary of word-freq pairs to the JiebaTokenizer's dictionary.
/// \param[in] user_dict Vector of word-freq pairs to be added to the JiebaTokenizer's dictionary.
/// \return Status error code, returns OK if no error is encountered.
Status AddDict(const std::vector<std::pair<std::string, int64_t>> &user_dict) {
return AddDictChar(PairStringInt64ToPairCharInt64(user_dict));
}
/// \brief Add user defined dictionary of word-freq pairs to JiebaTokenizer's dictionary from a file.
/// Only valid word-freq pairs in user provided file will be added into the dictionary.
/// Rows containing invalid input will be ignored, no error nor warning Status is returned.
/// \brief Add user defined dictionary of word-freq pairs to the JiebaTokenizer's dictionary from a file.
/// Only valid word-freq pairs in user defined file will be added into the dictionary.
/// Rows containing invalid inputs will be ignored, no error nor warning status is returned.
/// \param[in] file_path Path to the dictionary which includes user defined word-freq pairs.
/// \return Status error code, returns OK if no error encountered.
/// \return Status error code, returns OK if no error is encountered.
Status AddDict(const std::string &file_path) { return AddDictChar(StringToChar(file_path)); }
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
/// \brief Parser user defined word by file.
/// \brief Parser user defined words by files.
/// \param[in] file_path Path to the user defined file.
/// \param[in] user_dict Vector of word-freq pairs extracted from the user provided file.
/// \param[in] user_dict Vector of word-freq pairs extracted from the user defined file.
Status ParserFile(const std::string &file_path, std::vector<std::pair<std::string, int64_t>> *const user_dict);
/// \brief Used to translate all API string to vector of char and back
/// \brief Used to translate all API strings to vector of char and reverse.
Status AddWordChar(const std::vector<char> &word, int64_t freq = 0);
/// \brief Used to translate all API string to vector of char and back
/// \brief Used to translate all API strings to vector of char and reverse.
Status AddDictChar(const std::vector<std::pair<std::vector<char>, int64_t>> &user_dict);
/// \brief Used to translate all API string to vector of char and back
/// \brief Used to translate all API strings to vector of char and reverse.
Status AddDictChar(const std::vector<char> &file_path);
struct Data;
@ -207,9 +208,9 @@ class Lookup final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] vocab a Vocab object.
/// \param[in] unknown_token word to use for lookup if the word being looked up is out of Vocabulary (oov).
/// If unknown_token is oov, runtime error will be thrown. If unknown_token is {}, which means that not to
/// specify unknown_token when word being out of Vocabulary (default={}).
/// \param[in] unknown_token Word is used for lookup. In case of the word is out of vocabulary (OOV),
/// the result of lookup will be replaced to unknown_token. If the unknown_token is not specified or it is OOV,
/// runtime error will be thrown (default={}, means no unknown_token is specified).
/// \param[in] data_type mindspore::DataType of the tensor after lookup; must be numeric, including bool.
/// (default=mindspore::DataType::kNumberTypeInt32).
explicit Lookup(const std::shared_ptr<Vocab> &vocab, const std::optional<std::string> &unknown_token = {},
@ -223,8 +224,8 @@ class Lookup final : public TensorTransform {
~Lookup() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -232,17 +233,17 @@ class Lookup final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief TensorOp to generate n-gram from a 1-D string Tensor.
/// \brief Generate n-gram from a 1-D string Tensor.
class Ngram final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] ngrams ngrams is a vector of positive integers. For example, if ngrams={4, 3}, then the result
/// would be a 4-gram followed by a 3-gram in the same tensor. If the number of words is not enough to make up
/// for a n-gram, an empty string will be returned.
/// would be a 4-gram followed by a 3-gram in the same tensor. If the number of words is not enough to make up
/// a n-gram, an empty string will be returned.
/// \param[in] left_pad {"pad_token", pad_width}. Padding performed on left side of the sequence. pad_width will
/// be capped at n-1. left_pad=("_",2) would pad left side of the sequence with "__" (default={"", 0}}).
/// be capped at n-1. left_pad=("_",2) would pad the left side of the sequence with "__" (default={"", 0}}).
/// \param[in] right_pad {"pad_token", pad_width}. Padding performed on right side of the sequence.pad_width will
/// be capped at n-1. right_pad=("-":2) would pad right side of the sequence with "--" (default={"", 0}}).
/// be capped at n-1. right_pad=("-":2) would pad the right side of the sequence with "--" (default={"", 0}}).
/// \param[in] separator Symbol used to join strings together (default=" ").
explicit Ngram(const std::vector<int32_t> &ngrams, const std::pair<std::string, int32_t> &left_pad = {"", 0},
const std::pair<std::string, int32_t> &right_pad = {"", 0}, const std::string &separator = " ")
@ -255,8 +256,8 @@ class Ngram final : public TensorTransform {
~Ngram() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -265,7 +266,7 @@ class Ngram final : public TensorTransform {
};
#ifndef _WIN32
/// \brief Apply normalize operation on UTF-8 string tensor.
/// \brief Apply normalize operation to UTF-8 string tensors.
class NormalizeUTF8 final : public TensorTransform {
public:
/// \brief Constructor.
@ -273,19 +274,19 @@ class NormalizeUTF8 final : public TensorTransform {
/// NormalizeForm::kNfkc,
/// NormalizeForm::kNfd, NormalizeForm::kNfkd](default=NormalizeForm::kNfkc).
/// See http://unicode.org/reports/tr15/ for details.
/// - NormalizeForm.NONE, do nothing for input string tensor.
/// - NormalizeForm.NFC, normalize with Normalization Form C.
/// - NormalizeForm.NFKC, normalize with Normalization Form KC.
/// - NormalizeForm.NFD, normalize with Normalization Form D.
/// - NormalizeForm.NFKD, normalize with Normalization Form KD.
/// - NormalizeForm.NONE, remain the input string tensor unchanged.
/// - NormalizeForm.NFC, normalizes with Normalization Form C.
/// - NormalizeForm.NFKC, normalizes with Normalization Form KC.
/// - NormalizeForm.NFD, normalizes with Normalization Form D.
/// - NormalizeForm.NFKD, normalizes with Normalization Form KD.
explicit NormalizeUTF8(NormalizeForm normalize_form = NormalizeForm::kNfkc);
/// \brief Destructor
~NormalizeUTF8() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -293,13 +294,13 @@ class NormalizeUTF8 final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Replace UTF-8 string tensor with 'replace' according to regular expression 'pattern'.
/// \brief Replace a UTF-8 string tensor with 'replace' according to regular expression 'pattern'.
class RegexReplace final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] pattern The regex expression patterns.
/// \param[in] replace The string to replace matched element.
/// \param[in] replace_all Confirm whether to replace all. If false, only replace first matched element;
/// \param[in] replace The string to replace the matched element.
/// \param[in] replace_all Confirm whether to replace all. If false, only replace the first matched element;
/// if true, replace all matched elements (default=true).
explicit RegexReplace(std::string pattern, std::string replace, bool replace_all = true)
: RegexReplace(StringToChar(pattern), StringToChar(replace), replace_all) {}
@ -310,8 +311,8 @@ class RegexReplace final : public TensorTransform {
~RegexReplace() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -319,15 +320,15 @@ class RegexReplace final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Tokenize a scalar tensor of UTF-8 string by regex expression pattern.
/// \brief Tokenize a scalar tensor of UTF-8 string by the regex expression pattern.
class RegexTokenizer final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] delim_pattern The pattern of regex delimiters.
/// \param[in] keep_delim_pattern The string matched by 'delim_pattern' can be kept as a token if it can be
/// \param[in] keep_delim_pattern The string matched with 'delim_pattern' can be kept as a token if it can be
/// matched by 'keep_delim_pattern'. The default value is an empty string ("").
/// which means that delimiters will not be kept as an output token (default="").
/// \param[in] with_offsets Whether or not output offsets of tokens (default=false).
/// \param[in] with_offsets Whether to output offsets of tokens (default=false).
explicit RegexTokenizer(std::string delim_pattern, std::string keep_delim_pattern = "", bool with_offsets = false)
: RegexTokenizer(StringToChar(delim_pattern), StringToChar(keep_delim_pattern), with_offsets) {}
@ -338,8 +339,8 @@ class RegexTokenizer final : public TensorTransform {
~RegexTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -348,18 +349,18 @@ class RegexTokenizer final : public TensorTransform {
};
#endif
/// \brief Tokenize scalar token or 1-D tokens to tokens by sentencepiece.
/// \brief Tokenize a scalar token or a 1-D token to tokens by sentencepiece.
class SentencePieceTokenizer final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] vocab a SentencePieceVocab object.
/// \param[in] out_type The type of output.
/// \param[in] out_type The type of the output.
SentencePieceTokenizer(const std::shared_ptr<SentencePieceVocab> &vocab,
mindspore::dataset::SPieceTokenizerOutType out_type);
/// \brief Constructor.
/// \param[in] vocab_path vocab model file path.
/// \param[in] out_type The type of output.
/// \param[in] out_type The type of the output.
SentencePieceTokenizer(const std::string &vocab_path, mindspore::dataset::SPieceTokenizerOutType out_type)
: SentencePieceTokenizer(StringToChar(vocab_path), out_type) {}
@ -369,8 +370,8 @@ class SentencePieceTokenizer final : public TensorTransform {
~SentencePieceTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -378,22 +379,22 @@ class SentencePieceTokenizer final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief TensorOp to construct a tensor from data (only 1-D for now), where each element in the dimension
/// \brief Construct a tensor from data (only 1-D for now), where each element in the dimension
/// axis is a slice of data starting at the corresponding position, with a specified width.
class SlidingWindow final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] width The width of the window. It must be an integer and greater than zero.
/// \param[in] axis The axis along which the sliding window is computed (default=0), axis support 0 or -1 only
/// for now.
/// \param[in] axis The axis where the sliding window is computed (default=0), axis only
/// supports 0 or -1 for now.
explicit SlidingWindow(const int32_t width, const int32_t axis = 0);
/// \brief Destructor
~SlidingWindow() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -401,7 +402,7 @@ class SlidingWindow final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Tensor operation to convert every element of a string tensor to a number.
/// \brief Convert every element in a string tensor to a number.
/// Strings are cast according to the rules specified in the following links:
/// https://en.cppreference.com/w/cpp/string/basic_string/stof,
/// https://en.cppreference.com/w/cpp/string/basic_string/stoul,
@ -416,8 +417,8 @@ class ToNumber final : public TensorTransform {
~ToNumber() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -436,8 +437,8 @@ class TruncateSequencePair final : public TensorTransform {
~TruncateSequencePair() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -449,15 +450,15 @@ class TruncateSequencePair final : public TensorTransform {
class UnicodeCharTokenizer final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] with_offsets Whether or not output offsets of tokens (default=false).
/// \param[in] with_offsets whether to output offsets of tokens (default=false).
explicit UnicodeCharTokenizer(bool with_offsets = false);
/// \brief Destructor
~UnicodeCharTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -465,16 +466,17 @@ class UnicodeCharTokenizer final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Tokenize scalar token or 1-D tokens to 1-D subword tokens.
/// \brief Tokenize scalar token or 1-D tokens to 1-D sub-word tokens.
class WordpieceTokenizer final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] vocab A Vocab object.
/// \param[in] suffix_indicator Used to show that the subword is the last part of a word (default='##').
/// \param[in] suffix_indicator This parameter is used to show that the sub-word
/// is the last part of a word (default='##').
/// \param[in] max_bytes_per_token Tokens exceeding this length will not be further split (default=100).
/// \param[in] unknown_token When a token cannot be found, return the token directly if 'unknown_token' is an empty
/// string, else return the string specified (default='[UNK]').
/// \param[in] with_offsets Whether or not output offsets of tokens (default=false).
/// string, else return the specified string (default='[UNK]').
/// \param[in] with_offsets whether to output offsets of tokens (default=false).
explicit WordpieceTokenizer(const std::shared_ptr<Vocab> &vocab, const std::string &suffix_indicator = "##",
int32_t max_bytes_per_token = 100, const std::string &unknown_token = "[UNK]",
bool with_offsets = false)
@ -488,8 +490,8 @@ class WordpieceTokenizer final : public TensorTransform {
~WordpieceTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -502,16 +504,16 @@ class WordpieceTokenizer final : public TensorTransform {
class UnicodeScriptTokenizer final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] keep_whitespace Whether or not emit whitespace tokens (default=false).
/// \param[in] with_offsets Whether or not output offsets of tokens (default=false).
/// \param[in] keep_whitespace whether to emit whitespace tokens (default=false).
/// \param[in] with_offsets whether to output offsets of tokens (default=false).
explicit UnicodeScriptTokenizer(bool keep_whitespace = false, bool with_offsets = false);
/// \brief Destructor
~UnicodeScriptTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
@ -523,15 +525,15 @@ class UnicodeScriptTokenizer final : public TensorTransform {
class WhitespaceTokenizer final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] with_offsets Whether or not output offsets of tokens (default=false).
/// \param[in] with_offsets whether to output offsets of tokens (default=false).
explicit WhitespaceTokenizer(bool with_offsets = false);
/// \brief Destructor
~WhitespaceTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to the TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:

84
mindspore/ccsrc/minddata/dataset/include/dataset/transforms.h
View File

@ -103,7 +103,7 @@ class Slice {
dsize_t step_;
};
/// \brief SliceOption used in Slice Op.
/// \brief SliceOption used in Slice TensorTransform.
class SliceOption {
public:
/// \param[in] all Slice the whole dimension
@ -127,8 +127,7 @@ class SliceOption {
// Transform operations for performing data transformation.
namespace transforms {
/// \brief Compose Op.
/// \note Compose a list of transforms into a single transform.
/// \brief Compose a list of transforms into a single transform.
class Compose final : public TensorTransform {
public:
/// \brief Constructor.
@ -145,7 +144,7 @@ class Compose final : public TensorTransform {
~Compose() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -154,21 +153,20 @@ class Compose final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Concatenate Op.
/// \note Tensor operation that concatenates all columns into a single tensor.
/// \brief Concatenate all tensors into a single tensor.
class Concatenate final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] axis Concatenate the tensors along given axis, only support 0 or -1 so far (default=0).
/// \param[in] prepend MSTensor to be prepended to the already concatenated tensors (default={}).
/// \param[in] append MSTensor to be appended to the already concatenated tensors (default={}).
/// \param[in] prepend MSTensor to be prepended to the concatenated tensors (default={}).
/// \param[in] append MSTensor to be appended to the concatenated tensors (default={}).
explicit Concatenate(int8_t axis = 0, MSTensor prepend = {}, MSTensor append = {});
/// \brief Destructor
~Concatenate() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -177,8 +175,7 @@ class Concatenate final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Duplicate Op.
/// \note Duplicate the input tensor to a new output tensor.
/// \brief Duplicate the input tensor to a new output tensor.
/// The input tensor is carried over to the output list.
class Duplicate final : public TensorTransform {
public:
@ -189,19 +186,18 @@ class Duplicate final : public TensorTransform {
~Duplicate() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief Fill Op.
/// \note Tensor operation to fill all elements in the tensor with the specified value.
/// \brief Fill all elements in the tensor with the specified value.
/// The output tensor will have the same shape and type as the input tensor.
class Fill final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] fill_value Scalar value to fill the tensor with.
/// Can only be MSTensor of the following types from mindspore::DataType:
/// It can only be MSTensor of the following types from mindspore::DataType:
/// String, Bool, Int8/16/32/64, UInt8/16/32/64, Float16/32/64.
explicit Fill(MSTensor fill_value);
@ -209,7 +205,7 @@ class Fill final : public TensorTransform {
~Fill() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -218,16 +214,15 @@ class Fill final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Mask Op.
/// \note Mask content of the input tensor with the given predicate.
/// \brief Mask content of the input tensor with the given predicate.
/// Any element of the tensor that matches the predicate will be evaluated to True, otherwise False.
class Mask final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] op One of the relational operators EQ, NE LT, GT, LE or GE.
/// \param[in] constant Constant to be compared to.
/// Can only be MSTensor of str, int, float, bool.
/// \param[in] de_type Type of the generated mask. Can only be numeric or boolean datatype.
/// \param[in] op One of the relational operators: EQ, NE LT, GT, LE or GE.
/// \param[in] constant Constant to be compared to. It can only be MSTensor of the following types
/// from mindspore::DataType: String, Int, Float, Bool.
/// \param[in] de_type Type of the generated mask. It can only be numeric or boolean datatype.
/// (default=mindspore::DataType::kNumberTypeBool)
explicit Mask(RelationalOp op, MSTensor constant,
mindspore::DataType ms_type = mindspore::DataType(mindspore::DataType::kNumberTypeBool));
@ -236,7 +231,7 @@ class Mask final : public TensorTransform {
~Mask() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -245,8 +240,7 @@ class Mask final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief OneHot Op.
/// \note Convert the labels into OneHot format.
/// \brief Convert the labels into OneHot format.
class OneHot final : public TensorTransform {
public:
/// \brief Constructor.
@ -257,7 +251,7 @@ class OneHot final : public TensorTransform {
~OneHot() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -266,13 +260,12 @@ class OneHot final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief PadEnd Op.
/// \note Pad input tensor according to pad_shape, need to have same rank.
/// \brief Pad input tensor according to pad_shape
class PadEnd final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] pad_shape List of integers representing the shape needed.
/// Dimensions that set to `None` will not be padded (i.e., original dim will be used).
/// \param[in] pad_shape List of integers representing the shape needed, need to have same rank with input tensor.
/// Dimensions that set to `-1` will not be padded (i.e., original dim will be used).
/// Shorter dimensions will truncate the values.
/// \param[in] pad_value Value used to pad (default={}).
explicit PadEnd(const std::vector<dsize_t> &pad_shape, MSTensor pad_value = {});
@ -281,7 +274,7 @@ class PadEnd final : public TensorTransform {
~PadEnd() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -290,8 +283,7 @@ class PadEnd final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomApply Op.
/// \note Randomly perform a series of transforms with a given probability.
/// \brief Randomly perform a series of transforms with a given probability.
class RandomApply final : public TensorTransform {
public:
/// \brief Constructor.
@ -311,7 +303,7 @@ class RandomApply final : public TensorTransform {
~RandomApply() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -320,8 +312,7 @@ class RandomApply final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomChoice Op.
/// \note Randomly selects one transform from a list of transforms to perform operation.
/// \brief Randomly select one transform from a list of transforms to perform on the input tensor.
class RandomChoice final : public TensorTransform {
public:
/// \brief Constructor.
@ -338,7 +329,7 @@ class RandomChoice final : public TensorTransform {
~RandomChoice() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -347,9 +338,8 @@ class RandomChoice final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Slice Op.
/// \note Slice operation to extract a tensor out using the given n slices.
/// The functionality of Slice is similar to NumPy's indexing feature.
/// \brief Extract a tensor out using the given n slices.
/// The functionality of Slice is similar to the feature of indexing of NumPy.
/// (Currently only rank-1 tensors are supported).
class Slice final : public TensorTransform {
public:
@ -361,7 +351,7 @@ class Slice final : public TensorTransform {
~Slice() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -370,8 +360,7 @@ class Slice final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief TypeCast Op.
/// \note Tensor operation to cast to a given MindSpore data type.
/// \brief Cast the MindSpore data type of a tensor to another.
class TypeCast final : public TensorTransform {
public:
/// \brief Constructor.
@ -382,7 +371,7 @@ class TypeCast final : public TensorTransform {
~TypeCast() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -391,9 +380,8 @@ class TypeCast final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Unique Op.
/// \note Return an output tensor containing all the unique elements of the input tensor in
/// the same order that they occur in the input tensor.
/// \brief Return an output tensor that contains all the unique elements of the input tensor in
/// the same order as they appear in the input tensor.
class Unique final : public TensorTransform {
public:
/// \brief Constructor.
@ -403,7 +391,7 @@ class Unique final : public TensorTransform {
~Unique() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};

317
mindspore/ccsrc/minddata/dataset/include/dataset/vision.h
View File

@ -36,12 +36,11 @@ class TensorOperation;
// Transform operations for performing computer vision.
namespace vision {
/// \brief AutoContrast TensorTransform.
/// \note Apply automatic contrast on input image.
/// \brief Apply automatic contrast on the input image.
class AutoContrast final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] cutoff Percent of pixels to cut off from the histogram, the valid range of cutoff value is 0 to 100.
/// \param[in] cutoff Percent of pixels to cut off from the histogram, the valid range of cutoff value is 0 to 50.
/// \param[in] ignore Pixel values to ignore.
explicit AutoContrast(float cutoff = 0.0, std::vector<uint32_t> ignore = {});
@ -49,7 +48,7 @@ class AutoContrast final : public TensorTransform {
~AutoContrast() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -58,30 +57,29 @@ class AutoContrast final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief BoundingBoxAugment TensorTransform.
/// \note Apply a given image transform on a random selection of bounding box regions of a given image.
/// \brief Apply a given image transform on a random selection of bounding box regions of a given image.
class BoundingBoxAugment final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] transform Raw pointer to a TensorTransform operation.
/// \param[in] transform Raw pointer to the TensorTransform operation.
/// \param[in] ratio Ratio of bounding boxes to apply augmentation on. Range: [0, 1] (default=0.3).
explicit BoundingBoxAugment(TensorTransform *transform, float ratio = 0.3);
/// \brief Constructor.
/// \param[in] transform Smart pointer to a TensorTransform operation.
/// \param[in] ratio Ratio of bounding boxes to apply augmentation on. Range: [0, 1] (default=0.3).
/// \param[in] transform Smart pointer to the TensorTransform operation.
/// \param[in] ratio Ratio of bounding boxes where augmentation is applied to. Range: [0, 1] (default=0.3).
explicit BoundingBoxAugment(const std::shared_ptr<TensorTransform> &transform, float ratio = 0.3);
/// \brief Constructor.
/// \param[in] transform Object pointer to a TensorTransform operation.
/// \param[in] ratio Ratio of bounding boxes to apply augmentation on. Range: [0, 1] (default=0.3).
/// \param[in] transform Object pointer to the TensorTransform operation.
/// \param[in] ratio Ratio of bounding boxes where augmentation is applied to. Range: [0, 1] (default=0.3).
explicit BoundingBoxAugment(const std::reference_wrapper<TensorTransform> transform, float ratio = 0.3);
/// \brief Destructor.
~BoundingBoxAugment() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -90,8 +88,7 @@ class BoundingBoxAugment final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Constructor to apply CutMix on a batch of images.
/// \note Masks a random section of each image with the corresponding part of another randomly
/// \brief Mask a random section of each image with the corresponding part of another randomly
/// selected image in that batch.
class CutMixBatch final : public TensorTransform {
public:
@ -105,7 +102,7 @@ class CutMixBatch final : public TensorTransform {
~CutMixBatch() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -114,8 +111,7 @@ class CutMixBatch final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief CutOut TensorOp.
/// \note Randomly cut (mask) out a given number of square patches from the input image.
/// \brief Randomly cut (mask) out a given number of square patches from the input image.
class CutOut final : public TensorTransform {
public:
/// \brief Constructor.
@ -127,7 +123,7 @@ class CutOut final : public TensorTransform {
~CutOut() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -136,8 +132,7 @@ class CutOut final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Equalize TensorTransform.
/// \note Apply histogram equalization on input image.
/// \brief Apply histogram equalization on the input image.
class Equalize final : public TensorTransform {
public:
/// \brief Constructor.
@ -147,13 +142,12 @@ class Equalize final : public TensorTransform {
~Equalize() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief HorizontalFlip TensorTransform.
/// \note Flip the input image horizontally.
/// \brief Flip the input image horizontally.
class HorizontalFlip final : public TensorTransform {
public:
/// \brief Constructor.
@ -163,13 +157,12 @@ class HorizontalFlip final : public TensorTransform {
~HorizontalFlip() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief HwcToChw TensorTransform.
/// \note Transpose the input image; shape (H, W, C) to shape (C, H, W).
/// \brief Transpose the input image; shape (H, W, C) to shape (C, H, W).
class HWC2CHW final : public TensorTransform {
public:
/// \brief Constructor.
@ -179,13 +172,12 @@ class HWC2CHW final : public TensorTransform {
~HWC2CHW() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief Invert TensorTransform.
/// \note Apply invert on input image in RGB mode.
/// \brief Apply invert on the input image in RGB mode.
class Invert final : public TensorTransform {
public:
/// \brief Constructor.
@ -195,13 +187,12 @@ class Invert final : public TensorTransform {
~Invert() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief MixUpBatch TensorTransform.
/// \note Apply MixUp transformation on an input batch of images and labels. The labels must be in
/// \brief Apply MixUp transformation on an input batch of images and labels. The labels must be in
/// one-hot format and Batch must be called before calling this function.
class MixUpBatch final : public TensorTransform {
public:
@ -213,7 +204,7 @@ class MixUpBatch final : public TensorTransform {
~MixUpBatch() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -222,15 +213,14 @@ class MixUpBatch final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief NormalizePad TensorTransform.
/// \note Normalize the input image with respect to mean and standard deviation and pad an extra
/// \brief Normalize the input image with respect to mean and standard deviation and pads an extra
/// channel with value zero.
class NormalizePad final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] mean A vector of mean values for each channel, w.r.t channel order.
/// \param[in] mean A vector of mean values for each channel, with respect to channel order.
/// The mean values must be in range [0.0, 255.0].
/// \param[in] std A vector of standard deviations for each channel, w.r.t. channel order.
/// \param[in] std A vector of standard deviations for each channel, with respect to channel order.
/// The standard deviation values must be in range (0.0, 255.0].
/// \param[in] dtype The output datatype of Tensor.
/// The standard deviation values must be "float32" or "float16"（default = "float32"）.
@ -244,7 +234,7 @@ class NormalizePad final : public TensorTransform {
~NormalizePad() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -253,16 +243,15 @@ class NormalizePad final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Pad TensorOp.
/// \note Pads the image according to padding parameters.
/// \brief Pad the image according to padding parameters.
class Pad final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] padding A vector representing the number of pixels to pad the image
/// If vector has one value, it pads all sides of the image with that value.
/// If vector has two values, it pads left and top with the first and
/// \param[in] padding A vector representing the number of pixels to pad the image.
/// If the vector has one value, it pads all sides of the image with that value.
/// If the vector has two values, it pads left and top with the first and
/// right and bottom with the second value.
/// If vector has four values, it pads left, top, right, and bottom with
/// If the vector has four values, it pads left, top, right, and bottom with
/// those values respectively.
/// \param[in] fill_value A vector representing the pixel intensity of the borders if the padding_mode is
/// BorderType.kConstant. If 1 value is provided, it is used for all RGB channels. If 3 values are provided,
@ -281,7 +270,7 @@ class Pad final : public TensorTransform {
~Pad() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -290,21 +279,21 @@ class Pad final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Blends an image with its grayscale version with random weights
/// \brief Blend an image with its grayscale version with random weights
/// t and 1 - t generated from a given range. If the range is trivial
/// then the weights are determinate and t equals the bound of the interval.
/// then the weights are determinate and t equals to the bound of the interval.
class RandomColor final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] t_lb Lower bound on the range of random weights.
/// \param[in] t_lb Upper bound on the range of random weights.
/// \param[in] t_lb Lower bound random weights.
/// \param[in] t_lb Upper bound random weights.
explicit RandomColor(float t_lb, float t_ub);
/// \brief Destructor.
~RandomColor() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -313,19 +302,18 @@ class RandomColor final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomColorAdjust TensorTransform.
/// \brief Randomly adjust the brightness, contrast, saturation, and hue of the input image.
class RandomColorAdjust final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] brightness Brightness adjustment factor. Must be a vector of one or two values
/// if it's a vector of two values it needs to be in the form of [min, max] (Default={1, 1}).
/// if it is a vector of two values it needs to be in the form of [min, max] (Default={1, 1}).
/// \param[in] contrast Contrast adjustment factor. Must be a vector of one or two values
/// if it's a vector of two values it needs to be in the form of [min, max] (Default={1, 1}).
/// if it is a vector of two values, it needs to be in the form of [min, max] (Default={1, 1}).
/// \param[in] saturation Saturation adjustment factor. Must be a vector of one or two values
/// if it's a vector of two values it needs to be in the form of [min, max] (Default={1, 1}).
/// if it is a vector of two values, it needs to be in the form of [min, max] (Default={1, 1}).
/// \param[in] hue Brightness adjustment factor. Must be a vector of one or two values
/// if it's a vector of two values it must be in the form of [min, max] where -0.5 <= min <= max <= 0.5
/// if it is a vector of two values, it must be in the form of [min, max] where -0.5 <= min <= max <= 0.5
/// (Default={0, 0}).
explicit RandomColorAdjust(std::vector<float> brightness = {1.0, 1.0}, std::vector<float> contrast = {1.0, 1.0},
std::vector<float> saturation = {1.0, 1.0}, std::vector<float> hue = {0.0, 0.0});
@ -334,7 +322,7 @@ class RandomColorAdjust final : public TensorTransform {
~RandomColorAdjust() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -343,22 +331,21 @@ class RandomColorAdjust final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomCrop TensorTransform.
/// \note Crop the input image at a random location.
/// \brief Crop the input image at a random location.
class RandomCrop final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] size A vector representing the output size of the cropped image.
/// If size is a single value, a square crop of size (size, size) is returned.
/// If size has 2 values, it should be (height, width).
/// \param[in] padding A vector representing the number of pixels to pad the image
/// If vector has one value, it pads all sides of the image with that value.
/// If vector has two values, it pads left and top with the first and
/// If the size is a single value, a squared crop of size (size, size) is returned.
/// If the size has 2 values, it should be (height, width).
/// \param[in] padding A vector representing the number of pixels to pad the image.
/// If the vector has one value, it pads all sides of the image with that value.
/// If the vector has two values, it pads left and top with the first and
/// right and bottom with the second value.
/// If vector has four values, it pads left, top, right, and bottom with
/// If the vector has four values, it pads left, top, right, and bottom with
/// those values respectively.
/// \param[in] pad_if_needed A boolean whether to pad the image if either side is smaller than
/// the given output size.
/// \param[in] pad_if_needed A boolean indicating that whether to pad the image
/// if either side is smaller than the given output size.
/// \param[in] fill_value A vector representing the pixel intensity of the borders if the padding_mode is
/// BorderType.kConstant. If 1 value is provided, it is used for all RGB channels.
/// If 3 values are provided, it is used to fill R, G, B channels respectively.
@ -370,7 +357,7 @@ class RandomCrop final : public TensorTransform {
~RandomCrop() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -379,14 +366,13 @@ class RandomCrop final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomCropDecodeResize TensorTransform.
/// \note Equivalent to RandomResizedCrop, but crops before decodes.
/// \brief Equivalent to RandomResizedCrop TensorTransform, but crop the image before decoding.
class RandomCropDecodeResize final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] size A vector representing the output size of the cropped image.
/// If size is a single value, a square crop of size (size, size) is returned.
/// If size has 2 values, it should be (height, width).
/// If the size is a single value, a squared crop of size (size, size) is returned.
/// If the size has 2 values, it should be (height, width).
/// \param[in] scale Range [min, max) of respective size of the
/// original size to be cropped (default=(0.08, 1.0)).
/// \param[in] ratio Range [min, max) of aspect ratio to be
@ -403,7 +389,7 @@ class RandomCropDecodeResize final : public TensorTransform {
~RandomCropDecodeResize() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -412,23 +398,22 @@ class RandomCropDecodeResize final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomCropWithBBox TensorTransform.
/// \note Crop the input image at a random location and adjust bounding boxes accordingly.
/// If cropped area is out of bbox, the return bbox will be empty.
/// \brief Crop the input image at a random location and adjust bounding boxes accordingly.
/// If the cropped area is out of bbox, the returned bbox will be empty.
class RandomCropWithBBox final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] size A vector representing the output size of the cropped image.
/// If size is a single value, a square crop of size (size, size) is returned.
/// If size has 2 values, it should be (height, width).
/// If the size is a single value, a squared crop of size (size, size) is returned.
/// If the size has 2 values, it should be (height, width).
/// \param[in] padding A vector representing the number of pixels to pad the image
/// If vector has one value, it pads all sides of the image with that value.
/// If vector has two values, it pads left and top with the first and
/// If the vector has one value, it pads all sides of the image with that value.
/// If the vector has two values, it pads left and top with the first and
/// right and bottom with the second value.
/// If vector has four values, it pads left, top, right, and bottom with
/// If the vector has four values, it pads left, top, right, and bottom with
/// those values respectively.
/// \param[in] pad_if_needed A boolean whether to pad the image if either side is smaller than
/// the given output size.
/// \param[in] pad_if_needed A boolean indicating that whether to pad the image
/// if either side is smaller than the given output size.
/// \param[in] fill_value A vector representing the pixel intensity of the borders if the padding_mode is
/// BorderType.kConstant. If 1 value is provided, it is used for all RGB channels.
/// If 3 values are provided, it is used to fill R, G, B channels respectively.
@ -442,7 +427,7 @@ class RandomCropWithBBox final : public TensorTransform {
~RandomCropWithBBox() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -451,8 +436,7 @@ class RandomCropWithBBox final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomHorizontalFlip TensorTransform.
/// \note Tensor operation to perform random horizontal flip.
/// \brief Randomly flip the input image horizontally with a given probability.
class RandomHorizontalFlip final : public TensorTransform {
public:
/// \brief Constructor.
@ -463,7 +447,7 @@ class RandomHorizontalFlip final : public TensorTransform {
~RandomHorizontalFlip() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -472,8 +456,7 @@ class RandomHorizontalFlip final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomHorizontalFlipWithBBox TensorTransform.
/// \note Flip the input image horizontally, randomly with a given probability and adjust bounding boxes accordingly.
/// \brief Randomly flip the input image horizontally with a given probability and adjust bounding boxes accordingly.
class RandomHorizontalFlipWithBBox final : public TensorTransform {
public:
/// \brief Constructor.
@ -484,7 +467,7 @@ class RandomHorizontalFlipWithBBox final : public TensorTransform {
~RandomHorizontalFlipWithBBox() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -493,8 +476,7 @@ class RandomHorizontalFlipWithBBox final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomPosterize TensorTransform.
/// \note Tensor operation to perform random posterize.
/// \brief Reduce the number of bits for each color channel randomly.
class RandomPosterize final : public TensorTransform {
public:
/// \brief Constructor.
@ -505,7 +487,7 @@ class RandomPosterize final : public TensorTransform {
~RandomPosterize() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -514,21 +496,20 @@ class RandomPosterize final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomResize TensorTransform.
/// \note Resize the input image using a randomly selected interpolation mode.
// the same image aspect ratio. If size has 2 values, it should be (height, width).
/// \brief Resize the input image using a randomly selected interpolation mode.
class RandomResize final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] size A vector representing the output size of the resized image.
/// If size is a single value, the smaller edge of the image will be resized to this value with.
/// If the size is a single value, the smaller edge of the image will be resized to this value with
// the same image aspect ratio. If the size has 2 values, it should be (height, width).
explicit RandomResize(std::vector<int32_t> size);
/// \brief Destructor.
~RandomResize() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -537,22 +518,21 @@ class RandomResize final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomResizeWithBBox TensorTransform.
/// \note Resize the input image using a randomly selected interpolation mode and adjust
/// \brief Resize the input image using a randomly selected interpolation mode and adjust
/// bounding boxes accordingly.
class RandomResizeWithBBox final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] size A vector representing the output size of the resized image.
/// If size is a single value, the smaller edge of the image will be resized to this value with
// the same image aspect ratio. If size has 2 values, it should be (height, width).
/// If the size is a single value, the smaller edge of the image will be resized to this value with
// the same image aspect ratio. If the size has 2 values, it should be (height, width).
explicit RandomResizeWithBBox(std::vector<int32_t> size);
/// \brief Destructor.
~RandomResizeWithBBox() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -561,14 +541,13 @@ class RandomResizeWithBBox final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomResizedCrop TensorTransform.
/// \note Crop the input image to a random size and aspect ratio.
/// \brief Crop the input image to a random size and aspect ratio.
class RandomResizedCrop final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] size A vector representing the output size of the cropped image.
/// If size is a single value, a square crop of size (size, size) is returned.
/// If size has 2 values, it should be (height, width).
/// If the size is a single value, a squared crop of size (size, size) is returned.
/// If the size has 2 values, it should be (height, width).
/// \param[in] scale Range [min, max) of respective size of the original
/// size to be cropped (default=(0.08, 1.0)).
/// \param[in] ratio Range [min, max) of aspect ratio to be cropped
@ -584,7 +563,7 @@ class RandomResizedCrop final : public TensorTransform {
~RandomResizedCrop() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -593,15 +572,14 @@ class RandomResizedCrop final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomResizedCropWithBBox TensorTransform.
/// \note Crop the input image to a random size and aspect ratio.
/// \brief Crop the input image to a random size and aspect ratio.
/// If cropped area is out of bbox, the return bbox will be empty.
class RandomResizedCropWithBBox final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] size A vector representing the output size of the cropped image.
/// If size is a single value, a square crop of size (size, size) is returned.
/// If size has 2 values, it should be (height, width).
/// If the size is a single value, a squared crop of size (size, size) is returned.
/// If the size has 2 values, it should be (height, width).
/// \param[in] scale Range [min, max) of respective size of the original
/// size to be cropped (default=(0.08, 1.0)).
/// \param[in] ratio Range [min, max) of aspect ratio to be cropped
@ -617,7 +595,7 @@ class RandomResizedCropWithBBox final : public TensorTransform {
~RandomResizedCropWithBBox() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -626,8 +604,7 @@ class RandomResizedCropWithBBox final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomRotation TensorOp.
/// \note Rotates the image according to parameters.
/// \brief Rotate the image according to parameters.
class RandomRotation final : public TensorTransform {
public:
/// \brief Constructor.
@ -635,7 +612,7 @@ class RandomRotation final : public TensorTransform {
/// \param[in] resample An enum for the mode of interpolation.
/// \param[in] expand A boolean representing whether the image is expanded after rotation.
/// \param[in] center A float vector of size 2, representing the x and y center of rotation.
/// \param[in] fill_value A vector representing the value to fill the area outside the transform.
/// \param[in] fill_value A vector representing the value to fill the area outside the transform
/// in the output image. If 1 value is provided, it is used for all RGB channels.
/// If 3 values are provided, it is used to fill R, G, B channels respectively.
RandomRotation(std::vector<float> degrees, InterpolationMode resample = InterpolationMode::kNearestNeighbour,
@ -646,7 +623,7 @@ class RandomRotation final : public TensorTransform {
~RandomRotation() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -655,11 +632,10 @@ class RandomRotation final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomSelectSubpolicy TensorTransform.
/// \note Choose a random sub-policy from a list to be applied on the input image. A sub-policy is a list of tuples
/// (op, prob), where op is a TensorTransform operation and prob is the probability that this op will be applied.
/// Once a sub-policy is selected, each op within the sub-policy with be applied in sequence according to its
/// probability.
/// \brief Choose a random sub-policy from a list to be applied on the input image. A sub-policy is a list of tuples
/// (operation, prob), where operation is a TensorTransform operation and prob is the probability that this
/// operation will be applied. Once a sub-policy is selected, each operation within the sub-policy with be
/// applied in sequence according to its probability.
class RandomSelectSubpolicy final : public TensorTransform {
public:
/// \brief Constructor.
@ -680,7 +656,7 @@ class RandomSelectSubpolicy final : public TensorTransform {
~RandomSelectSubpolicy() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -689,20 +665,20 @@ class RandomSelectSubpolicy final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomSharpness TensorTransform.
/// \note Tensor operation to perform random sharpness.
/// \brief Adjust the sharpness of the input image by a fixed or random degree.
class RandomSharpness final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] degrees A float vector of size 2, representing the starting and ending degree to uniformly.
/// sample from, to select a degree to adjust sharpness.
/// \param[in] degrees A float vector of size 2, representing the range of random sharpness
/// adjustment degrees. It should be in (min, max) format. If min=max, then it is a
/// single fixed magnitude operation (default = (0.1, 1.9)).
explicit RandomSharpness(std::vector<float> degrees = {0.1, 1.9});
/// \brief Destructor.
~RandomSharpness() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -711,20 +687,20 @@ class RandomSharpness final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomSolarize TensorTransform.
/// \note Invert pixels randomly within specified range. If min=max, it is a single fixed magnitude operation
/// to inverts all pixel above that threshold.
/// \brief Invert pixels randomly within a specified range.
class RandomSolarize final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] threshold A vector with two elements specifying the pixel range to invert.
/// Threshold values should always be in (min, max) format.
/// If min=max, it will to invert all pixels above min(max).
explicit RandomSolarize(std::vector<uint8_t> threshold = {0, 255});
/// \brief Destructor.
~RandomSolarize() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -733,8 +709,7 @@ class RandomSolarize final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomVerticalFlip TensorTransform.
/// \note Tensor operation to perform random vertical flip.
/// \brief Randomly flip the input image vertically with a given probability.
class RandomVerticalFlip final : public TensorTransform {
public:
/// \brief Constructor.
@ -745,7 +720,7 @@ class RandomVerticalFlip final : public TensorTransform {
~RandomVerticalFlip() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -754,8 +729,7 @@ class RandomVerticalFlip final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomVerticalFlipWithBBox TensorTransform.
/// \note Flip the input image vertically, randomly with a given probability and adjust bounding boxes accordingly.
/// \brief Randomly flip the input image vertically with a given probability and adjust bounding boxes accordingly.
class RandomVerticalFlipWithBBox final : public TensorTransform {
public:
/// \brief Constructor.
@ -766,7 +740,7 @@ class RandomVerticalFlipWithBBox final : public TensorTransform {
~RandomVerticalFlipWithBBox() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -775,8 +749,7 @@ class RandomVerticalFlipWithBBox final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RescaleOperation TensorTransform.
/// \note Tensor operation to rescale the input image.
/// \brief Rescale the pixel value of input image.
class Rescale final : public TensorTransform {
public:
/// \brief Constructor.
@ -788,7 +761,7 @@ class Rescale final : public TensorTransform {
~Rescale() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -797,14 +770,13 @@ class Rescale final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief ResizeWithBBox TensorTransform.
/// \note Resize the input image to the given size and adjust bounding boxes accordingly.
/// \brief Resize the input image to the given size and adjust bounding boxes accordingly.
class ResizeWithBBox final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] size The output size of the resized image.
/// If size is an integer, smaller edge of the image will be resized to this value with the same image aspect
/// ratio. If size is a sequence of length 2, it should be (height, width).
/// If the size is an integer, smaller edge of the image will be resized to this value with the same image aspect
/// ratio. If the size is a sequence of length 2, it should be (height, width).
/// \param[in] interpolation An enum for the mode of interpolation (default=InterpolationMode::kLinear).
explicit ResizeWithBBox(std::vector<int32_t> size, InterpolationMode interpolation = InterpolationMode::kLinear);
@ -812,7 +784,7 @@ class ResizeWithBBox final : public TensorTransform {
~ResizeWithBBox() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -821,8 +793,7 @@ class ResizeWithBBox final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RgbaToBgr TensorTransform.
/// \note Changes the input 4 channel RGBA tensor to 3 channel BGR.
/// \brief Change the format of input tensor from 4-channel RGBA to 3-channel BGR.
class RGBA2BGR final : public TensorTransform {
public:
/// \brief Constructor.
@ -832,13 +803,12 @@ class RGBA2BGR final : public TensorTransform {
~RGBA2BGR() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief RgbaToRgb TensorTransform.
/// \note Changes the input 4 channel RGBA tensor to 3 channel RGB.
/// \brief Change the input 4 channel RGBA tensor to 3 channel RGB.
class RGBA2RGB final : public TensorTransform {
public:
/// \brief Constructor.
@ -848,23 +818,22 @@ class RGBA2RGB final : public TensorTransform {
~RGBA2RGB() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief SoftDvppDecodeRandomCropResizeJpeg TensorTransform.
/// \note Tensor operation to decode, random crop and resize JPEG image using the simulation algorithm of
/// Ascend series chip DVPP module. The usage scenario is consistent with SoftDvppDecodeResizeJpeg.
/// \brief Decode, randomly crop and resize a JPEG image using the simulation algorithm of
/// Ascend series chip DVPP module. The application scenario is consistent with SoftDvppDecodeResizeJpeg.
/// The input image size should be in range [32*32, 8192*8192].
/// The zoom-out and zoom-in multiples of the image length and width should in the range [1/32, 16].
/// The zoom-out and zoom-in multiples of the image length and width should be in the range [1/32, 16].
/// Only images with an even resolution can be output. The output of odd resolution is not supported.
class SoftDvppDecodeRandomCropResizeJpeg final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] size A vector representing the output size of the resized image.
/// If size is a single value, smaller edge of the image will be resized to this value with
/// the same image aspect ratio. If size has 2 values, it should be (height, width).
/// If the size is a single value, smaller edge of the image will be resized to this value with
/// the same image aspect ratio. If the size has 2 values, it should be (height, width).
/// \param[in] scale Range [min, max) of respective size of the original
/// size to be cropped (default=(0.08, 1.0)).
/// \param[in] ratio Range [min, max) of aspect ratio to be cropped
@ -878,7 +847,7 @@ class SoftDvppDecodeRandomCropResizeJpeg final : public TensorTransform {
~SoftDvppDecodeRandomCropResizeJpeg() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -887,28 +856,27 @@ class SoftDvppDecodeRandomCropResizeJpeg final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief SoftDvppDecodeResizeJpeg TensorTransform.
/// \note Tensor operation to decode and resize JPEG image using the simulation algorithm of Ascend series
/// \brief Decode and resize a JPEG image using the simulation algorithm of Ascend series
/// chip DVPP module. It is recommended to use this algorithm in the following scenarios:
/// When training, the DVPP of the Ascend chip is not used,
/// and the DVPP of the Ascend chip is used during inference,
/// and the accuracy of inference is lower than the accuracy of training;
/// and the input image size should be in range [32*32, 8192*8192].
/// The zoom-out and zoom-in multiples of the image length and width should in the range [1/32, 16].
/// The zoom-out and zoom-in multiples of the image length and width should be in the range [1/32, 16].
/// Only images with an even resolution can be output. The output of odd resolution is not supported.
class SoftDvppDecodeResizeJpeg final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] size A vector representing the output size of the resized image.
/// If size is a single value, smaller edge of the image will be resized to this value with
/// the same image aspect ratio. If size has 2 values, it should be (height, width).
/// If the size is a single value, smaller edge of the image will be resized to this value with
/// the same image aspect ratio. If the size has 2 values, it should be (height, width).
explicit SoftDvppDecodeResizeJpeg(std::vector<int32_t> size);
/// \brief Destructor.
~SoftDvppDecodeResizeJpeg() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -917,8 +885,7 @@ class SoftDvppDecodeResizeJpeg final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief SwapRedBlue TensorOp.
/// \note Swaps the red and blue channels in image.
/// \brief Swap the red and blue channels of the input image.
class SwapRedBlue final : public TensorTransform {
public:
/// \brief Constructor.
@ -928,35 +895,34 @@ class SwapRedBlue final : public TensorTransform {
~SwapRedBlue() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief UniformAugment TensorTransform.
/// \note Tensor operation to perform randomly selected augmentation.
/// \brief Randomly perform transformations, as selected from input transform list, on the input tensor.
class UniformAugment final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] transforms Raw pointer to vector of TensorTransform operations.
/// \param[in] num_ops An integer representing the number of OPs to be selected and applied.
/// \param[in] num_ops An integer representing the number of operations to be selected and applied.
explicit UniformAugment(const std::vector<TensorTransform *> &transforms, int32_t num_ops = 2);
/// \brief Constructor.
/// \param[in] transforms Smart pointer to vector of TensorTransform operations.
/// \param[in] num_ops An integer representing the number of OPs to be selected and applied.
/// \param[in] num_ops An integer representing the number of operations to be selected and applied.
explicit UniformAugment(const std::vector<std::shared_ptr<TensorTransform>> &transforms, int32_t num_ops = 2);
/// \brief Constructor.
/// \param[in] transforms Object pointer to vector of TensorTransform operations.
/// \param[in] num_ops An integer representing the number of OPs to be selected and applied.
/// \param[in] num_ops An integer representing the number of operations to be selected and applied.
explicit UniformAugment(const std::vector<std::reference_wrapper<TensorTransform>> &transforms, int32_t num_ops = 2);
/// \brief Destructor.
~UniformAugment() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -965,8 +931,7 @@ class UniformAugment final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief VerticalFlip TensorTransform.
/// \note Flip the input image Vertically.
/// \brief Flip the input image vertically.
class VerticalFlip final : public TensorTransform {
public:
/// \brief Constructor.
@ -976,7 +941,7 @@ class VerticalFlip final : public TensorTransform {
~VerticalFlip() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};

18
mindspore/ccsrc/minddata/dataset/include/dataset/vision_ascend.h
View File

@ -34,17 +34,19 @@ namespace vision {
/* ##################################### API class ###########################################*/
/// \brief Decode and resize JPEG image using the hardware algorithm of
/// Ascend series chip DVPP module.
class DvppDecodeResizeJpeg final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] resize A vector of int value for each dimension, w.r.t H,W order.
/// \param[in] resize A vector of int value for each dimension, with respect to H,W order.
explicit DvppDecodeResizeJpeg(std::vector<uint32_t> resize);
/// \brief Destructor.
~DvppDecodeResizeJpeg() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -55,18 +57,20 @@ class DvppDecodeResizeJpeg final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Decode, resize and crop JPEG image using the hardware algorithm of
/// Ascend series chip DVPP module.
class DvppDecodeResizeCropJpeg final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] crop A vector of int value for each dimension after final crop, w.r.t H,W order.
/// \param[in] resize A vector of int value for each dimension after resize, w.r.t H,W order.
/// \param[in] crop A vector of int value for each dimension after final cropping, with respect to H,W order.
/// \param[in] resize A vector of int value for each dimension after resizing, with respect to H,W order.
explicit DvppDecodeResizeCropJpeg(std::vector<uint32_t> crop, std::vector<uint32_t> resize);
/// \brief Destructor.
~DvppDecodeResizeCropJpeg() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -77,6 +81,8 @@ class DvppDecodeResizeCropJpeg final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Decode PNG image using the hardware algorithm of
/// Ascend series chip DVPP module.
class DvppDecodePng final : public TensorTransform {
public:
/// \brief Constructor.
@ -86,7 +92,7 @@ class DvppDecodePng final : public TensorTransform {
~DvppDecodePng() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;

97
mindspore/ccsrc/minddata/dataset/include/dataset/vision_lite.h
View File

@ -35,18 +35,17 @@ namespace vision {
// Forward Declarations
class RotateOperation;
/// \brief Affine TensorTransform.
/// \note Apply affine transform on input image.
/// \brief Apply affine transform on the input image.
class Affine final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] degrees The degrees to rotate the image by.
/// \param[in] translation The value representing vertical and horizontal translation (default = {0.0, 0.0}).
/// The first value represent the x axis translation while the second represents y axis translation.
/// \param[in] degrees The degrees to rotate the image.
/// \param[in] translation The values representing vertical and horizontal translation (default = {0.0, 0.0}).
/// The first value represents the x axis translation while the second represents the y axis translation.
/// \param[in] scale The scaling factor for the image (default = 0.0).
/// \param[in] shear A float vector of size 2, representing the shear degrees (default = {0.0, 0.0}).
/// \param[in] interpolation An enum for the mode of interpolation.
/// \param[in] fill_value A vector representing the value to fill the area outside the transform
/// \param[in] fill_value A vector representing the value to fill the area outside the transformation
/// in the output image. If 1 value is provided, it is used for all RGB channels.
/// If 3 values are provided, it is used to fill R, G, B channels respectively.
explicit Affine(float_t degrees, const std::vector<float> &translation = {0.0, 0.0}, float scale = 0.0,
@ -57,7 +56,7 @@ class Affine final : public TensorTransform {
/// \brief Destructor.
~Affine() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -66,21 +65,20 @@ class Affine final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief CenterCrop TensorTransform.
/// \note Crops the input image at the center to the given size.
/// \brief Crop the input image at the center to the given size.
class CenterCrop final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] size A vector representing the output size of the cropped image.
/// If size is a single value, a square crop of size (size, size) is returned.
/// If size has 2 values, it should be (height, width).
/// If the size is a single value, a squared crop of size (size, size) is returned.
/// If the size has 2 values, it should be (height, width).
explicit CenterCrop(std::vector<int32_t> size);
/// \brief Destructor.
~CenterCrop() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -92,7 +90,7 @@ class CenterCrop final : public TensorTransform {
};
/// \brief RGB2BGR TensorTransform.
/// \notes Convert RGB image to BGR image
/// \notes Convert the format of input image from RGB to BGR.
class RGB2BGR final : public TensorTransform {
public:
/// \brief Constructor.
@ -102,13 +100,14 @@ class RGB2BGR final : public TensorTransform {
~RGB2BGR() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief RGB2GRAY TensorTransform.
/// \note Convert RGB image or color image to grayscale image.
/// \brief Convert a RGB image or color image to a grayscale one.
class RGB2GRAY final : public TensorTransform {
public:
/// \brief Constructor.
@ -118,27 +117,26 @@ class RGB2GRAY final : public TensorTransform {
~RGB2GRAY() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief Crop TensorTransform.
/// \note Crop an image based on location and crop size.
/// \brief Crop an image based on location and crop size.
class Crop final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] coordinates Starting location of crop. Must be a vector of two values, in the form of {x_coor, y_coor}.
/// \param[in] size Size of the cropped area.
/// If size is a single value, a square crop of size (size, size) is returned.
/// If size has 2 values, it should be (height, width).
/// If the size is a single value, a squared crop of size (size, size) is returned.
/// If the size has 2 values, it should be (height, width).
Crop(std::vector<int32_t> coordinates, std::vector<int32_t> size);
/// \brief Destructor.
~Crop() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -147,19 +145,18 @@ class Crop final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Decode TensorTransform.
/// \note Decode the input image in RGB mode.
/// \brief Decode the input image in RGB mode.
class Decode final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] rgb A boolean of whether to decode in RGB mode or not.
/// \param[in] rgb A boolean indicating whether to decode the image in RGB mode or not.
explicit Decode(bool rgb = true);
/// \brief Destructor.
~Decode() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -170,12 +167,11 @@ class Decode final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief GaussianBlur TensorTransform.
/// \notes Blur the input image with specified Gaussian kernel.
/// \brief Blur the input image with the specified Gaussian kernel.
class GaussianBlur final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] kernel_size A vector of Gaussian kernel size for width and height. The values must be positive and odd.
/// \param[in] kernel_size A vector of Gaussian kernel size for width and height. The value must be positive and odd.
/// \param[in] sigma A vector of Gaussian kernel standard deviation sigma for width and height. The values must be
/// positive. Using default value 0 means to calculate the sigma according to the kernel size.
GaussianBlur(const std::vector<int32_t> &kernel_size, const std::vector<float> &sigma = {0., 0.});
@ -184,7 +180,7 @@ class GaussianBlur final : public TensorTransform {
~GaussianBlur() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -193,14 +189,13 @@ class GaussianBlur final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Normalize TensorTransform.
/// \note Normalize the input image with respect to mean and standard deviation.
/// \brief Normalize the input image with respect to mean and standard deviation.
class Normalize final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] mean A vector of mean values for each channel, w.r.t channel order.
/// \param[in] mean A vector of mean values for each channel, with respect to channel order.
/// The mean values must be in range [0.0, 255.0].
/// \param[in] std A vector of standard deviations for each channel, w.r.t. channel order.
/// \param[in] std A vector of standard deviations for each channel, with respect to channel order.
/// The standard deviation values must be in range (0.0, 255.0].
Normalize(std::vector<float> mean, std::vector<float> std);
@ -208,7 +203,7 @@ class Normalize final : public TensorTransform {
~Normalize() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -219,25 +214,24 @@ class Normalize final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RandomAffine TensorTransform.
/// \note Applies a Random Affine transformation on input image in RGB or Greyscale mode.
/// \brief Apply a Random Affine transformation on the input image in RGB or Greyscale mode.
class RandomAffine final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] degrees A float vector of size 2, representing the starting and ending degree.
/// \param[in] translate_range A float vector of size 2 or 4, representing percentages of translation on x and y axes.
/// If size is 2, (min_dx, max_dx, 0, 0).
/// if size is 4, (min_dx, max_dx, min_dy, max_dy),
/// If the size is 2, (min_dx, max_dx, 0, 0).
/// If the size is 4, (min_dx, max_dx, min_dy, max_dy),
/// all values are in range [-1, 1].
/// \param[in] scale_range A float vector of size 2, representing the starting and ending scales in the range.
/// \param[in] shear_ranges A float vector of size 2 or 4, representing the starting and ending shear degrees
/// vertically and horizontally.
/// If size is 2, (min_shear_x, max_shear_x, 0, 0),
/// if size is 4, (min_shear_x, max_shear_x, min_shear_y, max_shear_y).
/// If the size is 2, (min_shear_x, max_shear_x, 0, 0),
/// if the size is 4, (min_shear_x, max_shear_x, min_shear_y, max_shear_y).
/// \param[in] interpolation An enum for the mode of interpolation.
/// \param[in] fill_value A vector representing the value to fill the area outside the transform
/// in the output image. If 1 value is provided, it is used for all RGB channels.
/// If 3 values are provided, it is used to fill R, G, B channels respectively.
/// If 3 values are provided, it is used to fill R, G and B channels respectively.
explicit RandomAffine(const std::vector<float_t> &degrees,
const std::vector<float_t> &translate_range = {0.0, 0.0, 0.0, 0.0},
const std::vector<float_t> &scale_range = {1.0, 1.0},
@ -248,7 +242,7 @@ class RandomAffine final : public TensorTransform {
/// \brief Destructor.
~RandomAffine() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -257,14 +251,13 @@ class RandomAffine final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Resize TensorTransform.
/// \note Resize the input image to the given size.
/// \brief Resize the input image to the given size.
class Resize final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] size A vector representing the output size of the resized image.
/// If size is a single value, the image will be resized to this value with
/// the same image aspect ratio. If size has 2 values, it should be (height, width).
/// If the size is a single value, the image will be resized to this value with
/// the same image aspect ratio. If the size has 2 values, it should be (height, width).
/// \param[in] interpolation An enum for the mode of interpolation.
explicit Resize(std::vector<int32_t> size, InterpolationMode interpolation = InterpolationMode::kLinear);
@ -272,7 +265,7 @@ class Resize final : public TensorTransform {
~Resize() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -283,8 +276,7 @@ class Resize final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief ResizePreserveAR TensorTransform.
/// \note Keep the original picture ratio and fill the rest.
/// \brief Keep the original picture ratio and fills the rest.
class ResizePreserveAR final : public TensorTransform {
public:
/// \brief Constructor.
@ -297,7 +289,7 @@ class ResizePreserveAR final : public TensorTransform {
~ResizePreserveAR() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -306,8 +298,7 @@ class ResizePreserveAR final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Rotate TensorTransform.
/// \note Rotate the input image according to parameters.
/// \brief Rotate the input image according to parameters.
class Rotate final : public TensorTransform {
public:
/// \brief Constructor.
@ -318,7 +309,7 @@ class Rotate final : public TensorTransform {
/// \param[in] resample An enum for the mode of interpolation.
/// \param[in] expand A boolean representing whether the image is expanded after rotation.
/// \param[in] center A float vector of size 2, representing the x and y center of rotation.
/// \param[in] fill_value A vector representing the value to fill the area outside the transform.
/// \param[in] fill_value A vector representing the value to fill the area outside the transform
/// in the output image. If 1 value is provided, it is used for all RGB channels.
/// If 3 values are provided, it is used to fill R, G, B channels respectively.
Rotate(float degrees, InterpolationMode resample = InterpolationMode::kNearestNeighbour, bool expand = false,
@ -328,7 +319,7 @@ class Rotate final : public TensorTransform {
~Rotate() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;

13
mindspore/ccsrc/minddata/dataset/kernels/image/resize_with_bbox_op.h
View File

@ -17,6 +17,7 @@
#define MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_RESIZE_WITH_BBOX_OP_H
#include <string>
#include <memory>
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/kernels/image/image_utils.h"
#include "minddata/dataset/kernels/tensor_op.h"
@ -36,9 +37,21 @@ class ResizeWithBBoxOp : public ResizeOp {
void Print(std::ostream &out) const override { out << Name() << ": " << size1_ << " " << size2_; }
// Use in pipeline mode
Status Compute(const TensorRow &input, TensorRow *output) override;
// Use in execute mode
// ResizeWithBBoxOp is inherited from ResizeOp and this function has been overridden by ResizeOp,
// thus we need to change the behaviour back to basic class (TensorOp).
Status Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) override {
return TensorOp::Compute(input, output);
}
std::string Name() const override { return kResizeWithBBoxOp; }
uint32_t NumInput() override { return 2; }
uint32_t NumOutput() override { return 2; }
};
} // namespace dataset
} // namespace mindspore

15
mindspore/ccsrc/minddata/dataset/liteapi/include/datasets.h
View File

@ -437,15 +437,16 @@ class ShuffleDataset : public Dataset {
~ShuffleDataset() = default;
};
/// \brief Function to create a SchemaObj
/// \param[in] schema_file Path of schema file
/// \note This api exists because std::string will constrained by ABI compile macro but char don't.
/// \return Shared pointer to the current schema
/// \brief Function to create a SchemaObj.
/// \param[in] schema_file Path of schema file.
/// \note The reason for using this API is that std::string will be constrained by the
/// compiler option '_GLIBCXX_USE_CXX11_ABI' while char is free of this restriction.
/// \return Shared pointer to the current schema.
std::shared_ptr<SchemaObj> SchemaCharIF(const std::vector<char> &schema_file);
/// \brief Function to create a SchemaObj
/// \param[in] schema_file Path of schema file
/// \return Shared pointer to the current schema
/// \brief Function to create a SchemaObj.
/// \param[in] schema_file Path of schema file.
/// \return Shared pointer to the current schema.
inline std::shared_ptr<SchemaObj> Schema(const std::string &schema_file = "") {
return SchemaCharIF(StringToChar(schema_file));
}

6
mindspore/ccsrc/minddata/dataset/text/kernels/truncate_sequence_pair_op.h
View File

@ -38,6 +38,12 @@ class TruncateSequencePairOp : public TensorOp {
std::string Name() const override { return kTruncateSequencePairOp; }
// Unknown input size
uint32_t NumInput() override { return -1; }
// Unknown output size
uint32_t NumOutput() override { return -1; }
private:
dsize_t max_length_;
};

60
mindspore/dataset/engine/datasets.py
View File

@ -1573,12 +1573,25 @@ class Dataset:
return self.dataset_size
def set_dynamic_columns(self, columns=None):
"""
Set dynamic shape information of source data, it should be set after the pipeline is defined.
Args:
columns (dict): A dict contains shape information of each column in dataset.
The value of shape[i] is :py:obj:`None` indicates that the data length of shape[i] is dynamic.
"""
if not isinstance(columns, dict):
raise TypeError("Pass a dict to set dynamic shape, example: {\"data1\": [16, None, 256]}")
self.dynamic_setting[0] = True
self.dynamic_setting[1] = columns
def dynamic_min_max_shapes(self):
"""
Get minimum and maximum data length of dynamic source data, for graph compilation of ME.
Returns:
lists, min_shapes, max_shapes of source data.
"""
if self.saved_min_shapes is None or self.saved_max_shapes is None:
self.saved_output_shapes, self.saved_min_shapes, self.saved_max_shapes = self._dynamic_output_shapes()
return self.saved_min_shapes, self.saved_max_shapes
@ -1591,7 +1604,7 @@ class Dataset:
lists, dynamic_shapes, min_shapes, max_shapes of source data.
"""
if not self.dynamic_setting[1]:
raise RuntimeError("dynamic_columns is not set, call set_dynamic_columns() first.")
raise RuntimeError("dynamic_columns is not set, call set_dynamic_columns() by final Dataset Op.")
if self.saved_output_shapes is not None and self.saved_min_shapes is not None and \
self.saved_max_shapes is not None:
@ -1648,12 +1661,16 @@ class Dataset:
dynamic_shapes.append(dynamic_shape.tolist())
else:
# Also append fix shape to keep order of column shape
if col in dynamic_columns:
logger.warning("column [" + col + "] has no dynamic shape but set by set_dynamic_columns()")
fix_shape = list(list(shape_set)[0])
max_shapes.append(fix_shape)
min_shapes.append(fix_shape)
dynamic_shapes.append(fix_shape)
if col in dynamic_columns:
logger.warning("column [" + col + "] has no dynamic shape but set by set_dynamic_columns()")
# Set min shape to 1 due to unknown shuffle
min_shapes[-1] = np.where(np.equal(dynamic_columns[col], None), 1, fix_shape).tolist()
# Set dynamic dim to -1 for ME
dynamic_shapes[-1] = np.where(np.equal(dynamic_columns[col], None), -1, fix_shape).tolist()
return dynamic_shapes, min_shapes, max_shapes
def num_classes(self):
@ -3109,7 +3126,7 @@ class ImageFolderDataset(MappableDataset):
decode (bool, optional): Decode the images after reading (default=False).
num_shards (int, optional): Number of shards that the dataset will be divided
into (default=None). When this argument is specified, `num_samples` reflects
the max sample number of per shard.
the maximum sample number of per shard.
shard_id (int, optional): The shard ID within num_shards (default=None). This
argument can only be specified when num_shards is also specified.
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
@ -3229,7 +3246,7 @@ class MnistDataset(MappableDataset):
sampler (Sampler, optional): Object used to choose samples from the
dataset (default=None, expected order behavior shown in the table).
num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).
When this argument is specified, `num_samples` reflects the max sample number of per shard.
When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
shard_id (int, optional): The shard ID within `num_shards` (default=None). This
argument can only be specified when `num_shards` is also specified.
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
@ -3347,7 +3364,7 @@ class MindDataset(MappableDataset):
- Shuffle.INFILE: Keep the file sequence the same but shuffle the data within each file.
num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).
When this argument is specified, 'num_samples' reflects the max sample number of per shard.
When this argument is specified, 'num_samples' reflects the maximum sample number of per shard.
shard_id (int, optional): The shard ID within num_shards (default=None). This
argument can only be specified when num_shards is also specified.
sampler (Sampler, optional): Object used to choose samples from the
@ -3786,8 +3803,8 @@ class GeneratorDataset(MappableDataset):
sampler (Union[Sampler, Iterable], optional): Object used to choose samples from the dataset. Random accessible
input is required (default=None, expected order behavior shown in the table).
num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).
Random accessible input is required. When this argument is specified, `num_samples`s reflects the max sample
number of per shard.
Random accessible input is required. When this argument is specified, `num_samples` reflects the maximum
sample number of per shard.
shard_id (int, optional): The shard ID within num_shards (default=None). This argument must be specified only
when num_shards is also specified. Random accessible input is required.
python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker process. This
@ -4007,12 +4024,13 @@ class TFRecordDataset(SourceDataset):
num_shards (int, optional): Number of shards that the dataset will be divided
into (default=None). When this argument is specified, `num_samples` reflects
the max sample number of per shard.
the maximum sample number of per shard.
shard_id (int, optional): The shard ID within num_shards (default=None). This
argument can only be specified when num_shards is also specified.
shard_equal_rows (bool, optional): Get equal rows for all shards(default=False). If shard_equal_rows
is false, number of rows of each shard may be not equal. This
argument should only be specified when num_shards is also specified.
is false, number of rows of each shard may be not equal, and may lead to a failure in distributed training.
When the number of samples of per TFRecord file are not equal, it is suggested to set to true.
This argument should only be specified when num_shards is also specified.
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
(default=None, which means no cache is used).
@ -4200,7 +4218,7 @@ class Cifar10Dataset(MappableDataset):
dataset (default=None, expected order behavior shown in the table).
num_shards (int, optional): Number of shards that the dataset will be divided
into (default=None). When this argument is specified, `num_samples` reflects
the max sample number of per shard.
the maximum sample number of per shard.
shard_id (int, optional): The shard ID within num_shards (default=None). This
argument can only be specified when num_shards is also specified.
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
@ -4324,7 +4342,7 @@ class Cifar100Dataset(MappableDataset):
dataset (default=None, expected order behavior shown in the table).
num_shards (int, optional): Number of shards that the dataset will be divided
into (default=None). When this argument is specified, 'num_samples' reflects
the max sample number of per shard.
the maximum sample number of per shard.
shard_id (int, optional): The shard ID within num_shards (default=None). This
argument can only be specified when num_shards is also specified.
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
@ -4437,7 +4455,7 @@ class RandomDataset(SourceDataset):
(default=None, expected order behavior shown in the table).
num_shards (int, optional): Number of shards that the dataset will be divided
into (default=None). When this argument is specified, 'num_samples' reflects
the max sample number of per shard.
the maximum sample number of per shard.
shard_id (int, optional): The shard ID within num_shards (default=None). This
argument can only be specified when num_shards is also specified.
"""
@ -4600,7 +4618,7 @@ class VOCDataset(MappableDataset):
(default=None, expected order behavior shown in the table).
num_shards (int, optional): Number of shards that the dataset will be divided
into (default=None). When this argument is specified, `num_samples` reflects
the max sample number of per shard.
the maximum sample number of per shard.
shard_id (int, optional): The shard ID within num_shards (default=None). This
argument can only be specified when num_shards is also specified.
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
@ -4771,7 +4789,7 @@ class CocoDataset(MappableDataset):
:py:obj:`[category_id, dtype=uint32]`, :py:obj:`[iscrowd, dtype=uint32]`.
- task = :py:obj:`Stuff`, output columns: :py:obj:`[image, dtype=uint8]`, :py:obj:`[segmentation,dtype=float32]`, \
:py:obj:`[iscrowd,dtype=uint32]`.
- task = :py:obj:`Keypoint, output columns: :py:obj:`[image, dtype=uint8]`, \
- task = :py:obj:`Keypoint`, output columns: :py:obj:`[image, dtype=uint8]`, \
:py:obj:`[keypoints, dtype=float32]`, :py:obj:`[num_keypoints, dtype=uint32]`.
- task = :py:obj:`Panoptic`, output columns: :py:obj:`[image, dtype=uint8]`, :py:obj:`[bbox, dtype=float32]`, \
:py:obj:`[category_id, dtype=uint32]`, :py:obj:`[iscrowd, dtype=uint32]`, :py:obj:`[area, dtype=uint32]`.
@ -4792,7 +4810,7 @@ class CocoDataset(MappableDataset):
(default=None, expected order behavior shown in the table).
num_shards (int, optional): Number of shards that the dataset will be divided
into (default=None). When this argument is specified, `num_samples` reflects
the max sample number of per shard.
the maximum sample number of per shard.
shard_id (int, optional): The shard ID within num_shards (default=None). This
argument can only be specified when num_shards is also specified.
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
@ -4973,7 +4991,7 @@ class CelebADataset(MappableDataset):
(default=None, will include all images).
num_shards (int, optional): Number of shards that the dataset will be divided
into (default=None). When this argument is specified, `num_samples` reflects
the max sample number of per shard.
the maximum sample number of per shard.
shard_id (int, optional): The shard ID within `num_shards` (default=None). This
argument can only be specified when `num_shards` is also specified.
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
@ -5183,7 +5201,7 @@ class CLUEDataset(SourceDataset):
- Shuffle.FILES: Shuffle files only.
num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).
When this argument is specified, `num_samples` reflects the max sample number of per shard.
When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
shard_id (int, optional): The shard ID within num_shards (default=None). This
argument can only be specified when num_shards is also specified.
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
@ -5268,7 +5286,7 @@ class CSVDataset(SourceDataset):
- Shuffle.FILES: Shuffle files only.
num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).
When this argument is specified, `num_samples` reflects the max sample number of per shard.
When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
shard_id (int, optional): The shard ID within num_shards (default=None). This
argument can only be specified when num_shards is also specified.
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
@ -5323,7 +5341,7 @@ class TextFileDataset(SourceDataset):
- Shuffle.FILES: Shuffle files only.
num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).
When this argument is specified, `num_samples` reflects the max sample number of per shard.
When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
shard_id (int, optional): The shard ID within num_shards (default=None). This
argument can only be specified when num_shards is also specified.
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.

17
tests/ut/cpp/dataset/execute_test.cc
View File

@ -279,4 +279,19 @@ TEST_F(MindDataTestExecute, TestRotate) {
Status rc = transform(image, &image);
EXPECT_EQ(rc, Status::OK());
}
}
TEST_F(MindDataTestExecute, TestResizeWithBBox) {
auto image = ReadFileToTensor("data/dataset/apple.jpg");
std::shared_ptr<TensorTransform> decode_op = std::make_shared<vision::Decode>();
std::shared_ptr<TensorTransform> resizewithbbox_op =
std::make_shared<vision::ResizeWithBBox>(std::vector<int32_t>{250, 500});
// Test Compute(Tensor, Tensor) method of ResizeWithBBox
auto transform = Execute({decode_op, resizewithbbox_op});
// Expect fail since Compute(Tensor, Tensor) is not a valid behaviour for this Op,
// while Compute(TensorRow, TensorRow) is the correct one.
Status rc = transform(image, &image);
EXPECT_FALSE(rc.IsOk());
}

59
tests/ut/python/dataset/test_datasets_get_dynamic_shape.py
View File

@ -24,7 +24,7 @@ def generator0():
def test_get_dynamic_min_max_shapes_0():
logger.info("Test dynamic_min_max_shapes with dynamic shape columns")
logger.info("Test dynamic_min_max_shapes with dynamic shape columns.")
dataset = ds.GeneratorDataset(generator0, ["data1", "data2", "data3"])
@ -47,7 +47,7 @@ def generator1():
def test_get_dynamic_min_max_shapes_1():
logger.info("Test dynamic_min_max_shapes with dynamic shape column and fix shape column")
logger.info("Test dynamic_min_max_shapes with dynamic shape column and fix shape column.")
dataset = ds.GeneratorDataset(generator1, ["data1", "data2"])
@ -66,7 +66,7 @@ def test_get_dynamic_min_max_shapes_1():
def test_get_dynamic_min_max_shapes_2():
logger.info("Test dynamic_min_max_shapes with all dynamic config")
logger.info("Test dynamic_min_max_shapes with setting all columns to dynamic.")
dataset = ds.GeneratorDataset(generator1, ["data1", "data2"])
@ -88,7 +88,7 @@ def generator2():
def test_get_dynamic_min_max_shapes_3():
logger.info("Test dynamic_min_max_shapes with only config dynamic column")
logger.info("Test dynamic_min_max_shapes only config dynamic column.")
dataset = ds.GeneratorDataset(generator2, ["data1", "data2"])
@ -100,14 +100,57 @@ def test_get_dynamic_min_max_shapes_3():
min_shapes, max_shapes = dataset.dynamic_min_max_shapes()
# check result
# column with fix shape will be also appended to shapes list
# column with fixed shape will also be appended to shapes list
np.testing.assert_array_equal(min_shapes, [[16, 1, 83], [5, 5]])
np.testing.assert_array_equal(max_shapes, [[16, 99, 83], [5, 5]])
np.testing.assert_array_equal(dynamic_shapes, [[16, -1, 83], [5, 5]])
def test_get_dynamic_min_max_shapes_4():
logger.info("Test dynamic_min_max_shapes with unexpected column setting")
logger.info("Test dynamic_min_max_shapes with dynamic setting for column with fixed shape.")
dataset = ds.GeneratorDataset(generator2, ["data1", "data2"])
# only dynamic shape is required to config
dataset.set_dynamic_columns(columns={"data1": [16, None, 83], "data2": [None, 5]})
# get dynamic information
dynamic_shapes = dataset.output_shapes()
min_shapes, max_shapes = dataset.dynamic_min_max_shapes()
# check result
# column with fixed shape will also be appended to shapes list
np.testing.assert_array_equal(min_shapes, [[16, 1, 83], [1, 5]])
np.testing.assert_array_equal(max_shapes, [[16, 99, 83], [5, 5]])
np.testing.assert_array_equal(dynamic_shapes, [[16, -1, 83], [-1, 5]])
def test_get_dynamic_min_max_shapes_5():
logger.info("Test dynamic_min_max_shapes with NumpySlicesDataset.")
np_data = [
[[1, 2], [3, 4]],
[[5, 6], [7, 8]],
[[9, 10], [11, 12]],
[[13, 14], [15, 16]]
]
dataset = ds.NumpySlicesDataset(np_data, column_names=["col1"])
dataset.set_dynamic_columns(columns={"col1": [2, None]})
# get dynamic information
dynamic_shapes = dataset.output_shapes()
min_shapes, max_shapes = dataset.dynamic_min_max_shapes()
# check result
# column with fixed shape will also be appended to shapes list
np.testing.assert_array_equal(min_shapes, [[2, 1]])
np.testing.assert_array_equal(max_shapes, [[2, 2]])
np.testing.assert_array_equal(dynamic_shapes, [[2, -1]])
def test_get_dynamic_min_max_shapes_6():
logger.info("Test dynamic_min_max_shapes with unexpected column setting.")
dataset = ds.GeneratorDataset(generator1, ["data1", "data2"])
@ -120,7 +163,7 @@ def test_get_dynamic_min_max_shapes_4():
# dynamic column is not set
dataset.set_dynamic_columns(columns=dict())
dataset.dynamic_min_max_shapes()
assert "dynamic_columns is not set, call set_dynamic_columns() first" in str(info.value)
assert "dynamic_columns is not set, call set_dynamic_columns()" in str(info.value)
with pytest.raises(RuntimeError) as info:
# dynamic column is not set
@ -153,4 +196,6 @@ if __name__ == "__main__":
test_get_dynamic_min_max_shapes_2()
test_get_dynamic_min_max_shapes_3()
test_get_dynamic_min_max_shapes_4()
test_get_dynamic_min_max_shapes_5()
test_get_dynamic_min_max_shapes_6()