mindspore-Ecosystem
/
mindspore
21
0
Fork 27
Code Issues Pull Requests 6 Projects Releases Wiki Activity

correct error message of soft_dvpp ops and fix example of IterSampler

This commit is contained in:
Xiao Tianci 2021-03-16 20:29:14 +08:00
parent a37e157697
commit 4f1dbc6cd5
4 changed files with 40 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
mindspore/ccsrc/minddata/dataset/kernels/image/soft_dvpp/soft_dvpp_decode_random_crop_resize_jpeg_op.cc
View File

@ -1,5 +1,5 @@
/** /**
* Copyright 2020 Huawei Technologies Co., Ltd * Copyright 2020-2021 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -52,13 +52,14 @@ Status SoftDvppDecodeRandomCropResizeJpegOp::Compute(const std::shared_ptr<Tenso
std::shared_ptr<Tensor> *output) { std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output); IO_CHECK(input, output);
if (!IsNonEmptyJPEG(input)) { if (!IsNonEmptyJPEG(input)) {
RETURN_STATUS_UNEXPECTED("SoftDvppDecodeRandomCropResizeJpeg only support process jpeg image."); RETURN_STATUS_UNEXPECTED("SoftDvppDecodeRandomCropResizeJpeg: only support processing raw jpeg image.");
} }
SoftDpCropInfo crop_info; SoftDpCropInfo crop_info;
RETURN_IF_NOT_OK(GetCropInfo(input, &crop_info)); RETURN_IF_NOT_OK(GetCropInfo(input, &crop_info));
try { try {
unsigned char *buffer = const_cast<unsigned char *>(input->GetBuffer()); unsigned char *buffer = const_cast<unsigned char *>(input->GetBuffer());
CHECK_FAIL_RETURN_UNEXPECTED(buffer != nullptr, "The input image buffer is empty."); CHECK_FAIL_RETURN_UNEXPECTED(buffer != nullptr,
"SoftDvppDecodeRandomCropResizeJpeg: the input image buffer is empty.");
SoftDpProcsessInfo info; SoftDpProcsessInfo info;
info.input_buffer = static_cast<uint8_t *>(buffer); info.input_buffer = static_cast<uint8_t *>(buffer);
info.input_buffer_size = input->SizeInBytes(); info.input_buffer_size = input->SizeInBytes();
@ -69,14 +70,14 @@ Status SoftDvppDecodeRandomCropResizeJpegOp::Compute(const std::shared_ptr<Tenso
info.output_buffer_size = target_width_ * target_height_ * 3; info.output_buffer_size = target_width_ * target_height_ * 3;
info.is_v_before_u = true; info.is_v_before_u = true;
int ret = DecodeAndCropAndResizeJpeg(&info, crop_info); int ret = DecodeAndCropAndResizeJpeg(&info, crop_info);
std::string error_info("Soft dvpp DecodeAndResizeJpeg failed with return code: "); std::string error_info("SoftDvppDecodeRandomCropResizeJpeg: failed with return code: ");
error_info += std::to_string(ret); error_info += std::to_string(ret) + ", please check the log information for more details.";
CHECK_FAIL_RETURN_UNEXPECTED(ret == 0, error_info); CHECK_FAIL_RETURN_UNEXPECTED(ret == 0, error_info);
std::shared_ptr<CVTensor> cv_tensor = nullptr; std::shared_ptr<CVTensor> cv_tensor = nullptr;
RETURN_IF_NOT_OK(CVTensor::CreateFromMat(out_rgb_img, &cv_tensor)); RETURN_IF_NOT_OK(CVTensor::CreateFromMat(out_rgb_img, &cv_tensor));
*output = std::static_pointer_cast<Tensor>(cv_tensor); *output = std::static_pointer_cast<Tensor>(cv_tensor);
} catch (const cv::Exception &e) { } catch (const cv::Exception &e) {
std::string error = "Error in SoftDvppDecodeRandomCropResizeJpegOp:" + std::string(e.what()); std::string error = "SoftDvppDecodeRandomCropResizeJpeg:" + std::string(e.what());
RETURN_STATUS_UNEXPECTED(error); RETURN_STATUS_UNEXPECTED(error);
} }
return Status::OK(); return Status::OK();

18
mindspore/ccsrc/minddata/dataset/kernels/image/soft_dvpp/soft_dvpp_decode_resize_jpeg_op.cc
View File

@ -1,5 +1,5 @@
/** /**
* Copyright 2020 Huawei Technologies Co., Ltd * Copyright 2020-2021 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -28,11 +28,11 @@ namespace dataset {
Status SoftDvppDecodeResizeJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) { Status SoftDvppDecodeResizeJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output); IO_CHECK(input, output);
if (!IsNonEmptyJPEG(input)) { if (!IsNonEmptyJPEG(input)) {
RETURN_STATUS_UNEXPECTED("SoftDvppDecodeReiszeJpegOp only support process jpeg image."); RETURN_STATUS_UNEXPECTED("SoftDvppDecodeReiszeJpeg: only support processing raw jpeg image.");
} }
try { try {
unsigned char *buffer = const_cast<unsigned char *>(input->GetBuffer()); unsigned char *buffer = const_cast<unsigned char *>(input->GetBuffer());
CHECK_FAIL_RETURN_UNEXPECTED(buffer != nullptr, "The input image buffer is empty."); CHECK_FAIL_RETURN_UNEXPECTED(buffer != nullptr, "SoftDvppDecodeReiszeJpeg: the input image buffer is empty.");
SoftDpProcsessInfo info; SoftDpProcsessInfo info;
info.input_buffer = static_cast<uint8_t *>(buffer); info.input_buffer = static_cast<uint8_t *>(buffer);
info.input_buffer_size = input->SizeInBytes(); info.input_buffer_size = input->SizeInBytes();
@ -43,11 +43,11 @@ Status SoftDvppDecodeResizeJpegOp::Compute(const std::shared_ptr<Tensor> &input,
if (target_width_ == 0) { if (target_width_ == 0) {
if (input_h < input_w) { if (input_h < input_w) {
CHECK_FAIL_RETURN_UNEXPECTED(input_h != 0, "The input height is 0"); CHECK_FAIL_RETURN_UNEXPECTED(input_h != 0, "SoftDvppDecodeReiszeJpeg: the input height is 0.");
info.output_height = target_height_; info.output_height = target_height_;
info.output_width = static_cast<int>(std::lround(static_cast<float>(input_w) / input_h * info.output_height)); info.output_width = static_cast<int>(std::lround(static_cast<float>(input_w) / input_h * info.output_height));
} else { } else {
CHECK_FAIL_RETURN_UNEXPECTED(input_w != 0, "The input width is 0"); CHECK_FAIL_RETURN_UNEXPECTED(input_w != 0, "SoftDvppDecodeReiszeJpeg: the input width is 0.");
info.output_width = target_height_; info.output_width = target_height_;
info.output_height = static_cast<int>(std::lround(static_cast<float>(input_h) / input_w * info.output_width)); info.output_height = static_cast<int>(std::lround(static_cast<float>(input_h) / input_w * info.output_width));
} }
@ -62,14 +62,14 @@ Status SoftDvppDecodeResizeJpegOp::Compute(const std::shared_ptr<Tensor> &input,
info.is_v_before_u = true; info.is_v_before_u = true;
int ret = DecodeAndResizeJpeg(&info); int ret = DecodeAndResizeJpeg(&info);
std::string error_info("Soft dvpp DecodeAndResizeJpeg failed with return code: "); std::string error_info("SoftDvppDecodeReiszeJpeg: failed with return code: ");
error_info += std::to_string(ret); error_info += std::to_string(ret) + ", please check the log information for more details.";
CHECK_FAIL_RETURN_UNEXPECTED(ret == 0, error_info); CHECK_FAIL_RETURN_UNEXPECTED(ret == 0, error_info);
std::shared_ptr<CVTensor> cv_tensor = nullptr; std::shared_ptr<CVTensor> cv_tensor = nullptr;
RETURN_IF_NOT_OK(CVTensor::CreateFromMat(out_rgb_img, &cv_tensor)); RETURN_IF_NOT_OK(CVTensor::CreateFromMat(out_rgb_img, &cv_tensor));
*output = std::static_pointer_cast<Tensor>(cv_tensor); *output = std::static_pointer_cast<Tensor>(cv_tensor);
} catch (const cv::Exception &e) { } catch (const cv::Exception &e) {
std::string error = "Error in SoftDvppDecodeResizeJpegOp:" + std::string(e.what()); std::string error = "SoftDvppDecodeResizeJpeg:" + std::string(e.what());
RETURN_STATUS_UNEXPECTED(error); RETURN_STATUS_UNEXPECTED(error);
} }
return Status::OK(); return Status::OK();
@ -82,7 +82,7 @@ Status SoftDvppDecodeResizeJpegOp::OutputShape(const std::vector<TensorShape> &i
TensorShape out({-1, -1, 3}); // we don't know what is output image size, but we know it should be 3 channels TensorShape out({-1, -1, 3}); // we don't know what is output image size, but we know it should be 3 channels
if (inputs[0].Rank() == 1) outputs.emplace_back(out); if (inputs[0].Rank() == 1) outputs.emplace_back(out);
if (!outputs.empty()) return Status::OK(); if (!outputs.empty()) return Status::OK();
return Status(StatusCode::kMDUnexpectedError, "Input has a wrong shape"); return Status(StatusCode::kMDUnexpectedError, "SoftDvppDecodeReiszeJpeg: input has a wrong shape.");
} }
} // namespace dataset } // namespace dataset

16
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/vision_ir.cc
View File

@ -1512,6 +1512,14 @@ SoftDvppDecodeRandomCropResizeJpegOperation::SoftDvppDecodeRandomCropResizeJpegO
Status SoftDvppDecodeRandomCropResizeJpegOperation::ValidateParams() { Status SoftDvppDecodeRandomCropResizeJpegOperation::ValidateParams() {
// size // size
RETURN_IF_NOT_OK(ValidateVectorSize("SoftDvppDecodeRandomCropResizeJpeg", size_)); RETURN_IF_NOT_OK(ValidateVectorSize("SoftDvppDecodeRandomCropResizeJpeg", size_));
for (int32_t i = 0; i < size_.size(); i++) {
if (size_[i] % 2 == 1) {
std::string err_msg = "SoftDvppDecodeRandomCropResizeJpeg: size[" + std::to_string(i) +
"] must be even values, got: " + std::to_string(size_[i]);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
}
// scale // scale
RETURN_IF_NOT_OK(ValidateVectorScale("SoftDvppDecodeRandomCropResizeJpeg", scale_)); RETURN_IF_NOT_OK(ValidateVectorScale("SoftDvppDecodeRandomCropResizeJpeg", scale_));
// ratio // ratio
@ -1554,6 +1562,14 @@ SoftDvppDecodeResizeJpegOperation::SoftDvppDecodeResizeJpegOperation(std::vector
Status SoftDvppDecodeResizeJpegOperation::ValidateParams() { Status SoftDvppDecodeResizeJpegOperation::ValidateParams() {
RETURN_IF_NOT_OK(ValidateVectorSize("SoftDvppDecodeResizeJpeg", size_)); RETURN_IF_NOT_OK(ValidateVectorSize("SoftDvppDecodeResizeJpeg", size_));
for (int32_t i = 0; i < size_.size(); i++) {
if (size_[i] % 2 == 1) {
std::string err_msg = "SoftDvppDecodeResizeJpeg: size[" + std::to_string(i) +
"] must be even values, got: " + std::to_string(size_[i]);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
}
return Status::OK(); return Status::OK();
} }

16
mindspore/dataset/engine/samplers.py
View File

@ -25,6 +25,7 @@ import mindspore._c_dataengine as cde
import mindspore.dataset as ds import mindspore.dataset as ds
from ..core import validator_helpers as validator from ..core import validator_helpers as validator
def select_sampler(num_samples, input_sampler, shuffle, num_shards, shard_id): def select_sampler(num_samples, input_sampler, shuffle, num_shards, shard_id):
""" """
Create sampler based on user input. Create sampler based on user input.
@ -615,8 +616,8 @@ class SubsetSampler(BuiltinSampler):
Examples: Examples:
>>> indices = [0, 1, 2, 3, 4, 5] >>> indices = [0, 1, 2, 3, 4, 5]
>>> >>>
>>> # creates a SubsetRandomSampler, will sample from the provided indices >>> # creates a SubsetSampler, will sample from the provided indices
>>> sampler = ds.SubsetRandomSampler(indices) >>> sampler = ds.SubsetSampler(indices)
>>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir, >>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir,
... num_parallel_workers=8, ... num_parallel_workers=8,
... sampler=sampler) ... sampler=sampler)
@ -694,7 +695,7 @@ class SubsetRandomSampler(SubsetSampler):
Samples the elements randomly from a sequence of indices. Samples the elements randomly from a sequence of indices.
Args: Args:
indices (list[int]): A sequence of indices. indices (Any iterable python object but string): A sequence of indices.
num_samples (int, optional): Number of elements to sample (default=None, all elements). num_samples (int, optional): Number of elements to sample (default=None, all elements).
Examples: Examples:
@ -740,17 +741,16 @@ class IterSampler(Sampler):
num_samples (int, optional): Number of elements to sample (default=None, all elements). num_samples (int, optional): Number of elements to sample (default=None, all elements).
Examples: Examples:
>>> class MySampler(): >>> class MySampler:
>>> def __iter__(self): ... def __iter__(self):
>>> for i in range(99, -1, -1): ... for i in range(99, -1, -1):
>>> yield i ... yield i
>>> # creates an IterSampler >>> # creates an IterSampler
>>> sampler = ds.IterSampler(sampler=MySampler()) >>> sampler = ds.IterSampler(sampler=MySampler())
>>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir, >>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir,
... num_parallel_workers=8, ... num_parallel_workers=8,
... sampler=sampler) ... sampler=sampler)
""" """
def __init__(self, sampler, num_samples=None): def __init__(self, sampler, num_samples=None):