p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!9175 fix a bug in random resize crop op and some lint 

From: @tiancixiao
Reviewed-by: @liucunwei,@jonyguo,@liucunwei
Signed-off-by: @liucunwei
This commit is contained in:
mindspore-ci-bot 2020-11-30 20:03:51 +08:00 committed by Gitee
parent f0671ce513 12da8717b7
commit 762793cb8f
7 changed files with 90 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
mindspore/ccsrc/minddata/dataset/api/config.cc
View File

@ -29,7 +29,7 @@ std::shared_ptr<ConfigManager> _config = GlobalContext::config_manager();
// Function to set the seed to be used in any random generator
bool set_seed(int32_t seed) {
if (seed < 0 || seed > INT32_MAX) {
if (seed < 0) {
MS_LOG(ERROR) << "Seed given is not within the required range: " << seed;
return false;
}
@ -42,7 +42,7 @@ uint32_t get_seed() { return _config->seed(); }
// Function to set the number of rows to be prefetched
bool set_prefetch_size(int32_t prefetch_size) {
if (prefetch_size <= 0 || prefetch_size > INT32_MAX) {
if (prefetch_size <= 0) {
MS_LOG(ERROR) << "Prefetch size given is not within the required range: " << prefetch_size;
return false;
}
@ -55,7 +55,7 @@ int32_t get_prefetch_size() { return _config->op_connector_size(); }
// Function to set the default number of parallel workers
bool set_num_parallel_workers(int32_t num_parallel_workers) {
if (num_parallel_workers <= 0 || num_parallel_workers > INT32_MAX) {
if (num_parallel_workers <= 0) {
MS_LOG(ERROR) << "Number of parallel workers given is not within the required range: " << num_parallel_workers;
return false;
}
@ -68,7 +68,7 @@ int32_t get_num_parallel_workers() { return _config->num_parallel_workers(); }
// Function to set the default interval (in milliseconds) for monitor sampling
bool set_monitor_sampling_interval(int32_t interval) {
if (interval <= 0 || interval > INT32_MAX) {
if (interval <= 0) {
MS_LOG(ERROR) << "Interval given is not within the required range: " << interval;
return false;
}
@ -81,7 +81,7 @@ int32_t get_monitor_sampling_interval() { return _config->monitor_sampling_inter
// Function to set the default timeout (in seconds) for DSWaitedCallback
bool set_callback_timeback(int32_t timeout) {
if (timeout <= 0 || timeout > INT32_MAX) {
if (timeout <= 0) {
MS_LOG(ERROR) << "Timeout given is not within the required range: " << timeout;
return false;
}

106
mindspore/ccsrc/minddata/dataset/api/vision.cc
View File

@ -805,12 +805,17 @@ Status RandomAffineOperation::ValidateParams() {
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
for (int32_t i = 0; i < scale_range_.size(); ++i) {
if (scale_range_[i] <= 0) {
std::string err_msg = "RandomAffine: scale must be greater than 0, got: " + std::to_string(fill_value_[i]);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_range_[0] < 0) {
std::string err_msg =
"RandomAffine: min scale range must be greater than or equal to 0, got: " + std::to_string(scale_range_[0]);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_range_[1] <= 0) {
std::string err_msg =
"RandomAffine: max scale range must be greater than 0, got: " + std::to_string(scale_range_[1]);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_range_[0] > scale_range_[1]) {
std::string err_msg =
@ -1111,18 +1116,17 @@ Status RandomCropDecodeResizeOperation::ValidateParams() {
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
for (int32_t i = 0; i < scale_.size(); ++i) {
if (scale_[i] < 0) {
std::string err_msg = "RandomCropDecodeResize: invalid scale, scale must be greater than or equal to 0, got: " +
std::to_string(scale_[i]);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_[i] == INT_MAX) {
std::string err_msg = "RandomCropDecodeResize: invalid scale, scale too large, got: " + std::to_string(scale_[i]);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_[0] < 0) {
std::string err_msg = "RandomCropDecodeResize: invalid scale, min scale must be greater than or equal to 0, got: " +
std::to_string(scale_[0]);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_[1] <= 0) {
std::string err_msg =
"RandomCropDecodeResize: invalid scale, max scale must be greater than 0, got: " + std::to_string(scale_[1]);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_[0] > scale_[1]) {
std::string err_msg = "RandomCropDecodeResize: scale should be in (min,max) format. Got (max,min).";
@ -1136,14 +1140,9 @@ Status RandomCropDecodeResizeOperation::ValidateParams() {
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
for (int32_t i = 0; i < ratio_.size(); ++i) {
if (ratio_[i] < 0) {
std::string err_msg = "RandomCropDecodeResize: invalid ratio, ratio must be greater than or equal to 0, got: " +
std::to_string(ratio_[i]);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (ratio_[i] == INT_MAX) {
std::string err_msg = "RandomCropDecodeResize: invalid ratio, ratio too large, got: " + std::to_string(ratio_[i]);
if (ratio_[i] <= 0) {
std::string err_msg =
"RandomCropDecodeResize: invalid ratio, ratio must be greater than 0, got: " + std::to_string(ratio_[i]);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
@ -1423,9 +1422,15 @@ Status RandomResizedCropOperation::ValidateParams() {
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_[0] < 0 || scale_[1] < 0) {
std::string err_msg = "RandomResizedCrop: scale must be greater than or equal to 0.";
MS_LOG(ERROR) << "RandomResizedCrop: scale must be greater than or equal to 0, got: " << scale_;
if (scale_[0] < 0) {
std::string err_msg = "RandomResizedCrop: min scale must be greater than or equal to 0.";
MS_LOG(ERROR) << "RandomResizedCrop: min scale must be greater than or equal to 0, got: " +
std::to_string(scale_[0]);
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_[1] <= 0) {
std::string err_msg = "RandomResizedCrop: max scale must be greater than 0.";
MS_LOG(ERROR) << "RandomResizedCrop: max scale must be greater than 0, got: " + std::to_string(scale_[1]);
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_[1] < scale_[0]) {
@ -1441,9 +1446,9 @@ Status RandomResizedCropOperation::ValidateParams() {
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (ratio_[0] < 0 || ratio_[1] < 0) {
std::string err_msg = "RandomResizedCrop: ratio must be greater than or equal to 0.";
MS_LOG(ERROR) << "RandomResizedCrop: ratio must be greater than or equal to 0, got: " << ratio_;
if (ratio_[0] <= 0 || ratio_[1] <= 0) {
std::string err_msg = "RandomResizedCrop: ratio must be greater than 0.";
MS_LOG(ERROR) << "RandomResizedCrop: ratio must be greater than 0, got: " << ratio_;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (ratio_[1] < ratio_[0]) {
@ -1502,9 +1507,15 @@ Status RandomResizedCropWithBBoxOperation::ValidateParams() {
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_[0] < 0 || scale_[1] < 0) {
std::string err_msg = "RandomResizedCropWithBBox: scale must be greater than or equal to 0.";
MS_LOG(ERROR) << "RandomResizedCropWithBBox: scale must be greater than or equal to 0, got: " << scale_;
if (scale_[0] < 0) {
std::string err_msg = "RandomResizedCropWithBBox: min scale must be greater than or equal to 0.";
MS_LOG(ERROR) << "RandomResizedCropWithBBox: min scale must be greater than or equal to 0, got: " +
std::to_string(scale_[0]);
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_[1] <= 0) {
std::string err_msg = "RandomResizedCropWithBBox: max scale must be greater than 0.";
MS_LOG(ERROR) << "RandomResizedCropWithBBox: max scale must be greater than 0, got: " + std::to_string(scale_[1]);
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_[1] < scale_[0]) {
@ -1520,9 +1531,9 @@ Status RandomResizedCropWithBBoxOperation::ValidateParams() {
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (ratio_[0] < 0 || ratio_[1] < 0) {
std::string err_msg = "RandomResizedCropWithBBox: ratio must be greater than or equal to 0.";
MS_LOG(ERROR) << "RandomResizedCropWithBBox: ratio must be greater than or equal to 0, got: " << ratio_;
if (ratio_[0] <= 0 || ratio_[1] <= 0) {
std::string err_msg = "RandomResizedCropWithBBox: ratio must be greater than 0.";
MS_LOG(ERROR) << "RandomResizedCropWithBBox: ratio must be greater than 0, got: " << ratio_;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (ratio_[1] < ratio_[0]) {
@ -1884,9 +1895,16 @@ Status SoftDvppDecodeRandomCropResizeJpegOperation::ValidateParams() {
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_[0] < 0 || scale_[1] < 0) {
std::string err_msg = "SoftDvppDecodeRandomCropResizeJpeg: scale must be greater than or equal to 0.";
MS_LOG(ERROR) << "SoftDvppDecodeRandomCropResizeJpeg: scale must be greater than or equal to 0, got: " << scale_;
if (scale_[0] < 0) {
std::string err_msg = "SoftDvppDecodeRandomCropResizeJpeg: min scale must be greater than or equal to 0.";
MS_LOG(ERROR) << "SoftDvppDecodeRandomCropResizeJpeg: min scale must be greater than or equal to 0, got: " +
std::to_string(scale_[0]);
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_[1] <= 0) {
std::string err_msg = "SoftDvppDecodeRandomCropResizeJpeg: max scale must be greater than 0.";
MS_LOG(ERROR) << "SoftDvppDecodeRandomCropResizeJpeg: max scale must be greater than 0, got: " +
std::to_string(scale_[1]);
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (scale_[1] < scale_[0]) {
@ -1902,9 +1920,9 @@ Status SoftDvppDecodeRandomCropResizeJpegOperation::ValidateParams() {
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (ratio_[0] < 0 || ratio_[1] < 0) {
std::string err_msg = "SoftDvppDecodeRandomCropResizeJpeg: ratio must be greater than or equal to 0.";
MS_LOG(ERROR) << "SoftDvppDecodeRandomCropResizeJpeg: ratio must be greater than or equal to 0, got: " << ratio_;
if (ratio_[0] <= 0 || ratio_[1] <= 0) {
std::string err_msg = "SoftDvppDecodeRandomCropResizeJpeg: ratio must be greater than 0.";
MS_LOG(ERROR) << "SoftDvppDecodeRandomCropResizeJpeg: ratio must be greater than 0, got: " << ratio_;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (ratio_[1] < ratio_[0]) {

2
mindspore/dataset/engine/datasets.py
View File

@ -246,7 +246,7 @@ class Dataset:
>>> # Create a dataset where every 100 rows is combined into a batch
>>> # and drops the last incomplete batch if there is one.
>>> column_names = ["col1", "col2"]
>>> buket_boundaries = [5, 10]
>>> bucket_boundaries = [5, 10]
>>> bucket_batch_sizes = [5, 1, 1]
>>> element_length_function = (lambda col1, col2: max(len(col1), len(col2)))
>>>

18
mindspore/dataset/vision/validators.py
View File

@ -221,13 +221,15 @@ def check_size_scale_ration_max_attempts_paras(size, scale, ratio, max_attempts)
if scale is not None:
type_check(scale, (tuple,), "scale")
type_check_list(scale, (float, int), "scale")
check_range(scale, [0, FLOAT_MAX_INTEGER])
if scale[0] > scale[1]:
raise ValueError("scale should be in (min,max) format. Got (max,min).")
check_range(scale, [0, FLOAT_MAX_INTEGER])
check_positive(scale[1], "scale[1]")
if ratio is not None:
type_check(ratio, (tuple,), "ratio")
type_check_list(ratio, (float, int), "ratio")
check_range(ratio, [0, FLOAT_MAX_INTEGER])
check_positive(ratio[0], "ratio[0]")
check_positive(ratio[1], "ratio[1]")
if ratio[0] > ratio[1]:
raise ValueError("ratio should be in (min,max) format. Got (max,min).")
if max_attempts is not None:
@ -434,8 +436,14 @@ def check_random_erasing(method):
[prob, scale, ratio, value, inplace, max_attempts], _ = parse_user_args(method, *args, **kwargs)
check_value(prob, [0., 1.], "prob")
if scale[0] > scale[1]:
raise ValueError("scale should be in (min,max) format. Got (max,min).")
check_range(scale, [0, FLOAT_MAX_INTEGER])
check_range(ratio, [0, FLOAT_MAX_INTEGER])
check_positive(scale[1], "scale[1]")
check_positive(ratio[0], "ratio[0]")
check_positive(ratio[1], "ratio[1]")
if ratio[0] > ratio[1]:
raise ValueError("ratio should be in (min,max) format. Got (max,min).")
check_erasing_value(value)
type_check(inplace, (bool,), "inplace")
check_value(max_attempts, (1, FLOAT_MAX_INTEGER))
@ -501,10 +509,10 @@ def check_random_affine(method):
type_check(scale, (tuple, list), "scale")
type_check_list(scale, (int, float), "scale")
if len(scale) == 2:
for i, s in enumerate(scale):
check_positive(s, "scale[{}]".format(i))
if scale[0] > scale[1]:
raise ValueError("Input scale[1] must be equal to or greater than scale[0].")
check_range(scale, [0, FLOAT_MAX_INTEGER])
check_positive(scale[1], "scale[1]")
else:
raise TypeError("scale should be a list or tuple of length 2.")

6
tests/ut/python/dataset/test_random_affine.py
View File

@ -234,14 +234,14 @@ def test_random_affine_exception_translation_range():
def test_random_affine_exception_scale_value():
"""
Test RandomAffine: scale is not positive, expected to raise ValueError
Test RandomAffine: scale is not valid, expected to raise ValueError
"""
logger.info("test_random_affine_exception_scale_value")
try:
_ = py_vision.RandomAffine(degrees=15, scale=(0.0, 1.1))
_ = py_vision.RandomAffine(degrees=15, scale=(0.0, 0.0))
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert str(e) == "Input scale[0] must be greater than 0."
assert str(e) == "Input scale[1] must be greater than 0."
try:
_ = py_vision.RandomAffine(degrees=15, scale=(2.0, 1.1))

8
tests/ut/python/dataset/test_random_crop_and_resize.py
View File

@ -255,7 +255,7 @@ def test_random_crop_and_resize_04_c():
data = data.map(operations=random_crop_and_resize_op, input_columns=["image"])
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Input is not within the required interval of (0 to 16777216)." in str(e)
assert "scale should be in (min,max) format. Got (max,min)." in str(e)
def test_random_crop_and_resize_04_py():
@ -278,7 +278,7 @@ def test_random_crop_and_resize_04_py():
data = data.map(operations=transform, input_columns=["image"])
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Input is not within the required interval of (0 to 16777216)." in str(e)
assert "scale should be in (min,max) format. Got (max,min)." in str(e)
def test_random_crop_and_resize_05_c():
@ -298,7 +298,7 @@ def test_random_crop_and_resize_05_c():
data = data.map(operations=random_crop_and_resize_op, input_columns=["image"])
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Input is not within the required interval of (0 to 16777216)." in str(e)
assert "ratio should be in (min,max) format. Got (max,min)." in str(e)
def test_random_crop_and_resize_05_py():
@ -321,7 +321,7 @@ def test_random_crop_and_resize_05_py():
data = data.map(operations=transform, input_columns=["image"])
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Input is not within the required interval of (0 to 16777216)." in str(e)
assert "ratio should be in (min,max) format. Got (max,min)." in str(e)
def test_random_crop_and_resize_comp(plot=False):

4
tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py
View File

@ -161,7 +161,7 @@ def test_random_resized_crop_with_bbox_op_invalid_c():
except ValueError as err:
logger.info("Got an exception in DE: {}".format(str(err)))
assert "Input is not within the required interval of (0 to 16777216)." in str(err)
assert "scale should be in (min,max) format. Got (max,min)." in str(err)
def test_random_resized_crop_with_bbox_op_invalid2_c():
@ -186,7 +186,7 @@ def test_random_resized_crop_with_bbox_op_invalid2_c():
except ValueError as err:
logger.info("Got an exception in DE: {}".format(str(err)))
assert "Input is not within the required interval of (0 to 16777216)." in str(err)
assert "ratio should be in (min,max) format. Got (max,min)." in str(err)
def test_random_resized_crop_with_bbox_op_bad_c():