From 416ad23e9d5bb925f6f2161edfc313d8a097873f Mon Sep 17 00:00:00 2001 From: luoyang Date: Tue, 9 Feb 2021 15:52:53 +0800 Subject: [PATCH] fix resize & randomcoloradjust fix sampler --- .../ccsrc/minddata/dataset/api/samplers.cc | 34 ++++--- .../dataset/kernels/image/image_utils.cc | 3 - mindspore/dataset/engine/samplers.py | 94 +++++++++++++++---- mindspore/dataset/vision/c_transforms.py | 11 ++- mindspore/dataset/vision/validators.py | 9 +- .../dataset/test_datasets_imagefolder.py | 4 +- tests/ut/python/dataset/test_sampler.py | 8 +- 7 files changed, 115 insertions(+), 48 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/api/samplers.cc b/mindspore/ccsrc/minddata/dataset/api/samplers.cc index fab24cb1f94..e8d7db6e0e1 100644 --- a/mindspore/ccsrc/minddata/dataset/api/samplers.cc +++ b/mindspore/ccsrc/minddata/dataset/api/samplers.cc @@ -173,21 +173,23 @@ DistributedSamplerObj::DistributedSamplerObj(int64_t num_shards, int64_t shard_i Status DistributedSamplerObj::ValidateParams() { if (num_shards_ <= 0) { - RETURN_STATUS_UNEXPECTED("DistributedSampler: invalid num_shards: " + std::to_string(num_shards_)); + RETURN_STATUS_UNEXPECTED("DistributedSampler: num_shards must be greater than 0, but got: " + + std::to_string(num_shards_)); } if (shard_id_ < 0 || shard_id_ >= num_shards_) { - RETURN_STATUS_UNEXPECTED("DistributedSampler: invalid input, shard_id: " + std::to_string(shard_id_) + - ", num_shards: " + std::to_string(num_shards_)); + RETURN_STATUS_UNEXPECTED("DistributedSampler: shard_id must be in range [0, " + std::to_string(num_shards_) + + "), but got: " + std::to_string(shard_id_)); } if (num_samples_ < 0) { - RETURN_STATUS_UNEXPECTED("DistributedSampler: invalid num_samples: " + std::to_string(num_samples_)); + RETURN_STATUS_UNEXPECTED("DistributedSampler: num_samples must be greater than or equal to 0, but got: " + + std::to_string(num_samples_)); } if (offset_ > num_shards_) { - RETURN_STATUS_UNEXPECTED("DistributedSampler: invalid offset: " + std::to_string(offset_) + - ", which should be no more than num_shards: " + std::to_string(num_shards_)); + RETURN_STATUS_UNEXPECTED("DistributedSampler: offset must be no more than num_shards(" + + std::to_string(num_shards_) + "), but got: " + std::to_string(offset_)); } return Status::OK(); @@ -237,11 +239,12 @@ PKSamplerObj::PKSamplerObj(int64_t num_val, bool shuffle, int64_t num_samples) Status PKSamplerObj::ValidateParams() { if (num_val_ <= 0) { - RETURN_STATUS_UNEXPECTED("PKSampler: invalid num_val: " + std::to_string(num_val_)); + RETURN_STATUS_UNEXPECTED("PKSampler: num_val must be greater than 0, but got: " + std::to_string(num_val_)); } if (num_samples_ < 0) { - RETURN_STATUS_UNEXPECTED("PKSampler: invalid num_samples: " + std::to_string(num_samples_)); + RETURN_STATUS_UNEXPECTED("PKSampler: num_samples must be greater than or equal to 0, but got: " + + std::to_string(num_samples_)); } return Status::OK(); } @@ -334,7 +337,8 @@ RandomSamplerObj::RandomSamplerObj(bool replacement, int64_t num_samples, bool r Status RandomSamplerObj::ValidateParams() { if (num_samples_ < 0) { - RETURN_STATUS_UNEXPECTED("RandomSampler: invalid num_samples: " + std::to_string(num_samples_)); + RETURN_STATUS_UNEXPECTED("RandomSampler: num_samples must be greater than or equal to 0, but got: " + + std::to_string(num_samples_)); } return Status::OK(); } @@ -381,11 +385,13 @@ SequentialSamplerObj::SequentialSamplerObj(int64_t start_index, int64_t num_samp Status SequentialSamplerObj::ValidateParams() { if (num_samples_ < 0) { - RETURN_STATUS_UNEXPECTED("SequentialSampler: invalid num_samples: " + std::to_string(num_samples_)); + RETURN_STATUS_UNEXPECTED("SequentialSampler: num_samples must be greater than or equal to 0, but got: " + + std::to_string(num_samples_)); } if (start_index_ < 0) { - RETURN_STATUS_UNEXPECTED("SequentialSampler: invalid start_index: " + std::to_string(start_index_)); + RETURN_STATUS_UNEXPECTED("SequentialSampler: start_index_ must be greater than or equal to 0, but got: " + + std::to_string(start_index_)); } return Status::OK(); @@ -431,7 +437,8 @@ SubsetSamplerObj::SubsetSamplerObj(std::vector indices, int64_t num_sam Status SubsetSamplerObj::ValidateParams() { if (num_samples_ < 0) { - RETURN_STATUS_UNEXPECTED("SubsetRandomSampler: invalid num_samples: " + std::to_string(num_samples_)); + RETURN_STATUS_UNEXPECTED("SubsetRandomSampler: num_samples must be greater than or equal to 0, but got: " + + std::to_string(num_samples_)); } return Status::OK(); @@ -530,7 +537,8 @@ Status WeightedRandomSamplerObj::ValidateParams() { RETURN_STATUS_UNEXPECTED("WeightedRandomSampler: elements of weights vector must not be all zero"); } if (num_samples_ < 0) { - RETURN_STATUS_UNEXPECTED("WeightedRandomSampler: invalid num_samples: " + std::to_string(num_samples_)); + RETURN_STATUS_UNEXPECTED("WeightedRandomSampler: num_samples must be greater than or equal to 0, but got: " + + std::to_string(num_samples_)); } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc index c61e8f3f3c4..0b7b73042ed 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc @@ -105,9 +105,6 @@ Status Resize(const std::shared_ptr &input, std::shared_ptr *out if (input_cv->Rank() != 3 && input_cv->Rank() != 2) { RETURN_STATUS_UNEXPECTED("Resize: input tensor is not in shape of or "); } - if (input_cv->shape()[2] != 3 && input_cv->shape()[2] != 1) { - RETURN_STATUS_UNEXPECTED("Resize: channel of input tesnor is not in 1 or 3."); - } cv::Mat in_image = input_cv->mat(); // resize image too large or too small diff --git a/mindspore/dataset/engine/samplers.py b/mindspore/dataset/engine/samplers.py index 42813406598..644fbbc6dba 100644 --- a/mindspore/dataset/engine/samplers.py +++ b/mindspore/dataset/engine/samplers.py @@ -328,21 +328,32 @@ class DistributedSampler(BuiltinSampler): ... sampler=sampler) Raises: - ValueError: If num_shards is not positive. - ValueError: If shard_id is smaller than 0 or equal to num_shards or larger than num_shards. - ValueError: If shuffle is not a boolean value. - ValueError: If offset is greater than num_shards. + TypeError: If num_shards is not an integer value. + TypeError: If shard_id is not an integer value. + TypeError: If shuffle is not a boolean value. + TypeError: If num_samples is not an integer value. + TypeError: If offset is not an integer value. + RuntimeError: If num_shards is not a positive value. + RuntimeError: If shard_id is smaller than 0 or equal to num_shards or larger than num_shards. + RuntimeError: If num_samples is a negative value. + RuntimeError: If offset is greater than num_shards. """ def __init__(self, num_shards, shard_id, shuffle=True, num_samples=None, offset=-1): if not isinstance(num_shards, int): - raise ValueError("num_shards must be integer but was: {}.".format(num_shards)) + raise TypeError("num_shards must be integer but was: {}.".format(num_shards)) if not isinstance(shard_id, int): - raise ValueError("shard_id must be integer but was: {}.".format(shard_id)) + raise TypeError("shard_id must be integer but was: {}.".format(shard_id)) if not isinstance(shuffle, bool): - raise ValueError("shuffle should be a boolean value, but got shuffle: {}.".format(shuffle)) + raise TypeError("shuffle must be a boolean value but was: {}.".format(shuffle)) + + if num_samples is not None and not isinstance(num_samples, int): + raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + + if not isinstance(offset, int): + raise TypeError("offset must be integer but was: {}.".format(offset)) self.num_shards = num_shards self.shard_id = shard_id @@ -408,20 +419,30 @@ class PKSampler(BuiltinSampler): ... sampler=sampler) Raises: - ValueError: If num_val is not positive. + TypeError: If num_val is not a positive value. + TypeError: If shuffle is not a boolean value. + TypeError: If class_column is not a str value. + TypeError: If num_samples is not an integer value. NotImplementedError: If num_class is not None. - ValueError: If shuffle is not boolean. + RuntimeError: If num_val is not a positive value. + RuntimeError: If num_samples is a negative value. """ def __init__(self, num_val, num_class=None, shuffle=False, class_column='label', num_samples=None): if not isinstance(num_val, int): - raise ValueError("num_val must be integer but was: {}.".format(num_val)) + raise TypeError("num_val must be integer but was: {}.".format(num_val)) if num_class is not None: raise NotImplementedError("Not supported to specify num_class for PKSampler.") if not isinstance(shuffle, bool): - raise ValueError("shuffle should be a boolean value, but got shuffle: {}.".format(shuffle)) + raise TypeError("shuffle must be a boolean value but was: {}.".format(shuffle)) + + if not isinstance(class_column, str): + raise TypeError("class_column must be a str value but was: {}.".format(class_column)) + + if num_samples is not None and not isinstance(num_samples, int): + raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) self.num_val = num_val self.shuffle = shuffle @@ -475,13 +496,17 @@ class RandomSampler(BuiltinSampler): ... sampler=sampler) Raises: - ValueError: If replacement is not boolean. - ValueError: If num_samples is not positive. + TypeError: If replacement is not a boolean value. + TypeError: If num_samples is not an integer value. + RuntimeError: If num_samples is a negative value. """ def __init__(self, replacement=False, num_samples=None): if not isinstance(replacement, bool): - raise ValueError("replacement should be a boolean value, but got replacement: {}.".format(replacement)) + raise TypeError("replacement must be a boolean value but was: {}.".format(replacement)) + + if num_samples is not None and not isinstance(num_samples, int): + raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) self.deterministic = False self.replacement = replacement @@ -527,9 +552,21 @@ class SequentialSampler(BuiltinSampler): >>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir, ... num_parallel_workers=8, ... sampler=sampler) + + Raises: + TypeError: If start_index is not an integer value. + TypeError: If num_samples is not an integer value. + RuntimeError: If start_index is a negative value. + RuntimeError: If num_samples is a negative value. """ def __init__(self, start_index=None, num_samples=None): + if start_index is not None and not isinstance(start_index, int): + raise TypeError("start_index must be integer but was: {}.".format(start_index)) + + if num_samples is not None and not isinstance(num_samples, int): + raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + self.start_index = start_index super().__init__(num_samples) @@ -578,6 +615,11 @@ class SubsetSampler(BuiltinSampler): >>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir, ... num_parallel_workers=8, ... sampler=sampler) + + Raises: + TypeError: If type of indices element is not a number. + TypeError: If num_samples is not an integer value. + RuntimeError: If num_samples is a negative value. """ def __init__(self, indices, num_samples=None): @@ -586,9 +628,12 @@ class SubsetSampler(BuiltinSampler): for i, item in enumerate(indices): if not isinstance(item, numbers.Number): - raise TypeError("type of indices element should be number, " + raise TypeError("type of indices element must be number, " "but got w[{}]: {}, type: {}.".format(i, item, type(item))) + if num_samples is not None and not isinstance(num_samples, int): + raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + self.indices = indices super().__init__(num_samples) @@ -640,6 +685,11 @@ class SubsetRandomSampler(SubsetSampler): >>> # creates a SubsetRandomSampler, will sample from the provided indices >>> sampler = ds.SubsetRandomSampler(indices) >>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler) + + Raises: + TypeError: If type of indices element is not a number. + TypeError: If num_samples is not an integer value. + RuntimeError: If num_samples is a negative value. """ def parse(self): @@ -678,8 +728,11 @@ class WeightedRandomSampler(BuiltinSampler): ... sampler=sampler) Raises: - ValueError: If num_samples is not positive. - ValueError: If replacement is not boolean. + TypeError: If type of weights element is not a number. + TypeError: If num_samples is not an integer value. + TypeError: If replacement is not a boolean value. + RuntimeError: If weights is empty or all zero. + RuntimeError: If num_samples is a negative value. """ def __init__(self, weights, num_samples=None, replacement=True): @@ -688,11 +741,14 @@ class WeightedRandomSampler(BuiltinSampler): for ind, w in enumerate(weights): if not isinstance(w, numbers.Number): - raise TypeError("type of weights element should be number, " + raise TypeError("type of weights element must be number, " "but got w[{}]: {}, type: {}.".format(ind, w, type(w))) + if num_samples is not None and not isinstance(num_samples, int): + raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + if not isinstance(replacement, bool): - raise ValueError("replacement should be a boolean value, but got replacement: {}.".format(replacement)) + raise TypeError("replacement must be a boolean value but was: {}.".format(replacement)) self.weights = weights self.replacement = replacement diff --git a/mindspore/dataset/vision/c_transforms.py b/mindspore/dataset/vision/c_transforms.py index 93e109beff0..195bf470687 100644 --- a/mindspore/dataset/vision/c_transforms.py +++ b/mindspore/dataset/vision/c_transforms.py @@ -615,16 +615,19 @@ class RandomColorAdjust(ImageTensorOperation): Randomly adjust the brightness, contrast, saturation, and hue of the input image. Args: - brightness (Union[float, tuple], optional): Brightness adjustment factor (default=(1, 1)). Cannot be negative. + brightness (Union[float, list, tuple], optional): Brightness adjustment factor (default=(1, 1)). + Cannot be negative. If it is a float, the factor is uniformly chosen from the range [max(0, 1-brightness), 1+brightness]. If it is a sequence, it should be [min, max] for the range. - contrast (Union[float, tuple], optional): Contrast adjustment factor (default=(1, 1)). Cannot be negative. + contrast (Union[float, list, tuple], optional): Contrast adjustment factor (default=(1, 1)). + Cannot be negative. If it is a float, the factor is uniformly chosen from the range [max(0, 1-contrast), 1+contrast]. If it is a sequence, it should be [min, max] for the range. - saturation (Union[float, tuple], optional): Saturation adjustment factor (default=(1, 1)). Cannot be negative. + saturation (Union[float, list, tuple], optional): Saturation adjustment factor (default=(1, 1)). + Cannot be negative. If it is a float, the factor is uniformly chosen from the range [max(0, 1-saturation), 1+saturation]. If it is a sequence, it should be [min, max] for the range. - hue (Union[float, tuple], optional): Hue adjustment factor (default=(0, 0)). + hue (Union[float, list, tuple], optional): Hue adjustment factor (default=(0, 0)). If it is a float, the range will be [-hue, hue]. Value should be 0 <= hue <= 0.5. If it is a sequence, it should be [min, max] where -0.5 <= min <= max <= 0.5. diff --git a/mindspore/dataset/vision/validators.py b/mindspore/dataset/vision/validators.py index b6229403b47..85ba40bbb95 100644 --- a/mindspore/dataset/vision/validators.py +++ b/mindspore/dataset/vision/validators.py @@ -141,10 +141,13 @@ def check_random_color_adjust_param(value, input_name, center=1, bound=(0, FLOAT if isinstance(value, numbers.Number): if value < 0: raise ValueError("The input value of {} cannot be negative.".format(input_name)) - elif isinstance(value, (list, tuple)) and len(value) == 2: - check_range(value, bound) + elif isinstance(value, (list, tuple)): + if len(value) != 2: + raise TypeError("If {0} is a sequence, the length must be 2.".format(input_name)) if value[0] > value[1]: - raise ValueError("value should be in (min,max) format. Got (max,min).") + raise ValueError("{0} value should be in (min,max) format. Got ({1}, {2}).".format(input_name, + value[0], value[1])) + check_range(value, bound) def check_erasing_value(value): diff --git a/tests/ut/python/dataset/test_datasets_imagefolder.py b/tests/ut/python/dataset/test_datasets_imagefolder.py index aac12537b31..1abc8175500 100644 --- a/tests/ut/python/dataset/test_datasets_imagefolder.py +++ b/tests/ut/python/dataset/test_datasets_imagefolder.py @@ -391,12 +391,12 @@ def test_weighted_random_sampler_exception(): Test error cases for WeightedRandomSampler """ logger.info("Test error cases for WeightedRandomSampler") - error_msg_1 = "type of weights element should be number" + error_msg_1 = "type of weights element must be number" with pytest.raises(TypeError, match=error_msg_1): weights = "" ds.WeightedRandomSampler(weights) - error_msg_2 = "type of weights element should be number" + error_msg_2 = "type of weights element must be number" with pytest.raises(TypeError, match=error_msg_2): weights = (0.9, 0.8, 1.1) ds.WeightedRandomSampler(weights) diff --git a/tests/ut/python/dataset/test_sampler.py b/tests/ut/python/dataset/test_sampler.py index 9a2e686f5e2..92401fb416b 100644 --- a/tests/ut/python/dataset/test_sampler.py +++ b/tests/ut/python/dataset/test_sampler.py @@ -187,8 +187,8 @@ def test_subset_sampler(): else: with pytest.raises(Exception) as error_info: pipeline() - print(str(error_info)) - assert exception_msg in str(error_info) + print(str(error_info.value)) + assert exception_msg in str(error_info.value) test_config([1, 2, 3]) test_config(list(range(10))) @@ -211,7 +211,7 @@ def test_subset_sampler(): test_config([0, 9, -6, 2], exception_msg="Sample ID (-6) is out of bound, expected range [0, 9]") # test_config([], exception_msg="Indices list is empty") # temporary until we check with MindDataset test_config([0, 9, 3, 2], num_samples=-1, - exception_msg="SubsetRandomSampler: invalid num_samples: -1") + exception_msg="SubsetRandomSampler: num_samples must be greater than or equal to 0") def test_sampler_chain(): @@ -263,7 +263,7 @@ def test_add_sampler_invalid_input(): def test_distributed_sampler_invalid_offset(): with pytest.raises(RuntimeError) as info: sampler = ds.DistributedSampler(num_shards=4, shard_id=0, shuffle=False, num_samples=None, offset=5).parse() - assert "DistributedSampler: invalid offset: 5, which should be no more than num_shards: 4" in str(info.value) + assert "DistributedSampler: offset must be no more than num_shards(4)" in str(info.value) def test_sampler_list():