diff --git a/cmake/package.cmake b/cmake/package.cmake index 37838d7f953..55cdac65b63 100644 --- a/cmake/package.cmake +++ b/cmake/package.cmake @@ -326,6 +326,7 @@ install( ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/transforms.h ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/vision.h ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/vision_lite.h + ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/vision_ascend.h ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/execute.h DESTINATION ${INSTALL_BASE_DIR}/include/minddata/dataset/include COMPONENT mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/generator_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/generator_node.cc index 3b839a85aa8..87146766c87 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/generator_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/generator_node.cc @@ -105,6 +105,11 @@ Status GeneratorNode::Build(std::vector> *const node_ // no validation is needed for generator op. Status GeneratorNode::ValidateParams() { RETURN_IF_NOT_OK(DatasetNode::ValidateParams()); + if (source_len_ == 0) { + std::string err_msg = "GeneratorNode: data row of input source must not be 0, got: " + std::to_string(source_len_); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/sharpness_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/sharpness_op.cc index 87d0a714ccf..94de8a42535 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/sharpness_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/sharpness_op.cc @@ -38,12 +38,6 @@ Status SharpnessOp::Compute(const std::shared_ptr &input, std::shared_pt RETURN_STATUS_UNEXPECTED("Sharpness: image shape is not or "); } - /// Get number of channels and image matrix - std::size_t num_of_channels = input_cv->shape()[2]; - if (num_of_channels != 1 && num_of_channels != 3) { - RETURN_STATUS_UNEXPECTED("Sharpness: image channel is not 1 or 3."); - } - /// creating a smoothing filter. 1, 1, 1, /// 1, 5, 1, /// 1, 1, 1 diff --git a/mindspore/dataset/core/validator_helpers.py b/mindspore/dataset/core/validator_helpers.py index 64f44984841..c57afc4a73b 100644 --- a/mindspore/dataset/core/validator_helpers.py +++ b/mindspore/dataset/core/validator_helpers.py @@ -344,8 +344,9 @@ def check_num_parallel_workers(value): def check_num_samples(value): type_check(value, (int,), "num_samples") - check_value(value, [0, INT32_MAX], "num_samples") - + if value < 0 or value > INT64_MAX: + raise ValueError( + "num_samples exceeds the boundary between {} and {}(INT64_MAX)!".format(0, INT64_MAX)) def validate_dataset_param_value(param_list, param_dict, param_type): for param_name in param_list: diff --git a/mindspore/dataset/engine/samplers.py b/mindspore/dataset/engine/samplers.py index 644fbbc6dba..75e44ea77e6 100644 --- a/mindspore/dataset/engine/samplers.py +++ b/mindspore/dataset/engine/samplers.py @@ -23,6 +23,7 @@ import numbers import numpy as np import mindspore._c_dataengine as cde import mindspore.dataset as ds +from ..core import validator_helpers as validator def select_sampler(num_samples, input_sampler, shuffle, num_shards, shard_id): @@ -349,8 +350,12 @@ class DistributedSampler(BuiltinSampler): if not isinstance(shuffle, bool): raise TypeError("shuffle must be a boolean value but was: {}.".format(shuffle)) - if num_samples is not None and not isinstance(num_samples, int): - raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + if num_samples is not None: + if not isinstance(num_samples, int): + raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + if num_samples < 0 or num_samples > validator.INT64_MAX: + raise ValueError("num_samples exceeds the boundary between {} and {}(INT64_MAX)!" + .format(0, validator.INT64_MAX)) if not isinstance(offset, int): raise TypeError("offset must be integer but was: {}.".format(offset)) @@ -441,8 +446,12 @@ class PKSampler(BuiltinSampler): if not isinstance(class_column, str): raise TypeError("class_column must be a str value but was: {}.".format(class_column)) - if num_samples is not None and not isinstance(num_samples, int): - raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + if num_samples is not None: + if not isinstance(num_samples, int): + raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + if num_samples < 0 or num_samples > validator.INT64_MAX: + raise ValueError("num_samples exceeds the boundary between {} and {}(INT64_MAX)!" + .format(0, validator.INT64_MAX)) self.num_val = num_val self.shuffle = shuffle @@ -505,8 +514,12 @@ class RandomSampler(BuiltinSampler): if not isinstance(replacement, bool): raise TypeError("replacement must be a boolean value but was: {}.".format(replacement)) - if num_samples is not None and not isinstance(num_samples, int): - raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + if num_samples is not None: + if not isinstance(num_samples, int): + raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + if num_samples < 0 or num_samples > validator.INT64_MAX: + raise ValueError("num_samples exceeds the boundary between {} and {}(INT64_MAX)!" + .format(0, validator.INT64_MAX)) self.deterministic = False self.replacement = replacement @@ -564,8 +577,12 @@ class SequentialSampler(BuiltinSampler): if start_index is not None and not isinstance(start_index, int): raise TypeError("start_index must be integer but was: {}.".format(start_index)) - if num_samples is not None and not isinstance(num_samples, int): - raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + if num_samples is not None: + if not isinstance(num_samples, int): + raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + if num_samples < 0 or num_samples > validator.INT64_MAX: + raise ValueError("num_samples exceeds the boundary between {} and {}(INT64_MAX)!" + .format(0, validator.INT64_MAX)) self.start_index = start_index super().__init__(num_samples) @@ -631,8 +648,12 @@ class SubsetSampler(BuiltinSampler): raise TypeError("type of indices element must be number, " "but got w[{}]: {}, type: {}.".format(i, item, type(item))) - if num_samples is not None and not isinstance(num_samples, int): - raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + if num_samples is not None: + if not isinstance(num_samples, int): + raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + if num_samples < 0 or num_samples > validator.INT64_MAX: + raise ValueError("num_samples exceeds the boundary between {} and {}(INT64_MAX)!" + .format(0, validator.INT64_MAX)) self.indices = indices super().__init__(num_samples) @@ -744,8 +765,12 @@ class WeightedRandomSampler(BuiltinSampler): raise TypeError("type of weights element must be number, " "but got w[{}]: {}, type: {}.".format(ind, w, type(w))) - if num_samples is not None and not isinstance(num_samples, int): - raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + if num_samples is not None: + if not isinstance(num_samples, int): + raise TypeError("num_samples must be integer but was: {}.".format(num_samples)) + if num_samples < 0 or num_samples > validator.INT64_MAX: + raise ValueError("num_samples exceeds the boundary between {} and {}(INT64_MAX)!" + .format(0, validator.INT64_MAX)) if not isinstance(replacement, bool): raise TypeError("replacement must be a boolean value but was: {}.".format(replacement)) diff --git a/mindspore/dataset/vision/c_transforms.py b/mindspore/dataset/vision/c_transforms.py index e63b60bf7ce..73c3eb48950 100644 --- a/mindspore/dataset/vision/c_transforms.py +++ b/mindspore/dataset/vision/c_transforms.py @@ -444,10 +444,9 @@ class Pad(ImageTensorOperation): If 4 values are provided as a list or tuple, it pads the left, top, right and bottom respectively. fill_value (Union[int, tuple], optional): The pixel intensity of the borders, only valid for - padding_mode Border.CONSTANT (default=0). + padding_mode Border.CONSTANT. If it is a 3-tuple, it is used to fill R, G, B channels respectively. If it is an integer, it is used for all RGB channels. - If it is a 3-tuple, it is used to fill R, G, B channels respectively. - The fill_value values must be in range [0, 255]. + The fill_value values must be in range [0, 255] (default=0). padding_mode (Border mode, optional): The method of padding (default=Border.CONSTANT). Can be any of [Border.CONSTANT, Border.EDGE, Border.REFLECT, Border.SYMMETRIC]. @@ -684,9 +683,10 @@ class RandomCrop(ImageTensorOperation): pad the left, top, right and bottom respectively. pad_if_needed (bool, optional): Pad the image if either side is smaller than the given output size (default=False). - fill_value (Union[int, tuple], optional): The pixel intensity of the borders if - the padding_mode is Border.CONSTANT (default=0). If it is a 3-tuple, it is used to - fill R, G, B channels respectively. + fill_value (Union[int, tuple], optional): The pixel intensity of the borders, only valid for + padding_mode Border.CONSTANT. If it is a 3-tuple, it is used to fill R, G, B channels respectively. + If it is an integer, it is used for all RGB channels. + The fill_value values must be in range [0, 255] (default=0). padding_mode (Border mode, optional): The method of padding (default=Border.CONSTANT). It can be any of [Border.CONSTANT, Border.EDGE, Border.REFLECT, Border.SYMMETRIC]. @@ -799,9 +799,10 @@ class RandomCropWithBBox(ImageTensorOperation): If 4 values are provided as a list or tuple, pad the left, top, right and bottom respectively. pad_if_needed (bool, optional): Pad the image if either side is smaller than the given output size (default=False). - fill_value (Union[int, tuple], optional): The pixel intensity of the borders if - the padding_mode is Border.CONSTANT (default=0). If it is a 3-tuple, it is used to - fill R, G, B channels respectively. + fill_value (Union[int, tuple], optional): The pixel intensity of the borders, only valid for + padding_mode Border.CONSTANT. If it is a 3-tuple, it is used to fill R, G, B channels respectively. + If it is an integer, it is used for all RGB channels. + The fill_value values must be in range [0, 255] (default=0). padding_mode (Border mode, optional): The method of padding (default=Border.CONSTANT). It can be any of [Border.CONSTANT, Border.EDGE, Border.REFLECT, Border.SYMMETRIC]. @@ -1105,10 +1106,10 @@ class RandomRotation(ImageTensorOperation): Note that the expand flag assumes rotation around the center and no translation. center (tuple, optional): Optional center of rotation (a 2-tuple) (default=None). Origin is the top left corner. None sets to the center of the image. - fill_value (Union[int, tuple], optional): Optional fill color for the area outside the rotated image - (default=0). - If it is a 3-tuple, it is used for R, G, B channels respectively. + fill_value (Union[int, tuple], optional): Optional fill color for the area outside the rotated image. + If it is a 3-tuple, it is used to fill R, G, B channels respectively. If it is an integer, it is used for all RGB channels. + The fill_value values must be in range [0, 255] (default=0). Examples: >>> from mindspore.dataset.vision import Inter diff --git a/tests/ut/python/dataset/test_datasets_textfileop.py b/tests/ut/python/dataset/test_datasets_textfileop.py index 897a8a9a6bb..a81196b7ae2 100644 --- a/tests/ut/python/dataset/test_datasets_textfileop.py +++ b/tests/ut/python/dataset/test_datasets_textfileop.py @@ -195,7 +195,7 @@ def test_textline_dataset_to_device(): def test_textline_dataset_exceptions(): with pytest.raises(ValueError) as error_info: _ = ds.TextFileDataset(DATA_FILE, num_samples=-1) - assert "Input num_samples is not within the required interval" in str(error_info.value) + assert "num_samples exceeds the boundary" in str(error_info.value) with pytest.raises(ValueError) as error_info: _ = ds.TextFileDataset("does/not/exist/no.txt") diff --git a/tests/ut/python/dataset/test_exceptions.py b/tests/ut/python/dataset/test_exceptions.py index 6f9d156174f..0b5c309d5fa 100644 --- a/tests/ut/python/dataset/test_exceptions.py +++ b/tests/ut/python/dataset/test_exceptions.py @@ -41,7 +41,7 @@ def test_exception_02(): num_samples = -1 with pytest.raises(ValueError) as info: ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], num_samples=num_samples) - assert 'Input num_samples is not within the required interval of (0 to 2147483647).' in str(info.value) + assert 'num_samples exceeds the boundary between 0 and 9223372036854775807(INT64_MAX)' in str(info.value) num_samples = 1 data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], num_samples=num_samples) diff --git a/tests/ut/python/dataset/test_minddataset_exception.py b/tests/ut/python/dataset/test_minddataset_exception.py index 0d3ccf5e44f..5e1468d369d 100644 --- a/tests/ut/python/dataset/test_minddataset_exception.py +++ b/tests/ut/python/dataset/test_minddataset_exception.py @@ -280,7 +280,7 @@ def test_cv_minddataset_partition_num_samples_equals_0(): with pytest.raises(ValueError) as error_info: partitions(5) try: - assert 'Input num_samples is not within the required interval of (0 to 2147483647).' in str(error_info.value) + assert 'num_samples exceeds the boundary between 0 and 9223372036854775807(INT64_MAX)' in str(error_info.value) except Exception as error: os.remove(CV_FILE_NAME) os.remove("{}.db".format(CV_FILE_NAME)) diff --git a/tests/ut/python/dataset/test_sampler.py b/tests/ut/python/dataset/test_sampler.py index 92401fb416b..3555409ee75 100644 --- a/tests/ut/python/dataset/test_sampler.py +++ b/tests/ut/python/dataset/test_sampler.py @@ -211,7 +211,7 @@ def test_subset_sampler(): test_config([0, 9, -6, 2], exception_msg="Sample ID (-6) is out of bound, expected range [0, 9]") # test_config([], exception_msg="Indices list is empty") # temporary until we check with MindDataset test_config([0, 9, 3, 2], num_samples=-1, - exception_msg="SubsetRandomSampler: num_samples must be greater than or equal to 0") + exception_msg="num_samples exceeds the boundary between 0 and 9223372036854775807(INT64_MAX)") def test_sampler_chain():