!14556 Fix validation of RGB2HSV/HSV2RGB/RandomResizedCrop in py_transform & Fix error msg of pythontokenizer

From: @luoyang42
Reviewed-by: @liucunwei,@heleiwang
Signed-off-by: @liucunwei

mindspore/dataset/core/validator_helpers.py

@@ -267,7 +267,7 @@ def type_check(arg, types, arg_name):
         if isinstance(arg, bool):
             raise TypeError("Argument {0} with value {1} is not of type {2}.".format(arg_name, print_value, types))
     if not isinstance(arg, types):
-        raise TypeError("Argument {0} with value {1} is not of type {2}.".format(arg_name, print_value, types))
+        raise TypeError("Argument {0} with value {1} is not of type {2}.".format(arg_name, print_value, list(types)))
 
 
 def check_filename(path):

mindspore/dataset/text/transforms.py

@@ -530,6 +530,7 @@ class PythonTokenizer:
 
     @check_python_tokenizer
     def __init__(self, tokenizer):
+        self.pyfunc = tokenizer
         self.tokenizer = np.vectorize(lambda x: np.array(tokenizer(x), dtype='U'), signature='()->(n)')
         self.random = False
 
@@ -538,7 +539,10 @@ class PythonTokenizer:
             raise TypeError("input should be a NumPy array. Got {}.".format(type(in_array)))
         if in_array.dtype.type is np.bytes_:
             in_array = to_str(in_array)
-        tokens = self.tokenizer(in_array)
+        try:
+            tokens = self.tokenizer(in_array)
+        except Exception as e:
+            raise RuntimeError("Error occurred in Pyfunc [" + str(self.pyfunc.__name__) + "], error message: " + str(e))
         return tokens
 
 

mindspore/dataset/vision/py_transforms.py

@@ -29,7 +29,7 @@ from . import py_transforms_util as util
 from .c_transforms import parse_padding
 from .validators import check_prob, check_crop, check_resize_interpolation, check_random_resize_crop, \
     check_normalize_py, check_normalizepad_py, check_random_crop, check_random_color_adjust, check_random_rotation, \
-    check_ten_crop, check_num_channels, check_pad, \
+    check_ten_crop, check_num_channels, check_pad, check_rgb_to_hsv, check_hsv_to_rgb, \
     check_random_perspective, check_random_erasing, check_cutout, check_linear_transform, check_random_affine, \
     check_mix_up, check_positive_degrees, check_uniform_augment_py, check_auto_contrast
 from .utils import Inter, Border
@@ -497,9 +497,9 @@ class RandomResizedCrop:
         size (Union[int, sequence]): The size of the output image.
             If size is an integer, a square crop of size (size, size) is returned.
             If size is a sequence of length 2, it should be (height, width).
-        scale (tuple, optional): Range (min, max) of respective size of the original size
+        scale (list, tuple, optional): Range (min, max) of respective size of the original size
             to be cropped (default=(0.08, 1.0)).
-        ratio (tuple, optional): Range (min, max) of aspect ratio to be cropped (default=(3. / 4., 4. / 3.)).
+        ratio (list, tuple, optional): Range (min, max) of aspect ratio to be cropped (default=(3. / 4., 4. / 3.)).
         interpolation (Inter mode, optional): Image interpolation mode (default=Inter.BILINEAR).
             It can be any of [Inter.NEAREST, Inter.ANTIALIAS, Inter.BILINEAR, Inter.BICUBIC].
 
@@ -1296,6 +1296,7 @@ class RgbToHsv:
         ...                                                 input_columns="image")
     """
 
+    @check_rgb_to_hsv
     def __init__(self, is_hwc=False):
         self.is_hwc = is_hwc
         self.random = False
@@ -1333,6 +1334,7 @@ class HsvToRgb:
         ...                                                 input_columns="image")
     """
 
+    @check_hsv_to_rgb
     def __init__(self, is_hwc=False):
         self.is_hwc = is_hwc
         self.random = False

mindspore/dataset/vision/validators.py

@@ -235,14 +235,18 @@ def check_size_scale_ration_max_attempts_paras(size, scale, ratio, max_attempts)
 
     check_crop_size(size)
     if scale is not None:
-        type_check(scale, (tuple,), "scale")
+        type_check(scale, (tuple, list), "scale")
+        if len(scale) != 2:
+            raise TypeError("scale should be a list/tuple of length 2.")
         type_check_list(scale, (float, int), "scale")
         if scale[0] > scale[1]:
             raise ValueError("scale should be in (min,max) format. Got (max,min).")
         check_range(scale, [0, FLOAT_MAX_INTEGER])
         check_positive(scale[1], "scale[1]")
     if ratio is not None:
-        type_check(ratio, (tuple,), "ratio")
+        type_check(ratio, (tuple, list), "ratio")
+        if len(ratio) != 2:
+            raise TypeError("ratio should be a list/tuple of length 2.")
         type_check_list(ratio, (float, int), "ratio")
         if ratio[0] > ratio[1]:
             raise ValueError("ratio should be in (min,max) format. Got (max,min).")
@@ -479,6 +483,28 @@ def check_mix_up(method):
     return new_method
 
 
+def check_rgb_to_hsv(method):
+    """Wrapper method to check the parameters of rgb_to_hsv."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        [is_hwc], _ = parse_user_args(method, *args, **kwargs)
+        type_check(is_hwc, (bool,), "is_hwc")
+        return method(self, *args, **kwargs)
+    return new_method
+
+
+def check_hsv_to_rgb(method):
+    """Wrapper method to check the parameters of hsv_to_rgb."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        [is_hwc], _ = parse_user_args(method, *args, **kwargs)
+        type_check(is_hwc, (bool,), "is_hwc")
+        return method(self, *args, **kwargs)
+    return new_method
+
+
 def check_random_erasing(method):
     """Wrapper method to check the parameters of random erasing."""
 

tests/ut/python/dataset/test_bucket_batch_by_length.py

@@ -65,7 +65,7 @@ def test_bucket_batch_invalid_input():
 
     with pytest.raises(TypeError) as info:
         _ = dataset.bucket_batch_by_length(invalid_column_names, bucket_boundaries, bucket_batch_sizes)
-    assert "Argument column_names[0] with value 1 is not of type (<class 'str'>,)." in str(info.value)
+    assert "Argument column_names[0] with value 1 is not of type [<class 'str'>]." in str(info.value)
 
     with pytest.raises(ValueError) as info:
         _ = dataset.bucket_batch_by_length(column_names, empty_bucket_boundaries, bucket_batch_sizes)
@@ -110,12 +110,12 @@ def test_bucket_batch_invalid_input():
     with pytest.raises(TypeError) as info:
         _ = dataset.bucket_batch_by_length(column_names, bucket_boundaries, bucket_batch_sizes,
                                            None, None, invalid_type_pad_to_bucket_boundary)
-    assert "Argument pad_to_bucket_boundary with value \"\" is not of type (<class \'bool\'>,)." in str(info.value)
+    assert "Argument pad_to_bucket_boundary with value \"\" is not of type [<class \'bool\'>]." in str(info.value)
 
     with pytest.raises(TypeError) as info:
         _ = dataset.bucket_batch_by_length(column_names, bucket_boundaries, bucket_batch_sizes,
                                            None, None, False, invalid_type_drop_remainder)
-    assert "Argument drop_remainder with value \"\" is not of type (<class 'bool'>,)." in str(info.value)
+    assert "Argument drop_remainder with value \"\" is not of type [<class 'bool'>]." in str(info.value)
 
 
 def test_bucket_batch_multi_bucket_no_padding():

tests/ut/python/dataset/test_c_random_apply.py

@@ -38,10 +38,10 @@ def test_random_apply():
     assert test_config([[0, 1, 2]], [ops.Compose([ops.Duplicate(), ops.Concatenate(), ops.Slice([0, 1, 2])])]) == [
         [0, 1, 2]]
     # test exception
-    assert "is not of type (<class 'list'>" in test_config([1, 0], ops.TypeCast(mstype.int32))
+    assert "is not of type [<class 'list'>]" in test_config([1, 0], ops.TypeCast(mstype.int32))
     assert "Input prob is not within the required interval" in test_config([0, 1], [ops.Slice([0, 1])], 1.1)
-    assert "is not of type (<class 'float'>" in test_config([1, 0], [ops.TypeCast(mstype.int32)], None)
-    assert "op_list with value None is not of type (<class 'list'>" in test_config([1, 0], None)
+    assert "is not of type [<class 'float'>, <class 'int'>]" in test_config([1, 0], [ops.TypeCast(mstype.int32)], None)
+    assert "op_list with value None is not of type [<class 'list'>]" in test_config([1, 0], None)
 
 
 if __name__ == "__main__":

tests/ut/python/dataset/test_dataset_numpy_slices.py

@@ -209,7 +209,7 @@ def test_numpy_slices_invalid_column_names_type():
 
     with pytest.raises(TypeError) as err:
         de.NumpySlicesDataset(np_data, column_names=[1], shuffle=False)
-    assert "Argument column_names[0] with value 1 is not of type (<class 'str'>,)." in str(err.value)
+    assert "Argument column_names[0] with value 1 is not of type [<class 'str'>]." in str(err.value)
 
 
 def test_numpy_slices_invalid_column_names_string():

tests/ut/python/dataset/test_datasets_cifarop.py

@@ -416,7 +416,7 @@ def test_cifar_usage():
     assert test_config("train") == 10000
     assert test_config("all") == 10000
     assert "usage is not within the valid set of ['train', 'test', 'all']" in test_config("invalid")
-    assert "Argument usage with value ['list'] is not of type (<class 'str'>,)" in test_config(["list"])
+    assert "Argument usage with value ['list'] is not of type [<class 'str'>]" in test_config(["list"])
     assert "no valid data matching the dataset API Cifar10Dataset" in test_config("test")
 
     # test the usage of CIFAR10

tests/ut/python/dataset/test_datasets_mnist.py

@@ -272,7 +272,7 @@ def test_mnist_usage():
     assert test_config("all") == 10000
     assert " no valid data matching the dataset API MnistDataset" in test_config("train")
     assert "usage is not within the valid set of ['train', 'test', 'all']" in test_config("invalid")
-    assert "Argument usage with value ['list'] is not of type (<class 'str'>,)" in test_config(["list"])
+    assert "Argument usage with value ['list'] is not of type [<class 'str'>]" in test_config(["list"])
 
     # change this directory to the folder that contains all mnist files
     all_files_path = None

tests/ut/python/dataset/test_exceptions.py

@@ -30,7 +30,7 @@ def test_exception_01():
     data = ds.TFRecordDataset(DATA_DIR, columns_list=["image"])
     with pytest.raises(TypeError) as info:
         data.map(operations=vision.Resize(100, 100), input_columns=["image"])
-    assert "Argument interpolation with value 100 is not of type (<enum 'Inter'>,)" in str(info.value)
+    assert "Argument interpolation with value 100 is not of type [<enum 'Inter'>]" in str(info.value)
 
 
 def test_exception_02():

tests/ut/python/dataset/test_from_dataset.py

@@ -133,8 +133,8 @@ def test_from_dataset_exceptions():
 
     test_config("text", (), 1, "freq_range needs to be a tuple of 2 integers or an int and a None.")
     test_config("text", (2, 3), 1.2345,
-                "Argument top_k with value 1.2345 is not of type (<class 'int'>, <class 'NoneType'>)")
-    test_config(23, (2, 3), 1.2345, "Argument col[0] with value 23 is not of type (<class 'str'>,)")
+                "Argument top_k with value 1.2345 is not of type [<class 'int'>, <class 'NoneType'>]")
+    test_config(23, (2, 3), 1.2345, "Argument col[0] with value 23 is not of type [<class 'str'>]")
     test_config("text", (100, 1), 12, "frequency range [a,b] should be 0 <= a <= b (a,b are inclusive)")
     test_config("text", (2, 3), 0, "top_k must be greater than 0")
     test_config([123], (2, 3), -1, "top_k must be greater than 0")

tests/ut/python/dataset/test_linear_transformation.py

@@ -132,7 +132,7 @@ def test_linear_transformation_exception_01():
         data1 = data1.map(operations=transform, input_columns=["image"])
     except TypeError as e:
         logger.info("Got an exception in DE: {}".format(str(e)))
-        assert "Argument transformation_matrix with value None is not of type (<class 'numpy.ndarray'>,)" in str(e)
+        assert "Argument transformation_matrix with value None is not of type [<class 'numpy.ndarray'>]" in str(e)
 
 
 def test_linear_transformation_exception_02():
@@ -161,7 +161,7 @@ def test_linear_transformation_exception_02():
         data1 = data1.map(operations=transform, input_columns=["image"])
     except TypeError as e:
         logger.info("Got an exception in DE: {}".format(str(e)))
-        assert "Argument mean_vector with value None is not of type (<class 'numpy.ndarray'>,)" in str(e)
+        assert "Argument mean_vector with value None is not of type [<class 'numpy.ndarray'>]" in str(e)
 
 
 def test_linear_transformation_exception_03():

tests/ut/python/dataset/test_ngram_op.py

@@ -111,7 +111,7 @@ def test_corner_cases():
     # test left pad != right pad
     assert test_config("Lone Star", 4, ("The", 1), ("State", 1)) == ['The Lone Star State']
     # test invalid n
-    assert "gram[1] with value [1] is not of type (<class 'int'>,)" in test_config("Yours to Discover", [1, [1]])
+    assert "gram[1] with value [1] is not of type [<class 'int'>]" in test_config("Yours to Discover", [1, [1]])
     assert "n needs to be a non-empty list" in test_config("Yours to Discover", [])
     # test invalid pad
     assert "padding width need to be positive numbers" in test_config("Yours to Discover", [1], ("str", -1))

tests/ut/python/dataset/test_pad_end_op.py

@@ -63,7 +63,7 @@ def test_pad_end_exceptions():
 
     with pytest.raises(TypeError) as info:
         pad_compare([1, 2], 3, -1, [1, 2, -1])
-    assert "Argument pad_end with value 3 is not of type (<class 'list'>,)" in str(info.value)
+    assert "Argument pad_end with value 3 is not of type [<class 'list'>]" in str(info.value)
 
 
 if __name__ == "__main__":

tests/ut/python/dataset/test_random_affine.py

@@ -306,8 +306,8 @@ def test_random_affine_exception_translate_size():
     except TypeError as e:
         logger.info("Got an exception in DE: {}".format(str(e)))
         assert str(
-            e) == "Argument translate with value 0.1 is not of type (<class 'list'>," \
-                  " <class 'tuple'>)."
+            e) == "Argument translate with value 0.1 is not of type [<class 'list'>," \
+                  " <class 'tuple'>]."
 
 
 def test_random_affine_exception_scale_size():
@@ -320,8 +320,8 @@ def test_random_affine_exception_scale_size():
         _ = py_vision.RandomAffine(degrees=15, scale=(0.5))
     except TypeError as e:
         logger.info("Got an exception in DE: {}".format(str(e)))
-        assert str(e) == "Argument scale with value 0.5 is not of type (<class 'tuple'>," \
-                         " <class 'list'>)."
+        assert str(e) == "Argument scale with value 0.5 is not of type [<class 'tuple'>," \
+                         " <class 'list'>]."
 
 
 def test_random_affine_exception_shear_size():

tests/ut/python/dataset/test_random_color.py

@@ -221,7 +221,7 @@ def test_random_color_c_errors():
 
     with pytest.raises(TypeError) as error_info:
         vision.RandomColor(("col", 3))
-    assert "Argument degrees[0] with value col is not of type (<class 'int'>, <class 'float'>)." in str(
+    assert "Argument degrees[0] with value col is not of type [<class 'int'>, <class 'float'>]." in str(
         error_info.value)
 
     with pytest.raises(ValueError) as error_info:

tests/ut/python/dataset/test_random_crop_and_resize.py

@@ -395,7 +395,7 @@ def test_random_crop_and_resize_06():
         data.map(operations=random_crop_and_resize_op, input_columns=["image"])
     except TypeError as e:
         logger.info("Got an exception in DE: {}".format(str(e)))
-        assert "Argument scale with value \"\" is not of type (<class 'tuple'>,)" in str(e)
+        assert "Argument scale with value \"\" is not of type [<class 'tuple'>, <class 'list'>]" in str(e)
 
     try:
         random_crop_and_resize_op = c_vision.RandomResizedCrop((256, 512), scale=(1, "2"), ratio=(1, 0.5))
@@ -403,7 +403,7 @@ def test_random_crop_and_resize_06():
         data.map(operations=random_crop_and_resize_op, input_columns=["image"])
     except TypeError as e:
         logger.info("Got an exception in DE: {}".format(str(e)))
-        assert "Argument scale[1] with value 2 is not of type (<class 'float'>, <class 'int'>)." in str(e)
+        assert "Argument scale[1] with value 2 is not of type [<class 'float'>, <class 'int'>]." in str(e)
 
 
 if __name__ == "__main__":

tests/ut/python/dataset/test_random_posterize.py

@@ -186,7 +186,7 @@ def test_random_posterize_exception_bit():
         _ = c_vision.RandomPosterize(1.1)
     except TypeError as e:
         logger.info("Got an exception in DE: {}".format(str(e)))
-        assert str(e) == "Argument bits with value 1.1 is not of type (<class 'list'>, <class 'tuple'>, <class 'int'>)."
+        assert str(e) == "Argument bits with value 1.1 is not of type [<class 'list'>, <class 'tuple'>, <class 'int'>]."
     # Test wrong number of bits
     try:
         _ = c_vision.RandomPosterize((1, 1, 1))

tests/ut/python/dataset/test_random_solarize_op.py

@@ -114,7 +114,7 @@ def test_random_solarize_errors():
 
     with pytest.raises(TypeError) as error_info:
         vision.RandomSolarize((122.1, 140))
-    assert "Argument threshold[0] with value 122.1 is not of type (<class 'int'>,)." in str(error_info.value)
+    assert "Argument threshold[0] with value 122.1 is not of type [<class 'int'>]." in str(error_info.value)
 
     with pytest.raises(ValueError) as error_info:
         vision.RandomSolarize((122, 100, 30))

tests/ut/python/dataset/test_resize.py

@@ -128,7 +128,7 @@ def test_resize_op_invalid_input():
     test_invalid_input("invalid size parameter shape", (2, 3, 4), Inter.LINEAR, TypeError,
                        "Size should be a single integer or a list/tuple (h, w) of length 2.")
     test_invalid_input("invalid size parameter type in a tuple", (2.3, 3), Inter.LINEAR, TypeError,
-                       "Argument size at dim 0 with value 2.3 is not of type (<class 'int'>,)")
+                       "Argument size at dim 0 with value 2.3 is not of type [<class 'int'>]")
     test_invalid_input("invalid Interpolation value", (2.3, 3), None, KeyError, "None")
 
 

tests/ut/python/dataset/test_schema.py

@@ -48,7 +48,7 @@ def test_schema_exception():
 
     with pytest.raises(TypeError) as info:
         ds.Schema(1)
-    assert "Argument schema_file with value 1 is not of type (<class 'str'>,)" in str(info.value)
+    assert "Argument schema_file with value 1 is not of type [<class 'str'>]" in str(info.value)
 
     with pytest.raises(RuntimeError) as info:
         schema = ds.Schema(SCHEMA_FILE)

tests/ut/python/dataset/test_slice_op.py

@@ -299,12 +299,12 @@ def test_slice_exceptions():
     with pytest.raises(TypeError) as info:
         slice_compare([b"1", b"2", b"3", b"4", b"5"], [[[0, 1]]], [b"1", b"2", b"3", b"4", b"5"])
     assert "Argument slice_option[0] with value [0, 1] is not of type " \
-           "(<class 'int'>,)." in str(info.value)
+           "[<class 'int'>]." in str(info.value)
 
     with pytest.raises(TypeError) as info:
         slice_compare([b"1", b"2", b"3", b"4", b"5"], [[slice(3)]], [b"1", b"2", b"3", b"4", b"5"])
     assert "Argument slice_option[0] with value slice(None, 3, None) is not of type " \
-           "(<class 'int'>,)." in str(info.value)
+           "[<class 'int'>]." in str(info.value)
 
 
 if __name__ == "__main__":

tests/ut/python/dataset/test_sync_wait.py

@@ -282,7 +282,7 @@ def test_sync_exception_06():
     # try to create dataset with batch_size < 0
     with pytest.raises(TypeError) as e:
         dataset.sync_wait(condition_name="every batch", num_batch="123", callback=aug.update)
-    assert "is not of type (<class 'int'>" in str(e.value)
+    assert "is not of type [<class 'int'>]" in str(e.value)
 
 
 if __name__ == "__main__":

tests/ut/python/dataset/test_uniform_augment.py

@@ -247,7 +247,7 @@ def test_cpp_uniform_augment_exception_float_numops(num_ops=2.5):
 
     except Exception as e:
         logger.info("Got an exception in DE: {}".format(str(e)))
-        assert "Argument num_ops with value 2.5 is not of type (<class 'int'>,)" in str(e)
+        assert "Argument num_ops with value 2.5 is not of type [<class 'int'>]" in str(e)
 
 
 def test_cpp_uniform_augment_random_crop_badinput(num_ops=1):

tests/ut/python/dataset/test_vocab.py

@@ -201,7 +201,7 @@ def test_lookup_cast_type():
     assert test_config("unk", mstype.float32) != np.dtype("int32")
     assert test_config("unk") == np.dtype("int32")
     # test exception, data_type isn't the correct type
-    assert "tldr is not of type (<class 'mindspore._c_expression.typing.Type'>,)" in test_config("unk", "tldr")
+    assert "tldr is not of type [<class 'mindspore._c_expression.typing.Type'>]" in test_config("unk", "tldr")
     assert "Lookup does not support a string to string mapping, data_type can only be numeric." in \
            test_config("w1", mstype.string)