p60975123
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!12336 [MD] fix sampler log & transform log 

From: @luoyang42
Reviewed-by: 
Signed-off-by:
This commit is contained in:
mindspore-ci-bot 2021-02-10 15:33:15 +08:00 committed by Gitee
parent b0cc8662c4 416ad23e9d
commit 260c77f4ad
7 changed files with 115 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
mindspore/ccsrc/minddata/dataset/api/samplers.cc
View File

@ -173,21 +173,23 @@ DistributedSamplerObj::DistributedSamplerObj(int64_t num_shards, int64_t shard_i
Status DistributedSamplerObj::ValidateParams() {
if (num_shards_ <= 0) {
RETURN_STATUS_UNEXPECTED("DistributedSampler: invalid num_shards: " + std::to_string(num_shards_));
RETURN_STATUS_UNEXPECTED("DistributedSampler: num_shards must be greater than 0, but got: " +
std::to_string(num_shards_));
}
if (shard_id_ < 0 || shard_id_ >= num_shards_) {
RETURN_STATUS_UNEXPECTED("DistributedSampler: invalid input, shard_id: " + std::to_string(shard_id_) +
", num_shards: " + std::to_string(num_shards_));
RETURN_STATUS_UNEXPECTED("DistributedSampler: shard_id must be in range [0, " + std::to_string(num_shards_) +
"), but got: " + std::to_string(shard_id_));
}
if (num_samples_ < 0) {
RETURN_STATUS_UNEXPECTED("DistributedSampler: invalid num_samples: " + std::to_string(num_samples_));
RETURN_STATUS_UNEXPECTED("DistributedSampler: num_samples must be greater than or equal to 0, but got: " +
std::to_string(num_samples_));
}
if (offset_ > num_shards_) {
RETURN_STATUS_UNEXPECTED("DistributedSampler: invalid offset: " + std::to_string(offset_) +
", which should be no more than num_shards: " + std::to_string(num_shards_));
RETURN_STATUS_UNEXPECTED("DistributedSampler: offset must be no more than num_shards(" +
std::to_string(num_shards_) + "), but got: " + std::to_string(offset_));
}
return Status::OK();
@ -237,11 +239,12 @@ PKSamplerObj::PKSamplerObj(int64_t num_val, bool shuffle, int64_t num_samples)
Status PKSamplerObj::ValidateParams() {
if (num_val_ <= 0) {
RETURN_STATUS_UNEXPECTED("PKSampler: invalid num_val: " + std::to_string(num_val_));
RETURN_STATUS_UNEXPECTED("PKSampler: num_val must be greater than 0, but got: " + std::to_string(num_val_));
}
if (num_samples_ < 0) {
RETURN_STATUS_UNEXPECTED("PKSampler: invalid num_samples: " + std::to_string(num_samples_));
RETURN_STATUS_UNEXPECTED("PKSampler: num_samples must be greater than or equal to 0, but got: " +
std::to_string(num_samples_));
}
return Status::OK();
}
@ -334,7 +337,8 @@ RandomSamplerObj::RandomSamplerObj(bool replacement, int64_t num_samples, bool r
Status RandomSamplerObj::ValidateParams() {
if (num_samples_ < 0) {
RETURN_STATUS_UNEXPECTED("RandomSampler: invalid num_samples: " + std::to_string(num_samples_));
RETURN_STATUS_UNEXPECTED("RandomSampler: num_samples must be greater than or equal to 0, but got: " +
std::to_string(num_samples_));
}
return Status::OK();
}
@ -381,11 +385,13 @@ SequentialSamplerObj::SequentialSamplerObj(int64_t start_index, int64_t num_samp
Status SequentialSamplerObj::ValidateParams() {
if (num_samples_ < 0) {
RETURN_STATUS_UNEXPECTED("SequentialSampler: invalid num_samples: " + std::to_string(num_samples_));
RETURN_STATUS_UNEXPECTED("SequentialSampler: num_samples must be greater than or equal to 0, but got: " +
std::to_string(num_samples_));
}
if (start_index_ < 0) {
RETURN_STATUS_UNEXPECTED("SequentialSampler: invalid start_index: " + std::to_string(start_index_));
RETURN_STATUS_UNEXPECTED("SequentialSampler: start_index_ must be greater than or equal to 0, but got: " +
std::to_string(start_index_));
}
return Status::OK();
@ -431,7 +437,8 @@ SubsetSamplerObj::SubsetSamplerObj(std::vector<int64_t> indices, int64_t num_sam
Status SubsetSamplerObj::ValidateParams() {
if (num_samples_ < 0) {
RETURN_STATUS_UNEXPECTED("SubsetRandomSampler: invalid num_samples: " + std::to_string(num_samples_));
RETURN_STATUS_UNEXPECTED("SubsetRandomSampler: num_samples must be greater than or equal to 0, but got: " +
std::to_string(num_samples_));
}
return Status::OK();
@ -530,7 +537,8 @@ Status WeightedRandomSamplerObj::ValidateParams() {
RETURN_STATUS_UNEXPECTED("WeightedRandomSampler: elements of weights vector must not be all zero");
}
if (num_samples_ < 0) {
RETURN_STATUS_UNEXPECTED("WeightedRandomSampler: invalid num_samples: " + std::to_string(num_samples_));
RETURN_STATUS_UNEXPECTED("WeightedRandomSampler: num_samples must be greater than or equal to 0, but got: " +
std::to_string(num_samples_));
}
return Status::OK();
}

3
mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc
View File

@ -105,9 +105,6 @@ Status Resize(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *out
if (input_cv->Rank() != 3 && input_cv->Rank() != 2) {
RETURN_STATUS_UNEXPECTED("Resize: input tensor is not in shape of <H,W,C> or <H,W>");
}
if (input_cv->shape()[2] != 3 && input_cv->shape()[2] != 1) {
RETURN_STATUS_UNEXPECTED("Resize: channel of input tesnor is not in 1 or 3.");
}
cv::Mat in_image = input_cv->mat();
// resize image too large or too small

94
mindspore/dataset/engine/samplers.py
View File

@ -328,21 +328,32 @@ class DistributedSampler(BuiltinSampler):
... sampler=sampler)
Raises:
ValueError: If num_shards is not positive.
ValueError: If shard_id is smaller than 0 or equal to num_shards or larger than num_shards.
ValueError: If shuffle is not a boolean value.
ValueError: If offset is greater than num_shards.
TypeError: If num_shards is not an integer value.
TypeError: If shard_id is not an integer value.
TypeError: If shuffle is not a boolean value.
TypeError: If num_samples is not an integer value.
TypeError: If offset is not an integer value.
RuntimeError: If num_shards is not a positive value.
RuntimeError: If shard_id is smaller than 0 or equal to num_shards or larger than num_shards.
RuntimeError: If num_samples is a negative value.
RuntimeError: If offset is greater than num_shards.
"""
def __init__(self, num_shards, shard_id, shuffle=True, num_samples=None, offset=-1):
if not isinstance(num_shards, int):
raise ValueError("num_shards must be integer but was: {}.".format(num_shards))
raise TypeError("num_shards must be integer but was: {}.".format(num_shards))
if not isinstance(shard_id, int):
raise ValueError("shard_id must be integer but was: {}.".format(shard_id))
raise TypeError("shard_id must be integer but was: {}.".format(shard_id))
if not isinstance(shuffle, bool):
raise ValueError("shuffle should be a boolean value, but got shuffle: {}.".format(shuffle))
raise TypeError("shuffle must be a boolean value but was: {}.".format(shuffle))
if num_samples is not None and not isinstance(num_samples, int):
raise TypeError("num_samples must be integer but was: {}.".format(num_samples))
if not isinstance(offset, int):
raise TypeError("offset must be integer but was: {}.".format(offset))
self.num_shards = num_shards
self.shard_id = shard_id
@ -408,20 +419,30 @@ class PKSampler(BuiltinSampler):
... sampler=sampler)
Raises:
ValueError: If num_val is not positive.
TypeError: If num_val is not a positive value.
TypeError: If shuffle is not a boolean value.
TypeError: If class_column is not a str value.
TypeError: If num_samples is not an integer value.
NotImplementedError: If num_class is not None.
ValueError: If shuffle is not boolean.
RuntimeError: If num_val is not a positive value.
RuntimeError: If num_samples is a negative value.
"""
def __init__(self, num_val, num_class=None, shuffle=False, class_column='label', num_samples=None):
if not isinstance(num_val, int):
raise ValueError("num_val must be integer but was: {}.".format(num_val))
raise TypeError("num_val must be integer but was: {}.".format(num_val))
if num_class is not None:
raise NotImplementedError("Not supported to specify num_class for PKSampler.")
if not isinstance(shuffle, bool):
raise ValueError("shuffle should be a boolean value, but got shuffle: {}.".format(shuffle))
raise TypeError("shuffle must be a boolean value but was: {}.".format(shuffle))
if not isinstance(class_column, str):
raise TypeError("class_column must be a str value but was: {}.".format(class_column))
if num_samples is not None and not isinstance(num_samples, int):
raise TypeError("num_samples must be integer but was: {}.".format(num_samples))
self.num_val = num_val
self.shuffle = shuffle
@ -475,13 +496,17 @@ class RandomSampler(BuiltinSampler):
... sampler=sampler)
Raises:
ValueError: If replacement is not boolean.
ValueError: If num_samples is not positive.
TypeError: If replacement is not a boolean value.
TypeError: If num_samples is not an integer value.
RuntimeError: If num_samples is a negative value.
"""
def __init__(self, replacement=False, num_samples=None):
if not isinstance(replacement, bool):
raise ValueError("replacement should be a boolean value, but got replacement: {}.".format(replacement))
raise TypeError("replacement must be a boolean value but was: {}.".format(replacement))
if num_samples is not None and not isinstance(num_samples, int):
raise TypeError("num_samples must be integer but was: {}.".format(num_samples))
self.deterministic = False
self.replacement = replacement
@ -527,9 +552,21 @@ class SequentialSampler(BuiltinSampler):
>>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir,
... num_parallel_workers=8,
... sampler=sampler)
Raises:
TypeError: If start_index is not an integer value.
TypeError: If num_samples is not an integer value.
RuntimeError: If start_index is a negative value.
RuntimeError: If num_samples is a negative value.
"""
def __init__(self, start_index=None, num_samples=None):
if start_index is not None and not isinstance(start_index, int):
raise TypeError("start_index must be integer but was: {}.".format(start_index))
if num_samples is not None and not isinstance(num_samples, int):
raise TypeError("num_samples must be integer but was: {}.".format(num_samples))
self.start_index = start_index
super().__init__(num_samples)
@ -578,6 +615,11 @@ class SubsetSampler(BuiltinSampler):
>>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir,
... num_parallel_workers=8,
... sampler=sampler)
Raises:
TypeError: If type of indices element is not a number.
TypeError: If num_samples is not an integer value.
RuntimeError: If num_samples is a negative value.
"""
def __init__(self, indices, num_samples=None):
@ -586,9 +628,12 @@ class SubsetSampler(BuiltinSampler):
for i, item in enumerate(indices):
if not isinstance(item, numbers.Number):
raise TypeError("type of indices element should be number, "
raise TypeError("type of indices element must be number, "
"but got w[{}]: {}, type: {}.".format(i, item, type(item)))
if num_samples is not None and not isinstance(num_samples, int):
raise TypeError("num_samples must be integer but was: {}.".format(num_samples))
self.indices = indices
super().__init__(num_samples)
@ -640,6 +685,11 @@ class SubsetRandomSampler(SubsetSampler):
>>> # creates a SubsetRandomSampler, will sample from the provided indices
>>> sampler = ds.SubsetRandomSampler(indices)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
Raises:
TypeError: If type of indices element is not a number.
TypeError: If num_samples is not an integer value.
RuntimeError: If num_samples is a negative value.
"""
def parse(self):
@ -678,8 +728,11 @@ class WeightedRandomSampler(BuiltinSampler):
... sampler=sampler)
Raises:
ValueError: If num_samples is not positive.
ValueError: If replacement is not boolean.
TypeError: If type of weights element is not a number.
TypeError: If num_samples is not an integer value.
TypeError: If replacement is not a boolean value.
RuntimeError: If weights is empty or all zero.
RuntimeError: If num_samples is a negative value.
"""
def __init__(self, weights, num_samples=None, replacement=True):
@ -688,11 +741,14 @@ class WeightedRandomSampler(BuiltinSampler):
for ind, w in enumerate(weights):
if not isinstance(w, numbers.Number):
raise TypeError("type of weights element should be number, "
raise TypeError("type of weights element must be number, "
"but got w[{}]: {}, type: {}.".format(ind, w, type(w)))
if num_samples is not None and not isinstance(num_samples, int):
raise TypeError("num_samples must be integer but was: {}.".format(num_samples))
if not isinstance(replacement, bool):
raise ValueError("replacement should be a boolean value, but got replacement: {}.".format(replacement))
raise TypeError("replacement must be a boolean value but was: {}.".format(replacement))
self.weights = weights
self.replacement = replacement

11
mindspore/dataset/vision/c_transforms.py
View File

@ -616,16 +616,19 @@ class RandomColorAdjust(ImageTensorOperation):
Randomly adjust the brightness, contrast, saturation, and hue of the input image.
Args:
brightness (Union[float, tuple], optional): Brightness adjustment factor (default=(1, 1)). Cannot be negative.
brightness (Union[float, list, tuple], optional): Brightness adjustment factor (default=(1, 1)).
Cannot be negative.
If it is a float, the factor is uniformly chosen from the range [max(0, 1-brightness), 1+brightness].
If it is a sequence, it should be [min, max] for the range.
contrast (Union[float, tuple], optional): Contrast adjustment factor (default=(1, 1)). Cannot be negative.
contrast (Union[float, list, tuple], optional): Contrast adjustment factor (default=(1, 1)).
Cannot be negative.
If it is a float, the factor is uniformly chosen from the range [max(0, 1-contrast), 1+contrast].
If it is a sequence, it should be [min, max] for the range.
saturation (Union[float, tuple], optional): Saturation adjustment factor (default=(1, 1)). Cannot be negative.
saturation (Union[float, list, tuple], optional): Saturation adjustment factor (default=(1, 1)).
Cannot be negative.
If it is a float, the factor is uniformly chosen from the range [max(0, 1-saturation), 1+saturation].
If it is a sequence, it should be [min, max] for the range.
hue (Union[float, tuple], optional): Hue adjustment factor (default=(0, 0)).
hue (Union[float, list, tuple], optional): Hue adjustment factor (default=(0, 0)).
If it is a float, the range will be [-hue, hue]. Value should be 0 <= hue <= 0.5.
If it is a sequence, it should be [min, max] where -0.5 <= min <= max <= 0.5.

9
mindspore/dataset/vision/validators.py
View File

@ -141,10 +141,13 @@ def check_random_color_adjust_param(value, input_name, center=1, bound=(0, FLOAT
if isinstance(value, numbers.Number):
if value < 0:
raise ValueError("The input value of {} cannot be negative.".format(input_name))
elif isinstance(value, (list, tuple)) and len(value) == 2:
check_range(value, bound)
elif isinstance(value, (list, tuple)):
if len(value) != 2:
raise TypeError("If {0} is a sequence, the length must be 2.".format(input_name))
if value[0] > value[1]:
raise ValueError("value should be in (min,max) format. Got (max,min).")
raise ValueError("{0} value should be in (min,max) format. Got ({1}, {2}).".format(input_name,
value[0], value[1]))
check_range(value, bound)
def check_erasing_value(value):

4
tests/ut/python/dataset/test_datasets_imagefolder.py
View File

@ -391,12 +391,12 @@ def test_weighted_random_sampler_exception():
Test error cases for WeightedRandomSampler
"""
logger.info("Test error cases for WeightedRandomSampler")
error_msg_1 = "type of weights element should be number"
error_msg_1 = "type of weights element must be number"
with pytest.raises(TypeError, match=error_msg_1):
weights = ""
ds.WeightedRandomSampler(weights)
error_msg_2 = "type of weights element should be number"
error_msg_2 = "type of weights element must be number"
with pytest.raises(TypeError, match=error_msg_2):
weights = (0.9, 0.8, 1.1)
ds.WeightedRandomSampler(weights)

8
tests/ut/python/dataset/test_sampler.py
View File

@ -187,8 +187,8 @@ def test_subset_sampler():
else:
with pytest.raises(Exception) as error_info:
pipeline()
print(str(error_info))
assert exception_msg in str(error_info)
print(str(error_info.value))
assert exception_msg in str(error_info.value)
test_config([1, 2, 3])
test_config(list(range(10)))
@ -211,7 +211,7 @@ def test_subset_sampler():
test_config([0, 9, -6, 2], exception_msg="Sample ID (-6) is out of bound, expected range [0, 9]")
# test_config([], exception_msg="Indices list is empty") # temporary until we check with MindDataset
test_config([0, 9, 3, 2], num_samples=-1,
exception_msg="SubsetRandomSampler: invalid num_samples: -1")
exception_msg="SubsetRandomSampler: num_samples must be greater than or equal to 0")
def test_sampler_chain():
@ -263,7 +263,7 @@ def test_add_sampler_invalid_input():
def test_distributed_sampler_invalid_offset():
with pytest.raises(RuntimeError) as info:
sampler = ds.DistributedSampler(num_shards=4, shard_id=0, shuffle=False, num_samples=None, offset=5).parse()
assert "DistributedSampler: invalid offset: 5, which should be no more than num_shards: 4" in str(info.value)
assert "DistributedSampler: offset must be no more than num_shards(4)" in str(info.value)
def test_sampler_list():