From 6017b050ac2c59ed651ffd5999ceb7c440988af7 Mon Sep 17 00:00:00 2001 From: xiefangqi Date: Thu, 30 Jul 2020 14:40:42 +0800 Subject: [PATCH] add generatordataset support split --- .../minddata/dataset/kernels/image/image_utils.cc | 2 ++ mindspore/dataset/engine/datasets.py | 2 ++ tests/ut/python/dataset/test_pad.py | 12 ++++++++++++ 3 files changed, 16 insertions(+) diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc index acddc765d9..013e3482ea 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc @@ -875,6 +875,8 @@ Status Pad(const std::shared_ptr &input, std::shared_ptr *output std::shared_ptr output_cv; RETURN_IF_NOT_OK(CVTensor::CreateFromMat(out_image, &output_cv)); // pad the dimension if shape information is only 2 dimensional, this is grayscale + CHECK_FAIL_RETURN_UNEXPECTED(input_cv->Rank() == 3, + "Pad error: invalid image shape, only support 3 channels image."); int num_channels = input_cv->shape()[2]; if (input_cv->Rank() == 3 && num_channels == 1 && output_cv->Rank() == 2) output_cv->ExpandDim(2); *output = std::static_pointer_cast(output_cv); diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index a02d47eaa7..fb0d0ba1ca 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -2016,6 +2016,7 @@ class MapDataset(DatasetOp): new_op.columns_order = copy.deepcopy(self.columns_order, memodict) new_op.num_parallel_workers = copy.deepcopy(self.num_parallel_workers, memodict) new_op.parent = copy.deepcopy(self.parent, memodict) + new_op.ms_role = copy.deepcopy(self.ms_role, memodict) new_op.input_indexs = copy.deepcopy(self._input_indexs, memodict) new_op.python_multiprocessing = copy.deepcopy(self.python_multiprocessing, memodict) new_op.cache = copy.deepcopy(self.cache, memodict) @@ -3283,6 +3284,7 @@ class GeneratorDataset(MappableDataset): memodict[id(self)] = new_op new_op.children = copy.deepcopy(self.children, memodict) new_op.parent = copy.deepcopy(self.parent, memodict) + new_op.ms_role = copy.deepcopy(self.ms_role, memodict) new_op.num_parallel_workers = copy.deepcopy(self.num_parallel_workers, memodict) new_op.column_types = copy.deepcopy(self.column_types, memodict) new_op.column_names = copy.deepcopy(self.column_names, memodict) diff --git a/tests/ut/python/dataset/test_pad.py b/tests/ut/python/dataset/test_pad.py index a3038a4b91..d2c4e60dc5 100644 --- a/tests/ut/python/dataset/test_pad.py +++ b/tests/ut/python/dataset/test_pad.py @@ -148,8 +148,20 @@ def test_pad_md5(): filename2 = "pad_01_py_result.npz" save_and_check_md5(data2, filename2, generate_golden=GENERATE_GOLDEN) +def test_pad_exception(): + try: + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + pad_op = c_vision.Pad(150) + data1 = data1.map(input_columns=["image"], operations=pad_op) + for _ in data1.create_dict_iterator(): + pass + assert False + except RuntimeError as e: + assert "Pad error: invalid image shape, only support 3 channels image" in str(e) + if __name__ == "__main__": test_pad_op() test_pad_grayscale() test_pad_md5() + test_pad_exception()