From 1906ed1be8d9189cdc33678c2a6eb2419ead600b Mon Sep 17 00:00:00 2001 From: guansongsong Date: Tue, 28 Jul 2020 13:26:29 +0800 Subject: [PATCH] fix cache description again --- mindspore/dataset/engine/datasets.py | 3 ++ tests/ut/python/dataset/test_cache_map.py | 42 ++++++++++++----------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index f2eab10d660..93c5f7743c9 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -2144,6 +2144,7 @@ class RepeatDataset(DatasetOp): """ return self.count + class SkipDataset(DatasetOp): """ The result of applying Skip operator to the input Dataset. @@ -2409,6 +2410,7 @@ class TransferDataset(DatasetOp): def stop_send(self): self.iterator.depipeline.StopSend() + class RangeDataset(MappableDataset): """ A source dataset that reads and parses datasets stored on disk in a range. @@ -5319,6 +5321,7 @@ class BuildVocabDataset(DatasetOp): return new_op + class BuildSentencePieceVocabDataset(DatasetOp): """ Build a SentencePieceVocab from a dataset. diff --git a/tests/ut/python/dataset/test_cache_map.py b/tests/ut/python/dataset/test_cache_map.py index 4f8af60fce3..154a4208a00 100644 --- a/tests/ut/python/dataset/test_cache_map.py +++ b/tests/ut/python/dataset/test_cache_map.py @@ -24,6 +24,7 @@ DATA_DIR = "../data/dataset/testImageNetData/train/" GENERATE_GOLDEN = False + def test_cache_map_basic1(): """ Test mappable leaf with cache op right over the leaf @@ -117,28 +118,28 @@ def test_cache_map_basic3(): def test_cache_map_basic4(): - """ - Test different rows result in core dump - """ - logger.info("Test cache basic 4") - some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True) + """ + Test different rows result in core dump + """ + logger.info("Test cache basic 4") + some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True) - # This DATA_DIR only has 2 images in it - ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache) - decode_op = c_vision.Decode() - ds1 = ds1.repeat(4) - ds1 = ds1.map(input_columns=["image"], operations=decode_op) - logger.info("ds1.dataset_size is ", ds1.get_dataset_size()) - shape = ds1.output_shapes() - logger.info(shape) - num_iter = 0 - for _ in ds1.create_dict_iterator(): - logger.info("get data from dataset") - num_iter += 1 + # This DATA_DIR only has 2 images in it + ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache) + decode_op = c_vision.Decode() + ds1 = ds1.repeat(4) + ds1 = ds1.map(input_columns=["image"], operations=decode_op) + logger.info("ds1.dataset_size is ", ds1.get_dataset_size()) + shape = ds1.output_shapes() + logger.info(shape) + num_iter = 0 + for _ in ds1.create_dict_iterator(): + logger.info("get data from dataset") + num_iter += 1 - logger.info("Number of data in ds1: {} ".format(num_iter)) - assert num_iter == 8 - logger.info('test_cache_basic3 Ended.\n') + logger.info("Number of data in ds1: {} ".format(num_iter)) + assert num_iter == 8 + logger.info('test_cache_basic3 Ended.\n') def test_cache_map_failure1(): @@ -177,6 +178,7 @@ def test_cache_map_failure1(): assert num_iter == 0 logger.info('test_cache_failure1 Ended.\n') + if __name__ == '__main__': test_cache_map_basic1() logger.info("test_cache_map_basic1 success.")