diff --git a/tests/ut/python/dataset/test_allpass_biquad.py b/tests/ut/python/dataset/test_allpass_biquad.py index 29805ab6df3..716b4a85623 100644 --- a/tests/ut/python/dataset/test_allpass_biquad.py +++ b/tests/ut/python/dataset/test_allpass_biquad.py @@ -56,7 +56,7 @@ def test_func_allpass_biquad_pipeline(): # Filtered waveform by allpassbiquad dataset = dataset.map(input_columns=["channel"], operations=allpass_biquad_op, num_parallel_workers=8) i = 0 - for item in dataset.create_dict_iterator(output_numpy=True): + for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): count_unequal_element(expect_waveform[i, :], item['channel'], 0.0001, 0.0001) i += 1 diff --git a/tests/ut/python/dataset/test_angle.py b/tests/ut/python/dataset/test_angle.py index 1de8e2fd0a2..2cefde88048 100755 --- a/tests/ut/python/dataset/test_angle.py +++ b/tests/ut/python/dataset/test_angle.py @@ -57,7 +57,7 @@ def test_func_angle_002(): dataset = ds.NumpySlicesDataset(data, column_names=["col1", "col2"], shuffle=False) angle_op = a_c_trans.Angle() dataset = dataset.map(operations=angle_op, input_columns=["col1"]) - for item1, item2 in zip(dataset.create_dict_iterator(output_numpy=True), expected): + for item1, item2 in zip(dataset.create_dict_iterator(num_epochs=1, output_numpy=True), expected): count_unequal_element(item2, item1['col1'], 0.0001, 0.0001) @@ -74,7 +74,7 @@ def test_func_angle_003(): dataset = dataset.map(operations=angle_op, input_columns=["col1"]) num_itr = 0 with pytest.raises(RuntimeError, match="input tensor type should be int, float or double"): - for _ in dataset.create_dict_iterator(output_numpy=True): + for _ in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): num_itr += 1 diff --git a/tests/ut/python/dataset/test_band_biquad.py b/tests/ut/python/dataset/test_band_biquad.py index 6136159cbd7..c21633cd210 100644 --- a/tests/ut/python/dataset/test_band_biquad.py +++ b/tests/ut/python/dataset/test_band_biquad.py @@ -59,7 +59,7 @@ def test_func_band_biquad_pipeline(): dataset = dataset.map( input_columns=["channel"], operations=band_biquad_op, num_parallel_workers=8) i = 0 - for item in dataset.create_dict_iterator(output_numpy=True): + for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): count_unequal_element(expect_waveform[i, :], item['channel'], 0.0001, 0.0001) i += 1 diff --git a/tests/ut/python/dataset/test_bandpass_biquad.py b/tests/ut/python/dataset/test_bandpass_biquad.py index caa8277dc35..809785e8635 100644 --- a/tests/ut/python/dataset/test_bandpass_biquad.py +++ b/tests/ut/python/dataset/test_bandpass_biquad.py @@ -58,7 +58,7 @@ def test_func_bandpass_biquad_pipeline(): # Filtered waveform by bandpassbiquad dataset = dataset.map(input_columns=["channel"], operations=bandpass_biquad_op, num_parallel_workers=8) i = 0 - for item in dataset.create_dict_iterator(output_numpy=True): + for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): count_unequal_element(expect_waveform[i, :], item['channel'], 0.0001, 0.0001) i += 1 diff --git a/tests/ut/python/dataset/test_bandreject_biquad.py b/tests/ut/python/dataset/test_bandreject_biquad.py index af04d34de25..038e646b160 100644 --- a/tests/ut/python/dataset/test_bandreject_biquad.py +++ b/tests/ut/python/dataset/test_bandreject_biquad.py @@ -61,7 +61,7 @@ def test_func_bandreject_biquad_pipeline(): dataset = dataset.map( input_columns=["channel"], operations=bandreject_biquad_op, num_parallel_workers=8) i = 0 - for item in dataset.create_dict_iterator(output_numpy=True): + for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): count_unequal_element(expect_waveform[i, :], item['channel'], 0.0001, 0.0001) i += 1 diff --git a/tests/ut/python/dataset/test_bass_biquad.py b/tests/ut/python/dataset/test_bass_biquad.py index 41f1e7c87cf..fd7a2e2bbba 100644 --- a/tests/ut/python/dataset/test_bass_biquad.py +++ b/tests/ut/python/dataset/test_bass_biquad.py @@ -59,7 +59,7 @@ def test_func_bass_biquad_pipeline(): dataset = dataset.map( input_columns=["channel"], operations=bass_biquad_op, num_parallel_workers=8) i = 0 - for item in dataset.create_dict_iterator(output_numpy=True): + for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): count_unequal_element(expect_waveform[i, :], item['channel'], 0.0001, 0.0001) i += 1 diff --git a/tests/ut/python/dataset/test_biquad.py b/tests/ut/python/dataset/test_biquad.py index 7ab3e75635e..14f5ae3b5a6 100644 --- a/tests/ut/python/dataset/test_biquad.py +++ b/tests/ut/python/dataset/test_biquad.py @@ -56,7 +56,7 @@ def test_func_biquad_pipeline(): # Filtered waveform by biquad dataset = dataset.map(input_columns=["audio"], operations=biquad_op, num_parallel_workers=8) i = 0 - for item in dataset.create_dict_iterator(output_numpy=True): + for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): count_unequal_element(expect_waveform[i, :], item['audio'], 0.0001, 0.0001) i += 1 diff --git a/tests/ut/python/dataset/test_cache_map.py b/tests/ut/python/dataset/test_cache_map.py index d278ca657ff..ad2e0579f18 100644 --- a/tests/ut/python/dataset/test_cache_map.py +++ b/tests/ut/python/dataset/test_cache_map.py @@ -168,7 +168,7 @@ def test_cache_map_basic4(): data = data.repeat(4) num_iter = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -283,7 +283,7 @@ def test_cache_map_failure2(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in dsz.create_dict_iterator(): + for _ in dsz.create_dict_iterator(num_epochs=1): num_iter += 1 assert "ZipNode is not supported as a descendant operator under a cache" in str(e.value) @@ -322,7 +322,7 @@ def test_cache_map_failure3(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 assert "BatchNode is not supported as a descendant operator under a cache" in str(e.value) @@ -364,7 +364,7 @@ def test_cache_map_failure4(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 assert "FilterNode is not supported as a descendant operator under a cache" in str(e.value) @@ -405,7 +405,7 @@ def test_cache_map_failure5(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_iter += 1 assert "MapNode containing random operation is not supported as a descendant of cache" in str(e.value) @@ -446,7 +446,7 @@ def test_cache_map_failure7(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_iter += 1 assert "There is currently no support for GeneratorOp under cache" in str(e.value) @@ -524,7 +524,7 @@ def test_cache_map_failure9(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 assert "TakeNode (possibly from Split) is not supported as a descendant operator under a cache" in str(e.value) @@ -566,7 +566,7 @@ def test_cache_map_failure10(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 assert "SkipNode is not supported as a descendant operator under a cache" in str(e.value) @@ -597,7 +597,7 @@ def test_cache_map_failure11(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 assert "Unexpected error. Server is not set up with spill support" in str(e.value) @@ -646,13 +646,13 @@ def test_cache_map_split1(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 assert "TakeNode (possibly from Split) is not supported as a descendant operator under a cache" in str(e.value) with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in ds2.create_dict_iterator(): + for _ in ds2.create_dict_iterator(num_epochs=1): num_iter += 1 assert "TakeNode (possibly from Split) is not supported as a descendant operator under a cache" in str(e.value) logger.info('test_cache_split1 Ended.\n') @@ -694,12 +694,12 @@ def test_cache_map_split2(): ds2 = ds2.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 assert num_iter == 12 num_iter = 0 - for _ in ds2.create_dict_iterator(): + for _ in ds2.create_dict_iterator(num_epochs=1): num_iter += 1 assert num_iter == 24 logger.info('test_cache_split2 Ended.\n') @@ -805,13 +805,13 @@ def test_cache_map_running_twice1(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) assert num_iter == 8 num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) assert num_iter == 8 @@ -848,7 +848,7 @@ def test_cache_map_running_twice2(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -885,7 +885,7 @@ def test_cache_map_extra_small_size1(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -922,7 +922,7 @@ def test_cache_map_extra_small_size2(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -960,7 +960,7 @@ def test_cache_map_no_image(): with pytest.raises(RuntimeError): num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 assert num_iter == 0 @@ -996,7 +996,7 @@ def test_cache_map_parallel_pipeline1(shard): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -1033,7 +1033,7 @@ def test_cache_map_parallel_pipeline2(shard): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -1070,7 +1070,7 @@ def test_cache_map_parallel_workers(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -1107,7 +1107,7 @@ def test_cache_map_server_workers_1(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -1144,7 +1144,7 @@ def test_cache_map_server_workers_100(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -1181,7 +1181,7 @@ def test_cache_map_num_connections_1(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -1218,7 +1218,7 @@ def test_cache_map_num_connections_100(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -1255,7 +1255,7 @@ def test_cache_map_prefetch_size_1(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -1292,7 +1292,7 @@ def test_cache_map_prefetch_size_100(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -1404,7 +1404,7 @@ def test_cache_map_epoch_ctrl2(): num_epoch = 5 # iter1 will always assume there is a next epoch and never shutdown - iter1 = ds1.create_dict_iterator() + iter1 = ds1.create_dict_iterator(num_epochs=-1) epoch_count = 0 for _ in range(num_epoch): @@ -1451,7 +1451,7 @@ def test_cache_map_epoch_ctrl3(): num_epoch = 5 # iter1 will always assume there is a next epoch and never shutdown - iter1 = ds1.create_dict_iterator() + iter1 = ds1.create_dict_iterator(num_epochs=num_epoch) epoch_count = 0 for _ in range(num_epoch): @@ -2097,7 +2097,7 @@ def test_cache_map_python_sampler1(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) assert num_iter == 8 @@ -2133,7 +2133,7 @@ def test_cache_map_python_sampler2(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) assert num_iter == 8 @@ -2200,7 +2200,7 @@ def test_cache_map_interrupt_and_rerun(): some_cache = ds.DatasetCache(session_id=session_id, size=0) ds1 = ds.Cifar10Dataset(CIFAR10_DATA_DIR, cache=some_cache) - iter1 = ds1.create_dict_iterator() + iter1 = ds1.create_dict_iterator(num_epochs=-1) num_iter = 0 with pytest.raises(AttributeError) as e: @@ -2252,7 +2252,7 @@ def test_cache_map_dataset_size1(): assert dataset_size == 2 num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -2289,7 +2289,7 @@ def test_cache_map_dataset_size2(): assert dataset_size == 2 num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) diff --git a/tests/ut/python/dataset/test_cache_nomap.py b/tests/ut/python/dataset/test_cache_nomap.py index 7758efce89d..786d7c3e3ac 100644 --- a/tests/ut/python/dataset/test_cache_nomap.py +++ b/tests/ut/python/dataset/test_cache_nomap.py @@ -629,13 +629,13 @@ def test_cache_nomap_running_twice1(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) assert num_iter == 12 num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) assert num_iter == 12 @@ -672,7 +672,7 @@ def test_cache_nomap_running_twice2(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -708,7 +708,7 @@ def test_cache_nomap_extra_small_size1(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -886,7 +886,7 @@ def test_cache_nomap_server_workers_1(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -923,7 +923,7 @@ def test_cache_nomap_server_workers_100(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -960,7 +960,7 @@ def test_cache_nomap_num_connections_1(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -997,7 +997,7 @@ def test_cache_nomap_num_connections_100(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -1034,7 +1034,7 @@ def test_cache_nomap_prefetch_size_1(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -1071,7 +1071,7 @@ def test_cache_nomap_prefetch_size_100(): ds1 = ds1.repeat(4) num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -1147,7 +1147,7 @@ def test_cache_nomap_session_destroy(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 assert "Unexpected error" in str(e.value) @@ -1185,7 +1185,7 @@ def test_cache_nomap_server_stop(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 assert "Network error. Cache server with port 50052 is unreachable. Make sure the server is running." in \ str(e.value) @@ -1218,7 +1218,7 @@ def test_cache_nomap_interrupt_and_rerun(): # User-created sampler here ds1 = ds.RandomDataset(schema=schema, total_rows=10000, num_parallel_workers=4, cache=some_cache) - iter1 = ds1.create_dict_iterator() + iter1 = ds1.create_dict_iterator(num_epochs=-1) num_iter = 0 with pytest.raises(AttributeError) as e: @@ -1312,7 +1312,7 @@ def test_cache_nomap_epoch_ctrl2(): num_epoch = 5 # iter1 will always assume there is a next epoch and never shutdown - iter1 = ds1.create_dict_iterator() + iter1 = ds1.create_dict_iterator(num_epochs=-1) epoch_count = 0 for _ in range(num_epoch): @@ -1359,7 +1359,7 @@ def test_cache_nomap_epoch_ctrl3(): num_epoch = 5 # iter1 will always assume there is a next epoch and never shutdown - iter1 = ds1.create_dict_iterator() + iter1 = ds1.create_dict_iterator(num_epochs=num_epoch) epoch_count = 0 for _ in range(num_epoch): @@ -2052,7 +2052,7 @@ def test_cache_nomap_failure2(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in dsz.create_dict_iterator(): + for _ in dsz.create_dict_iterator(num_epochs=1): num_iter += 1 assert "ZipNode is not supported as a descendant operator under a cache" in str(e.value) @@ -2091,7 +2091,7 @@ def test_cache_nomap_failure3(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 assert "BatchNode is not supported as a descendant operator under a cache" in str(e.value) @@ -2133,7 +2133,7 @@ def test_cache_nomap_failure4(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 assert "FilterNode is not supported as a descendant operator under a cache" in str(e.value) @@ -2173,7 +2173,7 @@ def test_cache_nomap_failure5(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_iter += 1 assert "MapNode containing random operation is not supported as a descendant of cache" in str(e.value) @@ -2217,7 +2217,7 @@ def test_cache_nomap_pyfunc_lambda(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in ds2.create_dict_iterator(): + for _ in ds2.create_dict_iterator(num_epochs=1): num_iter += 1 assert "MapNode containing random operation is not supported as a descendant of cache" in str(e.value) logger.info("test_cache_nomap_pyfunc_lambda Ended.\n") @@ -2259,7 +2259,7 @@ def test_cache_nomap_pyfunc_builtin(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in ds2.create_dict_iterator(): + for _ in ds2.create_dict_iterator(num_epochs=1): num_iter += 1 assert "MapNode containing random operation is not supported as a descendant of cache" in str(e.value) logger.info("test_cache_nomap_pyfunc_builtin Ended.\n") @@ -2308,7 +2308,7 @@ def test_cache_nomap_pyfunc_function(): with pytest.raises(RuntimeError) as e: num_iter = 0 - for _ in ds2.create_dict_iterator(): + for _ in ds2.create_dict_iterator(num_epochs=1): num_iter += 1 assert "MapNode containing random operation is not supported as a descendant of cache" in str(e.value) logger.info("test_cache_nomap_pyfunc_function Ended.\n") @@ -2341,7 +2341,7 @@ def test_cache_nomap_all_rows_cached(): num_total_rows = 271 # User-created sampler here ds1 = ds.RandomDataset(schema=schema, total_rows=num_total_rows, num_parallel_workers=4, cache=some_cache) - iter1 = ds1.create_dict_iterator() + iter1 = ds1.create_dict_iterator(num_epochs=1) num_iter = 0 for _ in iter1: @@ -2380,7 +2380,7 @@ def test_cache_nomap_dataset_size1(): assert dataset_size == 2 num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -2417,7 +2417,7 @@ def test_cache_nomap_dataset_size2(): assert dataset_size == 2 num_iter = 0 - for _ in ds1.create_dict_iterator(): + for _ in ds1.create_dict_iterator(num_epochs=1): num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) diff --git a/tests/ut/python/dataset/test_compose.py b/tests/ut/python/dataset/test_compose.py index 4067eed87b7..191b9b15ae3 100644 --- a/tests/ut/python/dataset/test_compose.py +++ b/tests/ut/python/dataset/test_compose.py @@ -39,7 +39,7 @@ def test_compose(): data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False) data = data.map(input_columns=["col"], operations=op_list) res = [] - for i in data.create_dict_iterator(output_numpy=True): + for i in data.create_dict_iterator(num_epochs=1, output_numpy=True): res.append(i["col"].tolist()) return res except (TypeError, ValueError) as e: @@ -114,7 +114,7 @@ def test_lambdas(): data = data.map(operations=op_list, input_columns=input_columns, output_columns=output_cols, column_order=output_cols) res = [] - for i in data.create_dict_iterator(output_numpy=True): + for i in data.create_dict_iterator(num_epochs=1, output_numpy=True): for col_name in output_cols: res.append(i[col_name].tolist()) return res @@ -141,7 +141,7 @@ def test_c_py_compose_transforms_module(): data = data.map(operations=op_list, input_columns=input_columns, output_columns=output_cols, column_order=output_cols) res = [] - for i in data.create_dict_iterator(output_numpy=True): + for i in data.create_dict_iterator(num_epochs=1, output_numpy=True): for col_name in output_cols: res.append(i[col_name].tolist()) return res @@ -229,7 +229,7 @@ def test_py_transforms_with_c_vision(): data = ds.ImageFolderDataset(dataset_dir=data_dir, shuffle=False) data = data.map(operations=op_list) res = [] - for i in data.create_dict_iterator(output_numpy=True): + for i in data.create_dict_iterator(num_epochs=1, output_numpy=True): for col_name in output_cols: res.append(i[col_name].tolist()) return res @@ -323,7 +323,7 @@ def test_compose_with_custom_function(): # res = [] - for i in data.create_dict_iterator(output_numpy=True): + for i in data.create_dict_iterator(num_epochs=1, output_numpy=True): res.append(i["col0"].tolist()) assert res == [[[3, 6], [9, 36]]] diff --git a/tests/ut/python/dataset/test_concat.py b/tests/ut/python/dataset/test_concat.py index dd2c668965e..b907523bb57 100644 --- a/tests/ut/python/dataset/test_concat.py +++ b/tests/ut/python/dataset/test_concat.py @@ -57,7 +57,7 @@ def test_concat_01(): data3 = data1 + data2 # Here i refers to index, d refers to data element - for i, d in enumerate(data3.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(data3.create_tuple_iterator(num_epochs=1, output_numpy=True)): t = d logger.info("data: %i", t[0][0]) assert i == t[0][0] @@ -76,7 +76,7 @@ def test_concat_02(): data3 = data1.concat(data2) # Here i refers to index, d refers to data element - for i, d in enumerate(data3.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(data3.create_tuple_iterator(num_epochs=1, output_numpy=True)): t = d logger.info("data: %i", t[0][0]) assert i == t[0][0] @@ -154,7 +154,7 @@ def test_concat_06(): dataset = data1 + data2 + data3 # Here i refers to index, d refers to data element - for i, d in enumerate(dataset.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(dataset.create_tuple_iterator(num_epochs=1, output_numpy=True)): t = d logger.info("data: %i", t[0][0]) assert i == t[0][0] @@ -175,7 +175,7 @@ def test_concat_07(): data4 = data1 + dataset # Here i refers to index, d refers to data element - for i, d in enumerate(data4.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(data4.create_tuple_iterator(num_epochs=1, output_numpy=True)): t = d logger.info("data: %i", t[0][0]) assert i == t[0][0] @@ -195,7 +195,7 @@ def test_concat_08(): data3 = data3.repeat(2) # Here i refers to index, d refers to data element - for i, d in enumerate(data3.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(data3.create_tuple_iterator(num_epochs=1, output_numpy=True)): t = d logger.info("data: %i", t[0][0]) assert i % 10 == t[0][0] @@ -217,7 +217,7 @@ def test_concat_09(): res = [0, 1, 2, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 3, 4, 5, 6, 7, 8, 9] # Here i refers to index, d refers to data element - for i, d in enumerate(data3.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(data3.create_tuple_iterator(num_epochs=1, output_numpy=True)): t = d logger.info("data: %i", t[0][0]) assert res[i] == t[0][0] @@ -238,7 +238,7 @@ def test_concat_10(): res = [0, 1, 2, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9] # Here i refers to index, d refers to data element - for i, d in enumerate(data3.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(data3.create_tuple_iterator(num_epochs=1, output_numpy=True)): t = d logger.info("data: %i", t[0][0]) assert res[i] == t[0][0] @@ -261,7 +261,7 @@ def test_concat_11(): res = [0, 10, 15, 20] # Here i refers to index, d refers to data element - for i, d in enumerate(data3.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(data3.create_tuple_iterator(num_epochs=1, output_numpy=True)): t = d logger.info("data: %i", t[0][0]) assert res[i] == t[0][0] @@ -285,7 +285,7 @@ def test_concat_12(): data3 = data3.shuffle(buffer_size=10) # Here i refers to index, d refers to data element - for i, d in enumerate(data3.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(data3.create_tuple_iterator(num_epochs=1, output_numpy=True)): t = d logger.info("data: %i", t[0][0]) assert res[i] == t[0][0] @@ -313,7 +313,7 @@ def test_concat_13(): data3 = data3.shuffle(buffer_size=int(data3.get_dataset_size())) # Here i refers to index, d refers to data element - for i, d in enumerate(data3.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(data3.create_tuple_iterator(num_epochs=1, output_numpy=True)): t = d logger.info("data: %i", t[0][0]) assert res[i] == t[0][0] @@ -341,11 +341,11 @@ def test_concat_14(): data3 = data1 + data2 expected, output = [], [] - for d in data1.create_tuple_iterator(output_numpy=True): + for d in data1.create_tuple_iterator(num_epochs=1, output_numpy=True): expected.append(d[0]) - for d in data2.create_tuple_iterator(output_numpy=True): + for d in data2.create_tuple_iterator(num_epochs=1, output_numpy=True): expected.append(d[0]) - for d in data3.create_tuple_iterator(output_numpy=True): + for d in data3.create_tuple_iterator(num_epochs=1, output_numpy=True): output.append(d[0]) assert len(expected) == len(output) diff --git a/tests/ut/python/dataset/test_concatenate_op.py b/tests/ut/python/dataset/test_concatenate_op.py index 1026ab6eda1..34c29d61ae2 100644 --- a/tests/ut/python/dataset/test_concatenate_op.py +++ b/tests/ut/python/dataset/test_concatenate_op.py @@ -34,7 +34,7 @@ def test_concatenate_op_all(): data = data.map(operations=concatenate_op, input_columns=["col"]) expected = np.array([1.4, 2., 3., 4., 4.5, 5., 6., 7., 8., 9., 10.3, 11., 12.]) - for data_row in data.create_tuple_iterator(output_numpy=True): + for data_row in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(data_row[0], expected) @@ -46,7 +46,7 @@ def test_concatenate_op_none(): concatenate_op = data_trans.Concatenate() data = data.map(operations=concatenate_op, input_columns=["col"]) - for data_row in data.create_tuple_iterator(output_numpy=True): + for data_row in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(data_row[0], np.array([5., 6., 7., 8.], dtype=np.float)) @@ -61,7 +61,7 @@ def test_concatenate_op_string(): data = data.map(operations=concatenate_op, input_columns=["col"]) expected = np.array(["dw", "df", "ss", "ad", "dwsdf", "df"], dtype='S') - for data_row in data.create_tuple_iterator(output_numpy=True): + for data_row in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(data_row[0], expected) @@ -77,7 +77,7 @@ def test_concatenate_op_multi_input_string(): data = data.map(operations=concatenate_op, input_columns=["col1", "col2"], column_order=["out1"], output_columns=["out1"]) expected = np.array(["dw", "df", "1", "2", "d", "3", "4", "e", "dwsdf", "df"], dtype='S') - for data_row in data.create_tuple_iterator(output_numpy=True): + for data_row in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(data_row[0], expected) @@ -92,7 +92,7 @@ def test_concatenate_op_multi_input_numeric(): data = data.map(operations=concatenate_op, input_columns=["col1", "col2"], column_order=["out1"], output_columns=["out1"]) expected = np.array([3, 5, 1, 2, 3, 4]) - for data_row in data.create_tuple_iterator(output_numpy=True): + for data_row in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(data_row[0], expected) @@ -158,7 +158,7 @@ def test_concatenate_op_negative_axis(): data = data.map(operations=concatenate_op, input_columns=["col"]) expected = np.array([1.4, 2., 3., 4., 4.5, 5., 6., 7., 8., 9., 10.3, 11., 12.]) - for data_row in data.create_tuple_iterator(output_numpy=True): + for data_row in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(data_row[0], expected) diff --git a/tests/ut/python/dataset/test_contrast.py b/tests/ut/python/dataset/test_contrast.py index b3fbad63acd..441ffff5694 100644 --- a/tests/ut/python/dataset/test_contrast.py +++ b/tests/ut/python/dataset/test_contrast.py @@ -60,7 +60,7 @@ def test_func_contrast_pipeline(): # Filtered waveform by contrast dataset = dataset.map(input_columns=["audio"], operations=contrast_op, num_parallel_workers=8) i = 0 - for item in dataset.create_dict_iterator(output_numpy=True): + for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): count_unequal_element(expect_waveform[i, :], item['audio'], 0.0001, 0.0001) i += 1 diff --git a/tests/ut/python/dataset/test_dataset_numpy_slices.py b/tests/ut/python/dataset/test_dataset_numpy_slices.py index 4d36db50448..4c483cdda0d 100644 --- a/tests/ut/python/dataset/test_dataset_numpy_slices.py +++ b/tests/ut/python/dataset/test_dataset_numpy_slices.py @@ -77,7 +77,7 @@ def test_numpy_slices_list_append(): ds = de.NumpySlicesDataset(res, column_names=["col1"], shuffle=False) - for i, data in enumerate(ds.create_tuple_iterator(output_numpy=True)): + for i, data in enumerate(ds.create_tuple_iterator(num_epochs=1, output_numpy=True)): assert np.equal(data, res[i]).all() @@ -99,7 +99,7 @@ def test_numpy_slices_tuple_1(): np_data = [([1, 2], [3, 4]), ([11, 12], [13, 14]), ([21, 22], [23, 24])] ds = de.NumpySlicesDataset(np_data, shuffle=False) - for i, data in enumerate(ds.create_tuple_iterator(output_numpy=True)): + for i, data in enumerate(ds.create_tuple_iterator(num_epochs=1, output_numpy=True)): assert np.equal(data, np_data[i]).all() assert sum([1 for _ in ds]) == 3 @@ -112,7 +112,7 @@ def test_numpy_slices_tuple_2(): expected = [[1, 3, 5], [2, 4, 6]] ds = de.NumpySlicesDataset(np_data, shuffle=False) - for i, data in enumerate(ds.create_tuple_iterator(output_numpy=True)): + for i, data in enumerate(ds.create_tuple_iterator(num_epochs=1, output_numpy=True)): assert np.equal(data, expected[i]).all() assert sum([1 for _ in ds]) == 2 @@ -156,7 +156,7 @@ def test_numpy_slices_csv_dict(): ds = de.NumpySlicesDataset(dict(df), shuffle=False) - for i, data in enumerate(ds.create_tuple_iterator(output_numpy=True)): + for i, data in enumerate(ds.create_tuple_iterator(num_epochs=1, output_numpy=True)): assert np.equal(data, res[i]).all() diff --git a/tests/ut/python/dataset/test_datasets_celeba.py b/tests/ut/python/dataset/test_datasets_celeba.py index 4f20d6873fe..7112178fea2 100644 --- a/tests/ut/python/dataset/test_datasets_celeba.py +++ b/tests/ut/python/dataset/test_datasets_celeba.py @@ -142,7 +142,7 @@ def test_celeba_dataset_exception_file_path(): try: data = ds.CelebADataset(DATA_DIR, shuffle=False) data = data.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): pass assert False except RuntimeError as e: @@ -152,7 +152,7 @@ def test_celeba_dataset_exception_file_path(): data = ds.CelebADataset(DATA_DIR, shuffle=False) data = data.map(operations=vision.Decode(), input_columns=["image"], num_parallel_workers=1) data = data.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): pass assert False except RuntimeError as e: @@ -161,7 +161,7 @@ def test_celeba_dataset_exception_file_path(): try: data = ds.CelebADataset(DATA_DIR, shuffle=False) data = data.map(operations=exception_func, input_columns=["attr"], num_parallel_workers=1) - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): pass assert False except RuntimeError as e: @@ -175,7 +175,7 @@ def test_celeba_sampler_exception(): logger.info("Test CelebA with bad sampler input") try: data = ds.CelebADataset(DATA_DIR, sampler="") - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): pass assert False except TypeError as e: diff --git a/tests/ut/python/dataset/test_datasets_cifarop.py b/tests/ut/python/dataset/test_datasets_cifarop.py index 10d1a8aaba6..b478b435f79 100644 --- a/tests/ut/python/dataset/test_datasets_cifarop.py +++ b/tests/ut/python/dataset/test_datasets_cifarop.py @@ -454,7 +454,7 @@ def test_cifar_exception_file_path(): data = ds.Cifar10Dataset(DATA_DIR_10) data = data.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) num_rows = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_rows += 1 assert False except RuntimeError as e: @@ -464,7 +464,7 @@ def test_cifar_exception_file_path(): data = ds.Cifar10Dataset(DATA_DIR_10) data = data.map(operations=exception_func, input_columns=["label"], num_parallel_workers=1) num_rows = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_rows += 1 assert False except RuntimeError as e: @@ -474,7 +474,7 @@ def test_cifar_exception_file_path(): data = ds.Cifar100Dataset(DATA_DIR_100) data = data.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) num_rows = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_rows += 1 assert False except RuntimeError as e: @@ -484,7 +484,7 @@ def test_cifar_exception_file_path(): data = ds.Cifar100Dataset(DATA_DIR_100) data = data.map(operations=exception_func, input_columns=["coarse_label"], num_parallel_workers=1) num_rows = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_rows += 1 assert False except RuntimeError as e: @@ -494,7 +494,7 @@ def test_cifar_exception_file_path(): data = ds.Cifar100Dataset(DATA_DIR_100) data = data.map(operations=exception_func, input_columns=["fine_label"], num_parallel_workers=1) num_rows = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_rows += 1 assert False except RuntimeError as e: diff --git a/tests/ut/python/dataset/test_datasets_cityscapes.py b/tests/ut/python/dataset/test_datasets_cityscapes.py index 3d20c25139e..e8cfe9f91c8 100644 --- a/tests/ut/python/dataset/test_datasets_cityscapes.py +++ b/tests/ut/python/dataset/test_datasets_cityscapes.py @@ -222,7 +222,7 @@ def test_cityscapes_exception(): data = ds.CityscapesDataset(DATASET_DIR, usage=usage, quality_mode=quality_mode, task=task) data = data.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) num_rows = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_rows += 1 assert False except RuntimeError as e: @@ -232,7 +232,7 @@ def test_cityscapes_exception(): data = ds.CityscapesDataset(DATASET_DIR, usage=usage, quality_mode=quality_mode, task=task) data = data.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) num_rows = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_rows += 1 assert False except RuntimeError as e: diff --git a/tests/ut/python/dataset/test_datasets_clue.py b/tests/ut/python/dataset/test_datasets_clue.py index ca485a58dea..d3f3a0f061d 100644 --- a/tests/ut/python/dataset/test_datasets_clue.py +++ b/tests/ut/python/dataset/test_datasets_clue.py @@ -378,7 +378,7 @@ def test_clue_exception_file_path(): try: data = ds.CLUEDataset(TRAIN_FILE, task='AFQMC', usage='train') data = data.map(operations=exception_func, input_columns=["label"], num_parallel_workers=1) - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): pass assert False except RuntimeError as e: @@ -387,7 +387,7 @@ def test_clue_exception_file_path(): try: data = ds.CLUEDataset(TRAIN_FILE, task='AFQMC', usage='train') data = data.map(operations=exception_func, input_columns=["sentence1"], num_parallel_workers=1) - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): pass assert False except RuntimeError as e: @@ -396,7 +396,7 @@ def test_clue_exception_file_path(): try: data = ds.CLUEDataset(TRAIN_FILE, task='AFQMC', usage='train') data = data.map(operations=exception_func, input_columns=["sentence2"], num_parallel_workers=1) - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): pass assert False except RuntimeError as e: diff --git a/tests/ut/python/dataset/test_datasets_coco.py b/tests/ut/python/dataset/test_datasets_coco.py index 9fa4caf6a3f..033a2732333 100644 --- a/tests/ut/python/dataset/test_datasets_coco.py +++ b/tests/ut/python/dataset/test_datasets_coco.py @@ -179,22 +179,22 @@ def test_coco_panoptic(): def test_coco_meta_column(): data1 = ds.CocoDataset(DATA_DIR, annotation_file=ANNOTATION_FILE, task="Detection", decode=True, shuffle=False, extra_metadata=True) - for item in data1.create_tuple_iterator(): + for item in data1.create_tuple_iterator(num_epochs=1): assert len(item) == 4 data2 = ds.CocoDataset(DATA_DIR, annotation_file=ANNOTATION_FILE, task="Stuff", decode=True, shuffle=False, extra_metadata=True) - for item in data2.create_tuple_iterator(): + for item in data2.create_tuple_iterator(num_epochs=1): assert len(item) == 3 data3 = ds.CocoDataset(DATA_DIR, annotation_file=KEYPOINT_FILE, task="Keypoint", decode=True, shuffle=False, extra_metadata=True) - for item in data3.create_tuple_iterator(): + for item in data3.create_tuple_iterator(num_epochs=1): assert len(item) == 3 data4 = ds.CocoDataset(DATA_DIR, annotation_file=PANOPTIC_FILE, task="Panoptic", decode=True, shuffle=False, extra_metadata=True) - for item in data4.create_tuple_iterator(): + for item in data4.create_tuple_iterator(num_epochs=1): assert len(item) == 5 @@ -204,7 +204,7 @@ def test_coco_detection_classindex(): assert class_index == {'person': [1], 'bicycle': [2], 'car': [3], 'cat': [4], 'dog': [5], 'monkey': [6], 'bag': [7], 'orange': [8]} num_iter = 0 - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): num_iter += 1 assert num_iter == 6 @@ -214,7 +214,7 @@ def test_coco_panootic_classindex(): class_index = data1.get_class_indexing() assert class_index == {'person': [1, 1], 'bicycle': [2, 1], 'car': [3, 1]} num_iter = 0 - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): num_iter += 1 assert num_iter == 2 @@ -252,7 +252,7 @@ def test_coco_case_2(): data1 = data1.map(operations=resize_op, input_columns=["image"]) data1 = data1.repeat(4) num_iter = 0 - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): num_iter += 1 assert num_iter == 24 @@ -264,7 +264,7 @@ def test_coco_case_3(): data1 = data1.map(operations=resize_op, input_columns=["image"]) data1 = data1.repeat(4) num_iter = 0 - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): num_iter += 1 assert num_iter == 24 @@ -272,7 +272,7 @@ def test_coco_case_3(): def test_coco_case_exception(): try: data1 = ds.CocoDataset("path_not_exist/", annotation_file=ANNOTATION_FILE, task="Detection") - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except ValueError as e: @@ -280,7 +280,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file="./file_not_exist", task="Detection") - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except ValueError as e: @@ -288,7 +288,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=ANNOTATION_FILE, task="Invalid task") - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except ValueError as e: @@ -296,7 +296,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=LACKOFIMAGE_FILE, task="Detection") - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -304,7 +304,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=INVALID_CATEGORY_ID_FILE, task="Detection") - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -312,7 +312,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=INVALID_FILE, task="Detection") - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -321,7 +321,7 @@ def test_coco_case_exception(): try: sampler = ds.PKSampler(3) data1 = ds.CocoDataset(DATA_DIR, annotation_file=INVALID_FILE, task="Detection", sampler=sampler) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except ValueError as e: @@ -333,7 +333,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=ANNOTATION_FILE, task="Detection") data1 = data1.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -343,7 +343,7 @@ def test_coco_case_exception(): data1 = ds.CocoDataset(DATA_DIR, annotation_file=ANNOTATION_FILE, task="Detection") data1 = data1.map(operations=vision.Decode(), input_columns=["image"], num_parallel_workers=1) data1 = data1.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -352,7 +352,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=ANNOTATION_FILE, task="Detection") data1 = data1.map(operations=exception_func, input_columns=["bbox"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -361,7 +361,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=ANNOTATION_FILE, task="Detection") data1 = data1.map(operations=exception_func, input_columns=["category_id"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -370,7 +370,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=ANNOTATION_FILE, task="Stuff") data1 = data1.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -380,7 +380,7 @@ def test_coco_case_exception(): data1 = ds.CocoDataset(DATA_DIR, annotation_file=ANNOTATION_FILE, task="Stuff") data1 = data1.map(operations=vision.Decode(), input_columns=["image"], num_parallel_workers=1) data1 = data1.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -389,7 +389,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=ANNOTATION_FILE, task="Stuff") data1 = data1.map(operations=exception_func, input_columns=["segmentation"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -398,7 +398,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=ANNOTATION_FILE, task="Stuff") data1 = data1.map(operations=exception_func, input_columns=["iscrowd"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -407,7 +407,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=KEYPOINT_FILE, task="Keypoint") data1 = data1.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -417,7 +417,7 @@ def test_coco_case_exception(): data1 = ds.CocoDataset(DATA_DIR, annotation_file=KEYPOINT_FILE, task="Keypoint") data1 = data1.map(operations=vision.Decode(), input_columns=["image"], num_parallel_workers=1) data1 = data1.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -426,7 +426,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=KEYPOINT_FILE, task="Keypoint") data1 = data1.map(operations=exception_func, input_columns=["keypoints"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -435,7 +435,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=KEYPOINT_FILE, task="Keypoint") data1 = data1.map(operations=exception_func, input_columns=["num_keypoints"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -444,7 +444,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=PANOPTIC_FILE, task="Panoptic") data1 = data1.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -454,7 +454,7 @@ def test_coco_case_exception(): data1 = ds.CocoDataset(DATA_DIR, annotation_file=PANOPTIC_FILE, task="Panoptic") data1 = data1.map(operations=vision.Decode(), input_columns=["image"], num_parallel_workers=1) data1 = data1.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -463,7 +463,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=PANOPTIC_FILE, task="Panoptic") data1 = data1.map(operations=exception_func, input_columns=["bbox"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -472,7 +472,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=PANOPTIC_FILE, task="Panoptic") data1 = data1.map(operations=exception_func, input_columns=["category_id"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -481,7 +481,7 @@ def test_coco_case_exception(): try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=PANOPTIC_FILE, task="Panoptic") data1 = data1.map(operations=exception_func, input_columns=["area"], num_parallel_workers=1) - for _ in data1.create_dict_iterator(output_numpy=True): + for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: diff --git a/tests/ut/python/dataset/test_datasets_div2k.py b/tests/ut/python/dataset/test_datasets_div2k.py index aa1bb9848d3..3d0816a8870 100644 --- a/tests/ut/python/dataset/test_datasets_div2k.py +++ b/tests/ut/python/dataset/test_datasets_div2k.py @@ -211,7 +211,7 @@ def test_div2k_exception(): data = ds.DIV2KDataset(DATASET_DIR, usage=usage, downgrade=downgrade, scale=scale) data = data.map(operations=exception_func, input_columns=["hr_image"], num_parallel_workers=1) num_rows = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_rows += 1 assert False except RuntimeError as e: @@ -221,7 +221,7 @@ def test_div2k_exception(): data = ds.DIV2KDataset(DATASET_DIR, usage=usage, downgrade=downgrade, scale=scale) data = data.map(operations=exception_func, input_columns=["hr_image"], num_parallel_workers=1) num_rows = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_rows += 1 assert False except RuntimeError as e: diff --git a/tests/ut/python/dataset/test_datasets_flickr.py b/tests/ut/python/dataset/test_datasets_flickr.py index 0b4c4c2de79..c3a18061433 100644 --- a/tests/ut/python/dataset/test_datasets_flickr.py +++ b/tests/ut/python/dataset/test_datasets_flickr.py @@ -130,7 +130,7 @@ def test_flickr30k_dataset_exception(): data = ds.FlickrDataset(FLICKR30K_DATASET_DIR, FLICKR30K_ANNOTATION_FILE_1, decode=True) data = data.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) num_rows = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_rows += 1 assert False except RuntimeError as e: @@ -140,7 +140,7 @@ def test_flickr30k_dataset_exception(): data = ds.FlickrDataset(FLICKR30K_DATASET_DIR, FLICKR30K_ANNOTATION_FILE_1, decode=True) data = data.map(operations=exception_func, input_columns=["annotation"], num_parallel_workers=1) num_rows = 0 - for _ in data.create_dict_iterator(): + for _ in data.create_dict_iterator(num_epochs=1): num_rows += 1 assert False except RuntimeError as e: diff --git a/tests/ut/python/dataset/test_datasets_generator.py b/tests/ut/python/dataset/test_datasets_generator.py index 37ab75a8f0e..d99599c6d2b 100644 --- a/tests/ut/python/dataset/test_datasets_generator.py +++ b/tests/ut/python/dataset/test_datasets_generator.py @@ -898,7 +898,7 @@ def test_func_generator_dataset_005(): column_names = ["col1", "col2"] dataset = ds.GeneratorDataset(MyData(result), column_names) i = 0 - for data in dataset.create_dict_iterator(output_numpy=True): + for data in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): assert "col1" in str(data.keys()) assert (data["col1"] == result[0]).all() assert (data["col2"] == result[1]).all() diff --git a/tests/ut/python/dataset/test_datasets_get_dataset_size.py b/tests/ut/python/dataset/test_datasets_get_dataset_size.py index 415e916de24..f47408e58b9 100644 --- a/tests/ut/python/dataset/test_datasets_get_dataset_size.py +++ b/tests/ut/python/dataset/test_datasets_get_dataset_size.py @@ -64,14 +64,14 @@ def test_imagenet_tf_file_dataset_size(): assert ds_shard_3_0.get_dataset_size() == 4 count = 0 - for _ in ds_shard_3_0.create_dict_iterator(): + for _ in ds_shard_3_0.create_dict_iterator(num_epochs=1): count += 1 assert ds_shard_3_0.get_dataset_size() == count # shard_equal_rows is set to False therefore, get_dataset_size must return count ds_shard_4_0 = ds.TFRecordDataset(IMAGENET_TFFILE_DIR, num_shards=4, shard_id=0) count = 0 - for _ in ds_shard_4_0.create_dict_iterator(): + for _ in ds_shard_4_0.create_dict_iterator(num_epochs=1): count += 1 assert ds_shard_4_0.get_dataset_size() == count @@ -254,7 +254,7 @@ def test_distributed_get_dataset_size(): assert dataset1.get_dataset_size() == 2000 count1 = 0 - for _ in dataset1.create_dict_iterator(): + for _ in dataset1.create_dict_iterator(num_epochs=1): count1 += 1 assert count1 == 2000 @@ -263,7 +263,7 @@ def test_distributed_get_dataset_size(): assert dataset2.get_dataset_size() == 2500 count2 = 0 - for _ in dataset2.create_dict_iterator(): + for _ in dataset2.create_dict_iterator(num_epochs=1): count2 += 1 assert count2 == 2500 diff --git a/tests/ut/python/dataset/test_datasets_voc.py b/tests/ut/python/dataset/test_datasets_voc.py index 83c1e0d6017..c73dbd15ee4 100644 --- a/tests/ut/python/dataset/test_datasets_voc.py +++ b/tests/ut/python/dataset/test_datasets_voc.py @@ -90,7 +90,7 @@ def test_voc_meta_column(): # scenario one: output 2 columns if without rename meta column data1 = ds.VOCDataset(DATA_DIR, task="Segmentation", usage="train", decode=True, shuffle=False, extra_metadata=True) num = 0 - for item in data1.create_tuple_iterator(): + for item in data1.create_tuple_iterator(num_epochs=1): assert len(item) == 2 num += 1 @@ -102,7 +102,7 @@ def test_voc_meta_column(): data2 = data2.map(operations=pyfunc1, input_columns=["image", "target"]) data2 = data2.rename("_meta-filename", "filename") num = 0 - for item in data2.create_tuple_iterator(output_numpy=True): + for item in data2.create_tuple_iterator(num_epochs=1, output_numpy=True): assert text.to_str(item[2]) == IMAGE_ID[num] num += 1 @@ -115,7 +115,7 @@ def test_voc_meta_column(): column_order=["_meta-filename", "img1", "img2", "label"]) data3 = data3.rename("_meta-filename", "filename") num = 0 - for item in data3.create_tuple_iterator(output_numpy=True): + for item in data3.create_tuple_iterator(num_epochs=1, output_numpy=True): assert text.to_str(item[0]) == IMAGE_ID[num] num += 1 @@ -128,7 +128,7 @@ def test_voc_meta_column(): column_order=["_meta-filename", "img1"]) data4 = data4.rename("_meta-filename", "filename") num = 0 - for item in data4.create_tuple_iterator(output_numpy=True): + for item in data4.create_tuple_iterator(num_epochs=1, output_numpy=True): assert text.to_str(item[0]) == IMAGE_ID[num] num += 1 @@ -248,7 +248,7 @@ def test_voc_exception(): try: data = ds.VOCDataset(DATA_DIR, task="Detection", usage="train", shuffle=False) data = data.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) - for _ in data.create_dict_iterator(output_numpy=True): + for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -258,7 +258,7 @@ def test_voc_exception(): data = ds.VOCDataset(DATA_DIR, task="Detection", usage="train", shuffle=False) data = data.map(operations=vision.Decode(), input_columns=["image"], num_parallel_workers=1) data = data.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) - for _ in data.create_dict_iterator(output_numpy=True): + for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -267,7 +267,7 @@ def test_voc_exception(): try: data = ds.VOCDataset(DATA_DIR, task="Detection", usage="train", shuffle=False) data = data.map(operations=exception_func, input_columns=["bbox"], num_parallel_workers=1) - for _ in data.create_dict_iterator(output_numpy=True): + for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -276,7 +276,7 @@ def test_voc_exception(): try: data = ds.VOCDataset(DATA_DIR, task="Detection", usage="train", shuffle=False) data = data.map(operations=exception_func, input_columns=["difficult"], num_parallel_workers=1) - for _ in data.create_dict_iterator(output_numpy=True): + for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -285,7 +285,7 @@ def test_voc_exception(): try: data = ds.VOCDataset(DATA_DIR, task="Detection", usage="train", shuffle=False) data = data.map(operations=exception_func, input_columns=["truncate"], num_parallel_workers=1) - for _ in data.create_dict_iterator(output_numpy=True): + for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -294,7 +294,7 @@ def test_voc_exception(): try: data = ds.VOCDataset(DATA_DIR, task="Segmentation", usage="train", shuffle=False) data = data.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) - for _ in data.create_dict_iterator(output_numpy=True): + for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -304,7 +304,7 @@ def test_voc_exception(): data = ds.VOCDataset(DATA_DIR, task="Segmentation", usage="train", shuffle=False) data = data.map(operations=vision.Decode(), input_columns=["image"], num_parallel_workers=1) data = data.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) - for _ in data.create_dict_iterator(output_numpy=True): + for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -313,7 +313,7 @@ def test_voc_exception(): try: data = ds.VOCDataset(DATA_DIR, task="Segmentation", usage="train", shuffle=False) data = data.map(operations=exception_func, input_columns=["target"], num_parallel_workers=1) - for _ in data.create_dict_iterator(output_numpy=True): + for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: @@ -323,7 +323,7 @@ def test_voc_exception(): data = ds.VOCDataset(DATA_DIR, task="Segmentation", usage="train", shuffle=False) data = data.map(operations=vision.Decode(), input_columns=["target"], num_parallel_workers=1) data = data.map(operations=exception_func, input_columns=["target"], num_parallel_workers=1) - for _ in data.create_dict_iterator(output_numpy=True): + for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True): pass assert False except RuntimeError as e: diff --git a/tests/ut/python/dataset/test_dc_shift.py b/tests/ut/python/dataset/test_dc_shift.py index 61feae6be9e..c7cb1bc7a20 100644 --- a/tests/ut/python/dataset/test_dc_shift.py +++ b/tests/ut/python/dataset/test_dc_shift.py @@ -50,7 +50,7 @@ def test_func_dc_shift_pipeline(): dataset = ds.NumpySlicesDataset(arr, column_names=["col1"], shuffle=False) dcshift_op = a_c_trans.DCShift(0.8, 0.03) dataset = dataset.map(operations=dcshift_op, input_columns=["col1"]) - for item1, item2 in zip(dataset.create_dict_iterator(output_numpy=True), expected): + for item1, item2 in zip(dataset.create_dict_iterator(num_epochs=1, output_numpy=True), expected): count_unequal_element(item2, item1['col1'], 0.0001, 0.0001) @@ -66,7 +66,7 @@ def test_func_dc_shift_pipeline_error(): with pytest.raises(ValueError, match=r"Input shift is not within the required interval of \[-2.0, 2.0\]."): dcshift_op = a_c_trans.DCShift(2.5, 0.03) dataset = dataset.map(operations=dcshift_op, input_columns=["col1"]) - for _ in dataset.create_dict_iterator(output_numpy=True): + for _ in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): num_itr += 1 diff --git a/tests/ut/python/dataset/test_deemph_biquad.py b/tests/ut/python/dataset/test_deemph_biquad.py index f3826224f95..3d167adbf47 100644 --- a/tests/ut/python/dataset/test_deemph_biquad.py +++ b/tests/ut/python/dataset/test_deemph_biquad.py @@ -56,7 +56,7 @@ def test_func_deemph_biquad_pipeline(): # Filtered waveform by deemphbiquad dataset = dataset.map(input_columns=["audio"], operations=deemph_biquad_op, num_parallel_workers=8) i = 0 - for data in dataset.create_dict_iterator(output_numpy=True): + for data in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): count_unequal_element(expect_waveform[i, :], data['audio'], 0.0001, 0.0001) i += 1 diff --git a/tests/ut/python/dataset/test_epoch_ctrl.py b/tests/ut/python/dataset/test_epoch_ctrl.py index b36b3136b40..90054554f89 100644 --- a/tests/ut/python/dataset/test_epoch_ctrl.py +++ b/tests/ut/python/dataset/test_epoch_ctrl.py @@ -50,8 +50,7 @@ def test_cifar10(): data1 = data1.repeat(num_repeat) data1 = data1.batch(batch_size, True) num_epoch = 5 - # iter1 will always assume there is a next epoch and never shutdown. - iter1 = data1.create_tuple_iterator() + iter1 = data1.create_tuple_iterator(num_epochs=num_epoch) epoch_count = 0 sample_count = 0 for _ in range(num_epoch): @@ -86,7 +85,7 @@ def test_decode_op(): num_epoch = 5 # iter1 will always assume there is a next epoch and never shutdown. - iter1 = data1.create_dict_iterator(output_numpy=True) + iter1 = data1.create_dict_iterator(num_epochs=-1, output_numpy=True) # iter 2 will stop and shutdown pipeline after num_epoch iter2 = data2.create_dict_iterator(num_epoch, output_numpy=True) for _ in range(num_epoch): @@ -169,7 +168,9 @@ def test_generator_dict_2(): # apply dataset operations data1 = ds.GeneratorDataset(generator_1d, ["data"]) - iter1 = data1.create_dict_iterator() + + # iter1 will always assume there is a next epoch and never shutdown + iter1 = data1.create_dict_iterator(num_epochs=-1) for _ in range(10): i = 0 for item in iter1: # each data is a dictionary @@ -192,7 +193,9 @@ def test_generator_dict_3(): # apply dataset operations data1 = ds.GeneratorDataset(generator_1d, ["data"]) - iter1 = data1.create_dict_iterator() + + # iter1 will always assume there is a next epoch and never shutdown + iter1 = data1.create_dict_iterator(num_epochs=-1) for _ in range(10): i = 0 for item in iter1: # each data is a dictionary @@ -200,7 +203,7 @@ def test_generator_dict_3(): np.testing.assert_array_equal(item["data"].asnumpy(), golden) i = i + 1 assert i == 64 - # optional + iter1.stop() # Expect a AttributeError since iter1 has been stopped. with pytest.raises(AttributeError) as info: @@ -360,7 +363,8 @@ def test_generator_tuple_2(): # apply dataset operations data1 = ds.GeneratorDataset(generator_1d, ["data"]) - iter1 = data1.create_tuple_iterator(output_numpy=True) + # iter1 will always assume there is a next epoch and never shutdown + iter1 = data1.create_tuple_iterator(num_epochs=-1, output_numpy=True) for _ in range(10): i = 0 for item in iter1: # each data is a dictionary @@ -383,7 +387,8 @@ def test_generator_tuple_3(): # apply dataset operations data1 = ds.GeneratorDataset(generator_1d, ["data"]) - iter1 = data1.create_tuple_iterator(output_numpy=True) + # iter1 will always assume there is a next epoch and never shutdown + iter1 = data1.create_tuple_iterator(num_epochs=-1, output_numpy=True) for _ in range(10): i = 0 for item in iter1: # each data is a dictionary @@ -391,7 +396,7 @@ def test_generator_tuple_3(): np.testing.assert_array_equal(item[0], golden) i = i + 1 assert i == 64 - # optional + iter1.stop() # Expect a AttributeError since iter1 has been stopped. with pytest.raises(AttributeError) as info: @@ -533,7 +538,8 @@ def test_generator_tuple_repeat_repeat_2(): data1 = ds.GeneratorDataset(generator_1d, ["data"]) data1 = data1.repeat(2) data1 = data1.repeat(3) - iter1 = data1.create_tuple_iterator(output_numpy=True) + # iter1 will always assume there is a next epoch and never shutdown + iter1 = data1.create_tuple_iterator(num_epochs=-1, output_numpy=True) for _ in range(10): i = 0 for item in iter1: # each data is a dictionary @@ -541,7 +547,7 @@ def test_generator_tuple_repeat_repeat_2(): np.testing.assert_array_equal(item[0], golden) i = i + 1 assert i == 64 * 2 * 3 - # optional + iter1.stop() # Expect a AttributeError since iter1 has been stopped. with pytest.raises(AttributeError) as info: @@ -559,7 +565,7 @@ def test_generator_tuple_repeat_repeat_3(): data1 = ds.GeneratorDataset(generator_1d, ["data"]) data1 = data1.repeat(2) data1 = data1.repeat(3) - iter1 = data1.create_tuple_iterator(output_numpy=True) + iter1 = data1.create_tuple_iterator(num_epochs=15, output_numpy=True) for _ in range(10): i = 0 for item in iter1: # each data is a dictionary @@ -658,7 +664,7 @@ def test_generator_tuple_infinite_repeat_repeat_4(): data1 = ds.GeneratorDataset(generator_1d, ["data"]) data1 = data1.repeat() data1 = data1.repeat() - iter1 = data1.create_tuple_iterator(output_numpy=True) + iter1 = data1.create_tuple_iterator(num_epochs=1, output_numpy=True) i = 0 for item in iter1: # each data is a dictionary @@ -680,7 +686,7 @@ def test_generator_reusedataset(): # apply dataset operations data1 = ds.GeneratorDataset(generator_1d, ["data"]) data1 = data1.repeat(2) - iter1 = data1.create_tuple_iterator(output_numpy=True) + iter1 = data1.create_tuple_iterator(num_epochs=10, output_numpy=True) for _ in range(10): i = 0 for item in iter1: # each data is a dictionary @@ -690,7 +696,7 @@ def test_generator_reusedataset(): assert i == 64 * 2 data1 = data1.repeat(3) - iter1 = data1.create_tuple_iterator(output_numpy=True) + iter1 = data1.create_tuple_iterator(num_epochs=5, output_numpy=True) for _ in range(5): i = 0 for item in iter1: # each data is a dictionary @@ -700,7 +706,7 @@ def test_generator_reusedataset(): assert i == 64 * 2 * 3 data1 = data1.batch(2) - iter1 = data1.create_dict_iterator(output_numpy=True) + iter1 = data1.create_dict_iterator(num_epochs=5, output_numpy=True) for _ in range(5): i = 0 sample = 0 diff --git a/tests/ut/python/dataset/test_equalizer_biquad.py b/tests/ut/python/dataset/test_equalizer_biquad.py index 5f35f2f45c3..9dbb364c6f0 100644 --- a/tests/ut/python/dataset/test_equalizer_biquad.py +++ b/tests/ut/python/dataset/test_equalizer_biquad.py @@ -60,7 +60,7 @@ def test_equalizer_biquad_pipeline(): # Filtered waveform by equalizer_biquad dataset = dataset.map(input_columns=["col1"], operations=equalizer_biquad_op, num_parallel_workers=4) i = 0 - for item in dataset.create_dict_iterator(output_numpy=True): + for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): count_unequal_element(expect_waveform[i, :], item["col1"], 0.0001, 0.0001) i += 1 diff --git a/tests/ut/python/dataset/test_fill_op.py b/tests/ut/python/dataset/test_fill_op.py index f500b266449..1edf5135e08 100644 --- a/tests/ut/python/dataset/test_fill_op.py +++ b/tests/ut/python/dataset/test_fill_op.py @@ -69,7 +69,7 @@ def test_fillop_string(): data = data.map(operations=fill_op, input_columns=["col"]) expected = np.array(['error', 'error'], dtype='S') - for data_row in data.create_tuple_iterator(output_numpy=True): + for data_row in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(data_row[0], expected) @@ -82,7 +82,7 @@ def test_fillop_bytes(): data = data.map(operations=fill_op, input_columns=["col"]) expected = np.array([b'abc', b'abc', b'abc'], dtype='S') - for data_row in data.create_tuple_iterator(output_numpy=True): + for data_row in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(data_row[0], expected) diff --git a/tests/ut/python/dataset/test_flat_map.py b/tests/ut/python/dataset/test_flat_map.py index a154ae1ecad..6a0585222b2 100644 --- a/tests/ut/python/dataset/test_flat_map.py +++ b/tests/ut/python/dataset/test_flat_map.py @@ -34,7 +34,7 @@ def test_flat_map_1(): data = data.flat_map(flat_map_func) count = 0 - for d in data.create_tuple_iterator(output_numpy=True): + for d in data.create_tuple_iterator(num_epochs=1, output_numpy=True): assert isinstance(d[0], np.ndarray) count += 1 assert count == 52 @@ -60,7 +60,7 @@ def test_flat_map_2(): data = data.flat_map(flat_map_func_2) count = 0 - for d in data.create_tuple_iterator(output_numpy=True): + for d in data.create_tuple_iterator(num_epochs=1, output_numpy=True): assert isinstance(d[0], np.ndarray) count += 1 assert count == 104 diff --git a/tests/ut/python/dataset/test_generator_reset_pass.py b/tests/ut/python/dataset/test_generator_reset_pass.py index 9ee6da15057..2a6c7be44df 100644 --- a/tests/ut/python/dataset/test_generator_reset_pass.py +++ b/tests/ut/python/dataset/test_generator_reset_pass.py @@ -121,9 +121,9 @@ def test_generator_reset_3(): concat1 = branch2 + branch3 concat2 = branch1 + concat1.repeat(3).skip(5).take(15) - itr = concat2.create_dict_iterator(output_numpy=True) - num_epochs = 5 + itr = concat2.create_dict_iterator(num_epochs=num_epochs, output_numpy=True) + output = np.array([0]) golden = np.array([0]) expected = np.array([2, 1, 2, 1, 12, 22, 23, 10, 11, 12, 10, 11, 12, 22, 23, 10, 11, 12, 10]) @@ -164,7 +164,7 @@ def test_generator_reset_5(): num_epochs = 2 output = np.array([0]) - itr = branch1.create_dict_iterator(output_numpy=True) + itr = branch1.create_dict_iterator(num_epochs=num_epochs, output_numpy=True) for _ in range(num_epochs): for item in itr: diff --git a/tests/ut/python/dataset/test_highpass_biquad.py b/tests/ut/python/dataset/test_highpass_biquad.py index 84088dc2e23..364f7a3a2de 100755 --- a/tests/ut/python/dataset/test_highpass_biquad.py +++ b/tests/ut/python/dataset/test_highpass_biquad.py @@ -63,7 +63,7 @@ def test_highpass_biquad_pipeline(): dataset = dataset.map( input_columns=["col1"], operations=highpass_biquad_op, num_parallel_workers=4) i = 0 - for item in dataset.create_dict_iterator(output_numpy=True): + for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): count_unequal_element(expect_waveform[i, :], item["col1"], 0.0001, 0.0001) i += 1 diff --git a/tests/ut/python/dataset/test_iterator.py b/tests/ut/python/dataset/test_iterator.py index 7a228843388..1e6a447a591 100644 --- a/tests/ut/python/dataset/test_iterator.py +++ b/tests/ut/python/dataset/test_iterator.py @@ -127,25 +127,25 @@ def test_iterator_weak_ref(): def test_iterator_exception(): data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR) try: - _ = data.create_dict_iterator(output_numpy="123") + _ = data.create_dict_iterator(num_epochs=1, output_numpy="123") assert False except TypeError as e: assert "Argument output_numpy with value 123 is not of type" in str(e) try: - _ = data.create_dict_iterator(output_numpy=123) + _ = data.create_dict_iterator(num_epochs=1, output_numpy=123) assert False except TypeError as e: assert "Argument output_numpy with value 123 is not of type" in str(e) try: - _ = data.create_tuple_iterator(output_numpy="123") + _ = data.create_tuple_iterator(num_epochs=1, output_numpy="123") assert False except TypeError as e: assert "Argument output_numpy with value 123 is not of type" in str(e) try: - _ = data.create_tuple_iterator(output_numpy=123) + _ = data.create_tuple_iterator(num_epochs=1, output_numpy=123) assert False except TypeError as e: assert "Argument output_numpy with value 123 is not of type" in str(e) diff --git a/tests/ut/python/dataset/test_lfilter.py b/tests/ut/python/dataset/test_lfilter.py index 90b7fb29b20..207ca15491c 100644 --- a/tests/ut/python/dataset/test_lfilter.py +++ b/tests/ut/python/dataset/test_lfilter.py @@ -61,7 +61,7 @@ def test_func_lfilter_pipeline(): # Filtered waveform by lfilter dataset = dataset.map(input_columns=["channel"], operations=lfilter_op, num_parallel_workers=8) i = 0 - for data in dataset.create_dict_iterator(output_numpy=True): + for data in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): count_unequal_element(expect_waveform[i, :], data['channel'], 0.0001, 0.0001) i += 1 diff --git a/tests/ut/python/dataset/test_lowpass_biquad.py b/tests/ut/python/dataset/test_lowpass_biquad.py index 9a918ba23e2..6c41a737107 100644 --- a/tests/ut/python/dataset/test_lowpass_biquad.py +++ b/tests/ut/python/dataset/test_lowpass_biquad.py @@ -62,7 +62,7 @@ def test_lowpass_biquad_pipeline(): dataset = dataset.map( input_columns=["col1"], operations=lowpass_biquad_op, num_parallel_workers=4) i = 0 - for _ in dataset.create_dict_iterator(output_numpy=True): + for _ in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): count_unequal_element(expect_waveform[i, :], _["col1"], 0.0001, 0.0001) i += 1 diff --git a/tests/ut/python/dataset/test_minddataset.py b/tests/ut/python/dataset/test_minddataset.py index af147dd8798..c53408ca6a4 100644 --- a/tests/ut/python/dataset/test_minddataset.py +++ b/tests/ut/python/dataset/test_minddataset.py @@ -843,7 +843,7 @@ def test_cv_minddataset_reader_basic_tutorial_5_epoch(add_and_remove_cv_file): assert data_set.get_dataset_size() == 10 for _ in range(5): num_iter = 0 - for data in data_set.create_tuple_iterator(output_numpy=True): + for data in data_set.create_tuple_iterator(num_epochs=1, output_numpy=True): logger.info("data is {}".format(data)) num_iter += 1 assert num_iter == 10 @@ -871,7 +871,7 @@ def test_cv_minddataset_reader_basic_tutorial_5_epoch_with_batch(add_and_remove_ assert data_set.get_dataset_size() == 5 for _ in range(5): num_iter = 0 - for data in data_set.create_tuple_iterator(output_numpy=True): + for data in data_set.create_tuple_iterator(num_epochs=1, output_numpy=True): logger.info("data is {}".format(data)) num_iter += 1 assert num_iter == 5 diff --git a/tests/ut/python/dataset/test_onehot_op.py b/tests/ut/python/dataset/test_onehot_op.py index dd9ed822943..2020c05f594 100644 --- a/tests/ut/python/dataset/test_onehot_op.py +++ b/tests/ut/python/dataset/test_onehot_op.py @@ -121,7 +121,7 @@ def test_one_hot_success(): trans = py_trans.Compose([one_hot_encode]) dataset = dataset.map(operations=trans, input_columns=["label"]) - for index, item in enumerate(dataset.create_dict_iterator(output_numpy=True)): + for index, item in enumerate(dataset.create_dict_iterator(num_epochs=1, output_numpy=True)): assert item["label"][index] == 1.0 def test_one_hot_success2(): @@ -146,7 +146,7 @@ def test_one_hot_success2(): trans = py_trans.Compose([one_hot_encode]) dataset = dataset.map(operations=trans, input_columns=["label"]) - for index, item in enumerate(dataset.create_dict_iterator(output_numpy=True)): + for index, item in enumerate(dataset.create_dict_iterator(num_epochs=1, output_numpy=True)): logger.info(item) assert item["label"][0][index] == 1.0 @@ -175,7 +175,7 @@ def test_one_hot_success3(): trans = py_trans.Compose([one_hot_encode]) dataset = dataset.map(operations=trans, input_columns=["label"]) - for item in dataset.create_dict_iterator(output_numpy=True): + for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): logger.info(item) for i in range(10): assert item["label"][i][0][i] == 1.0 @@ -203,7 +203,7 @@ def test_one_hot_type_error(): dataset = dataset.map(operations=trans, input_columns=["label"]) try: - for index, item in enumerate(dataset.create_dict_iterator(output_numpy=True)): + for index, item in enumerate(dataset.create_dict_iterator(num_epochs=1, output_numpy=True)): assert item["label"][index] == 1.0 except RuntimeError as e: assert "the input numpy type should be int" in str(e) diff --git a/tests/ut/python/dataset/test_opt.py b/tests/ut/python/dataset/test_opt.py index 579418c9121..80bafd8f4aa 100644 --- a/tests/ut/python/dataset/test_opt.py +++ b/tests/ut/python/dataset/test_opt.py @@ -39,7 +39,7 @@ def test_case_0(): data1 = data1.batch(2) expected_data = np.array([[[1], [2]], [[3], [0]]]) - for i, data_row in enumerate(data1.create_tuple_iterator(output_numpy=True)): + for i, data_row in enumerate(data1.create_tuple_iterator(num_epochs=1, output_numpy=True)): np.testing.assert_array_equal(data_row[0], expected_data[i]) # Restore configuration diff --git a/tests/ut/python/dataset/test_opt_pass.py b/tests/ut/python/dataset/test_opt_pass.py index 710254c6ca3..f3402e3279b 100644 --- a/tests/ut/python/dataset/test_opt_pass.py +++ b/tests/ut/python/dataset/test_opt_pass.py @@ -67,7 +67,8 @@ def test_shuffle(): data2 = ds.TFRecordDataset(FILES, schema=SCHEMA_FILE, shuffle=ds.Shuffle.FILES) data2 = data2.shuffle(10000) - for d1, d2 in zip(data1.create_tuple_iterator(output_numpy=True), data2.create_tuple_iterator(output_numpy=True)): + for d1, d2 in zip(data1.create_tuple_iterator(num_epochs=1, output_numpy=True), + data2.create_tuple_iterator(num_epochs=1, output_numpy=True)): for t1, t2 in zip(d1, d2): np.testing.assert_array_equal(t1, t2) @@ -77,7 +78,8 @@ def test_shuffle(): data2 = ds.TextFileDataset(DATA_ALL_FILE, shuffle=ds.Shuffle.FILES) data2 = data2.shuffle(10000) - for d1, d2 in zip(data1.create_tuple_iterator(output_numpy=True), data2.create_tuple_iterator(output_numpy=True)): + for d1, d2 in zip(data1.create_tuple_iterator(num_epochs=1, output_numpy=True), + data2.create_tuple_iterator(num_epochs=1, output_numpy=True)): for t1, t2 in zip(d1, d2): np.testing.assert_array_equal(t1, t2) @@ -87,7 +89,8 @@ def test_shuffle(): data2 = ds.CLUEDataset(TRAIN_FILE, task='AFQMC', usage='train', shuffle=ds.Shuffle.FILES) data2 = data2.shuffle(10000) - for d1, d2 in zip(data1.create_tuple_iterator(output_numpy=True), data2.create_tuple_iterator(output_numpy=True)): + for d1, d2 in zip(data1.create_tuple_iterator(num_epochs=1, output_numpy=True), + data2.create_tuple_iterator(num_epochs=1, output_numpy=True)): for t1, t2 in zip(d1, d2): np.testing.assert_array_equal(t1, t2) diff --git a/tests/ut/python/dataset/test_pad_end_op.py b/tests/ut/python/dataset/test_pad_end_op.py index d1b0c081890..05df7524e26 100644 --- a/tests/ut/python/dataset/test_pad_end_op.py +++ b/tests/ut/python/dataset/test_pad_end_op.py @@ -30,7 +30,7 @@ def pad_compare(array, pad_shape, pad_value, res): data = data.map(operations=ops.PadEnd(pad_shape, pad_value)) else: data = data.map(operations=ops.PadEnd(pad_shape)) - for d in data.create_tuple_iterator(output_numpy=True): + for d in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(res, d[0]) diff --git a/tests/ut/python/dataset/test_profiling.py b/tests/ut/python/dataset/test_profiling.py index c09fb58dd1f..e945315ca55 100644 --- a/tests/ut/python/dataset/test_profiling.py +++ b/tests/ut/python/dataset/test_profiling.py @@ -191,9 +191,9 @@ def test_profiling_inline_ops_pipeline1(): try: num_iter = 0 - # Note: Do not explicitly set num_epochs argument in create_tuple_iterator() call + # Note: set num_epochs=2 in create_tuple_iterator(), so that EpochCtrl op is added to the pipeline # Here i refers to index, d refers to data element - for i, d in enumerate(data3.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(data3.create_tuple_iterator(num_epochs=2, output_numpy=True)): num_iter += 1 t = d assert i == t[0][0] @@ -326,7 +326,7 @@ def test_profiling_basic_pipeline(): try: num_iter = 0 - # Note: If create_tuple_iterator() is called with num_epochs>1, then EpochCtrlOp is added to the pipeline + # Note: If create_dict_iterator() is called with num_epochs>1, then EpochCtrlOp is added to the pipeline for _ in data1.create_dict_iterator(num_epochs=2): num_iter += 1 @@ -379,7 +379,7 @@ def test_profiling_cifar10_pipeline(): try: num_iter = 0 - # Note: If create_tuple_iterator() is called with num_epochs=1, then EpochCtrlOp is NOT added to the pipeline + # Note: If create_dict_iterator() is called with num_epochs=1, then EpochCtrlOp is NOT added to the pipeline for _ in data1.create_dict_iterator(num_epochs=1): num_iter += 1 @@ -426,7 +426,7 @@ def test_profiling_seq_pipelines_epochctrl3(): try: # Test A - Call create_dict_iterator with num_epochs>1 num_iter = 0 - # Note: If create_tuple_iterator() is called with num_epochs>1, then EpochCtrlOp is added to the pipeline + # Note: If create_dict_iterator() is called with num_epochs>1, then EpochCtrlOp is added to the pipeline for _ in data1.create_dict_iterator(num_epochs=2): num_iter += 1 assert num_iter == 2 @@ -437,7 +437,7 @@ def test_profiling_seq_pipelines_epochctrl3(): # Test B - Call create_dict_iterator with num_epochs=1 num_iter = 0 - # Note: If create_tuple_iterator() is called with num_epochs=1, + # Note: If create_dict_iterator() is called with num_epochs=1, # then EpochCtrlOp should not be NOT added to the pipeline for _ in data1.create_dict_iterator(num_epochs=1): num_iter += 1 @@ -470,7 +470,7 @@ def test_profiling_seq_pipelines_epochctrl2(): try: # Test A - Call create_dict_iterator with num_epochs=1 num_iter = 0 - # Note: If create_tuple_iterator() is called with num_epochs=1, then EpochCtrlOp is NOT added to the pipeline + # Note: If create_dict_iterator() is called with num_epochs=1, then EpochCtrlOp is NOT added to the pipeline for _ in data2.create_dict_iterator(num_epochs=1): num_iter += 1 assert num_iter == 4 @@ -481,7 +481,7 @@ def test_profiling_seq_pipelines_epochctrl2(): # Test B - Call create_dict_iterator with num_epochs>1 num_iter = 0 - # Note: If create_tuple_iterator() is called with num_epochs>1, + # Note: If create_dict_iterator() is called with num_epochs>1, # then EpochCtrlOp should be added to the pipeline for _ in data2.create_dict_iterator(num_epochs=2): num_iter += 1 diff --git a/tests/ut/python/dataset/test_pyfunc.py b/tests/ut/python/dataset/test_pyfunc.py index 96ebc3fc5c9..d57d16c5e63 100644 --- a/tests/ut/python/dataset/test_pyfunc.py +++ b/tests/ut/python/dataset/test_pyfunc.py @@ -340,7 +340,7 @@ def test_func_with_yield_manifest_dataset_01(): data = data.map(operations=pass_func, input_columns=["image"], num_parallel_workers=1, python_multiprocessing=True) num_iter = 0 try: - for _ in data.create_dict_iterator(output_numpy=True): + for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True): num_iter += 1 except RuntimeError as e: assert "Can not pickle object, " in str(e) diff --git a/tests/ut/python/dataset/test_random_sharpness.py b/tests/ut/python/dataset/test_random_sharpness.py index 6b1db9620f0..7ea0a72a485 100644 --- a/tests/ut/python/dataset/test_random_sharpness.py +++ b/tests/ut/python/dataset/test_random_sharpness.py @@ -47,7 +47,7 @@ def test_random_sharpness_py(degrees=(0.7, 0.7), plot=False): ds_original = ds_original.batch(512) - for idx, (image, _) in enumerate(ds_original.create_tuple_iterator(output_numpy=True)): + for idx, (image, _) in enumerate(ds_original.create_tuple_iterator(num_epochs=1, output_numpy=True)): if idx == 0: images_original = np.transpose(image, (0, 2, 3, 1)) else: @@ -71,7 +71,7 @@ def test_random_sharpness_py(degrees=(0.7, 0.7), plot=False): ds_random_sharpness = ds_random_sharpness.batch(512) - for idx, (image, _) in enumerate(ds_random_sharpness.create_tuple_iterator(output_numpy=True)): + for idx, (image, _) in enumerate(ds_random_sharpness.create_tuple_iterator(num_epochs=1, output_numpy=True)): if idx == 0: images_random_sharpness = np.transpose(image, (0, 2, 3, 1)) else: @@ -136,7 +136,7 @@ def test_random_sharpness_c(degrees=(1.6, 1.6), plot=False): ds_original = ds_original.batch(512) - for idx, (image, _) in enumerate(ds_original.create_tuple_iterator(output_numpy=True)): + for idx, (image, _) in enumerate(ds_original.create_tuple_iterator(num_epochs=1, output_numpy=True)): if idx == 0: images_original = image else: @@ -159,7 +159,7 @@ def test_random_sharpness_c(degrees=(1.6, 1.6), plot=False): ds_random_sharpness = ds_random_sharpness.batch(512) - for idx, (image, _) in enumerate(ds_random_sharpness.create_tuple_iterator(output_numpy=True)): + for idx, (image, _) in enumerate(ds_random_sharpness.create_tuple_iterator(num_epochs=1, output_numpy=True)): if idx == 0: images_random_sharpness = image else: @@ -226,7 +226,7 @@ def test_random_sharpness_c_py(degrees=(1.0, 1.0), plot=False): ds_random_sharpness_py = ds_random_sharpness_py.batch(512) - for idx, (image, _) in enumerate(ds_random_sharpness_py.create_tuple_iterator(output_numpy=True)): + for idx, (image, _) in enumerate(ds_random_sharpness_py.create_tuple_iterator(num_epochs=1, output_numpy=True)): if idx == 0: images_random_sharpness_py = image @@ -242,7 +242,10 @@ def test_random_sharpness_c_py(degrees=(1.0, 1.0), plot=False): ds_images_random_sharpness_c = ds_images_random_sharpness_c.batch(512) - for idx, (image, _) in enumerate(ds_images_random_sharpness_c.create_tuple_iterator(output_numpy=True)): + for idx, (image, _) in enumerate( + ds_images_random_sharpness_c.create_tuple_iterator( + num_epochs=1, + output_numpy=True)): if idx == 0: images_random_sharpness_c = image diff --git a/tests/ut/python/dataset/test_repeat.py b/tests/ut/python/dataset/test_repeat.py index 196a9644fb6..9c4f2460067 100644 --- a/tests/ut/python/dataset/test_repeat.py +++ b/tests/ut/python/dataset/test_repeat.py @@ -117,7 +117,7 @@ def test_nested_repeat1(): data = data.repeat(2) data = data.repeat(3) - for i, d in enumerate(data.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(data.create_tuple_iterator(num_epochs=1, output_numpy=True)): assert i % 3 == d[0][0] assert sum([1 for _ in data]) == 2 * 3 * 3 @@ -129,7 +129,7 @@ def test_nested_repeat2(): data = data.repeat(1) data = data.repeat(1) - for i, d in enumerate(data.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(data.create_tuple_iterator(num_epochs=1, output_numpy=True)): assert i % 3 == d[0][0] assert sum([1 for _ in data]) == 3 @@ -141,7 +141,7 @@ def test_nested_repeat3(): data = data.repeat(1) data = data.repeat(2) - for i, d in enumerate(data.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(data.create_tuple_iterator(num_epochs=1, output_numpy=True)): assert i % 3 == d[0][0] assert sum([1 for _ in data]) == 2 * 3 @@ -153,7 +153,7 @@ def test_nested_repeat4(): data = data.repeat(2) data = data.repeat(1) - for i, d in enumerate(data.create_tuple_iterator(output_numpy=True)): + for i, d in enumerate(data.create_tuple_iterator(num_epochs=1, output_numpy=True)): assert i % 3 == d[0][0] assert sum([1 for _ in data]) == 2 * 3 diff --git a/tests/ut/python/dataset/test_sampler_chain.py b/tests/ut/python/dataset/test_sampler_chain.py index 59068b04b91..1356a8f1367 100644 --- a/tests/ut/python/dataset/test_sampler_chain.py +++ b/tests/ut/python/dataset/test_sampler_chain.py @@ -269,7 +269,7 @@ def test_voc_sampler_chain(): assert data1_size == 5 # Verify number of rows - assert sum([1 for _ in data1.create_dict_iterator(output_numpy=True)]) == 5 + assert sum([1 for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True)]) == 5 # Verify dataset contents res = [] diff --git a/tests/ut/python/dataset/test_serdes_dataset.py b/tests/ut/python/dataset/test_serdes_dataset.py index 07c239c1d18..f3320fdb6fb 100644 --- a/tests/ut/python/dataset/test_serdes_dataset.py +++ b/tests/ut/python/dataset/test_serdes_dataset.py @@ -307,8 +307,9 @@ def test_serdes_zip_dataset(remove_json_files=True): assert filecmp.cmp('zip_dataset_pipeline.json', 'zip_dataset_pipeline_1.json') rows = 0 - for d0, d3, d4 in zip(ds0.create_tuple_iterator(output_numpy=True), data3.create_tuple_iterator(output_numpy=True), - data4.create_tuple_iterator(output_numpy=True)): + for d0, d3, d4 in zip(ds0.create_tuple_iterator(num_epochs=1, output_numpy=True), + data3.create_tuple_iterator(num_epochs=1, output_numpy=True), + data4.create_tuple_iterator(num_epochs=1, output_numpy=True)): num_cols = len(d0) offset = 0 for t1 in d0: diff --git a/tests/ut/python/dataset/test_skip.py b/tests/ut/python/dataset/test_skip.py index a75e88e7bad..d1c7404a0f5 100644 --- a/tests/ut/python/dataset/test_skip.py +++ b/tests/ut/python/dataset/test_skip.py @@ -57,7 +57,7 @@ def test_generator_skip(): ds1 = ds1.skip(3) buf = [] - for data in ds1.create_tuple_iterator(output_numpy=True): + for data in ds1.create_tuple_iterator(num_epochs=1, output_numpy=True): buf.append(data[0][0]) assert len(buf) == 2 assert buf == [3, 4] @@ -70,7 +70,7 @@ def test_skip_1(): ds1 = ds1.skip(7) buf = [] - for data in ds1.create_tuple_iterator(output_numpy=True): + for data in ds1.create_tuple_iterator(num_epochs=1, output_numpy=True): buf.append(data[0][0]) assert buf == [] @@ -82,7 +82,7 @@ def test_skip_2(): ds1 = ds1.skip(0) buf = [] - for data in ds1.create_tuple_iterator(output_numpy=True): + for data in ds1.create_tuple_iterator(num_epochs=1, output_numpy=True): buf.append(data[0][0]) assert len(buf) == 5 assert buf == [0, 1, 2, 3, 4] @@ -98,7 +98,7 @@ def test_skip_repeat_1(): ds1 = ds1.skip(3) buf = [] - for data in ds1.create_tuple_iterator(output_numpy=True): + for data in ds1.create_tuple_iterator(num_epochs=1, output_numpy=True): buf.append(data[0][0]) assert len(buf) == 7 assert buf == [3, 4, 0, 1, 2, 3, 4] @@ -114,7 +114,7 @@ def test_skip_repeat_2(): ds1 = ds1.repeat(2) buf = [] - for data in ds1.create_tuple_iterator(output_numpy=True): + for data in ds1.create_tuple_iterator(num_epochs=1, output_numpy=True): buf.append(data[0][0]) assert len(buf) == 4 assert buf == [3, 4, 3, 4] @@ -133,7 +133,7 @@ def test_skip_repeat_3(): ds1 = ds1.repeat(3) buf = [] - for data in ds1.create_tuple_iterator(output_numpy=True): + for data in ds1.create_tuple_iterator(num_epochs=1, output_numpy=True): buf.append(data[0][0]) assert len(buf) == 6 assert buf == [3, 4, 3, 4, 3, 4] @@ -149,7 +149,7 @@ def test_skip_take_1(): ds1 = ds1.skip(2) buf = [] - for data in ds1.create_tuple_iterator(output_numpy=True): + for data in ds1.create_tuple_iterator(num_epochs=1, output_numpy=True): buf.append(data[0][0]) assert len(buf) == 2 assert buf == [2, 3] @@ -165,7 +165,7 @@ def test_skip_take_2(): ds1 = ds1.take(2) buf = [] - for data in ds1.create_tuple_iterator(output_numpy=True): + for data in ds1.create_tuple_iterator(num_epochs=1, output_numpy=True): buf.append(data[0][0]) assert len(buf) == 2 assert buf == [2, 3] @@ -182,7 +182,7 @@ def test_skip_filter_1(): dataset = dataset.filter(predicate=lambda data: data < 11, num_parallel_workers=4) buf = [] - for item in dataset.create_tuple_iterator(output_numpy=True): + for item in dataset.create_tuple_iterator(num_epochs=1, output_numpy=True): buf.append(item[0][0]) assert buf == [5, 6, 7, 8, 9, 10] @@ -193,7 +193,7 @@ def test_skip_filter_2(): dataset = dataset.skip(5) buf = [] - for item in dataset.create_tuple_iterator(output_numpy=True): + for item in dataset.create_tuple_iterator(num_epochs=1, output_numpy=True): buf.append(item[0][0]) assert buf == [5, 6, 7, 8, 9, 10] diff --git a/tests/ut/python/dataset/test_slice_op.py b/tests/ut/python/dataset/test_slice_op.py index ae71139f379..701ada6fc75 100644 --- a/tests/ut/python/dataset/test_slice_op.py +++ b/tests/ut/python/dataset/test_slice_op.py @@ -28,7 +28,7 @@ def slice_compare(array, indexing, expected_array): data = data.map(operations=ops.Slice(*indexing)) else: data = data.map(operations=ops.Slice(indexing)) - for d in data.create_dict_iterator(output_numpy=True): + for d in data.create_dict_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(expected_array, d['column_0']) @@ -141,7 +141,7 @@ def test_slice_multiple_rows(): data = ds.GeneratorDataset(gen, column_names=["col"]) indexing = slice(1, 4) data = data.map(operations=ops.Slice(indexing)) - for (d, exp_d) in zip(data.create_dict_iterator(output_numpy=True), exp_dataset): + for (d, exp_d) in zip(data.create_dict_iterator(num_epochs=1, output_numpy=True), exp_dataset): np.testing.assert_array_equal(exp_d, d['col']) @@ -158,12 +158,12 @@ def test_slice_none_and_ellipsis(): data = ds.GeneratorDataset(gen, column_names=["col"]) data = data.map(operations=ops.Slice(None)) - for (d, exp_d) in zip(data.create_dict_iterator(output_numpy=True), exp_dataset): + for (d, exp_d) in zip(data.create_dict_iterator(num_epochs=1, output_numpy=True), exp_dataset): np.testing.assert_array_equal(exp_d, d['col']) data = ds.GeneratorDataset(gen, column_names=["col"]) data = data.map(operations=ops.Slice(Ellipsis)) - for (d, exp_d) in zip(data.create_dict_iterator(output_numpy=True), exp_dataset): + for (d, exp_d) in zip(data.create_dict_iterator(num_epochs=1, output_numpy=True), exp_dataset): np.testing.assert_array_equal(exp_d, d['col']) @@ -280,7 +280,7 @@ def test_out_of_bounds_slicing_str(): data = [b"1", b"2", b"3", b"4", b"5"] data = ds.NumpySlicesDataset([data]) data = data.map(operations=ops.Slice(indexing)) - for d in data.create_dict_iterator(output_numpy=True): + for d in data.create_dict_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(expected_array, d['column_0']) diff --git a/tests/ut/python/dataset/test_slice_patches.py b/tests/ut/python/dataset/test_slice_patches.py index 159d994a812..435e8afe3fd 100644 --- a/tests/ut/python/dataset/test_slice_patches.py +++ b/tests/ut/python/dataset/test_slice_patches.py @@ -161,7 +161,7 @@ def test_slice_patches_08(): dataset = dataset.map(input_columns=["image"], output_columns=["img0", "img1", "img2", "img3"], column_order=["img0", "img1", "img2", "img3"], operations=slice_patches_op) - for item in dataset.create_dict_iterator(output_numpy=True): + for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): patch_shape = item['img0'].shape assert patch_shape == (28, 41, 256) @@ -185,7 +185,7 @@ def skip_test_slice_patches_11(): cols = ['img' + str(x) for x in range(10*13)] dataset = dataset.map(input_columns=["image"], output_columns=cols, column_order=cols, operations=slice_patches_op) - for item in dataset.create_dict_iterator(output_numpy=True): + for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): patch_shape = item['img0'].shape assert patch_shape == (700, 538, 256) diff --git a/tests/ut/python/dataset/test_sync_wait.py b/tests/ut/python/dataset/test_sync_wait.py index cdb5f10aa8c..61a85fa9340 100644 --- a/tests/ut/python/dataset/test_sync_wait.py +++ b/tests/ut/python/dataset/test_sync_wait.py @@ -264,7 +264,7 @@ def test_simple_sync_wait_empty_condition_name(): dataset = dataset.batch(batch_size) count = 0 - for data in dataset.create_dict_iterator(output_numpy=True): + for data in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): count += 1 data = {"loss": count} dataset.sync_update(condition_name="", data=data) diff --git a/tests/ut/python/dataset/test_tensor_empty.py b/tests/ut/python/dataset/test_tensor_empty.py index 7075df9a95d..4395ab1f430 100644 --- a/tests/ut/python/dataset/test_tensor_empty.py +++ b/tests/ut/python/dataset/test_tensor_empty.py @@ -25,7 +25,7 @@ def test_tensor_empty(): data = ds.GeneratorDataset(gen, column_names=["col1", "col2", "col3"]) - for d in data.create_tuple_iterator(output_numpy=True): + for d in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(np.array([], dtype=np.int64), d[0]) np.testing.assert_array_equal(np.array([], dtype='S').reshape([0, 4]), d[1]) np.testing.assert_array_equal(np.array([1], dtype=np.float64), d[2]) @@ -46,7 +46,7 @@ def test_tensor_empty_map(): data = data.map(operations=func, input_columns=["col1", "col2", "col3"]) - for d in data.create_tuple_iterator(output_numpy=True): + for d in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(np.array([1], dtype=np.int64), d[0]) np.testing.assert_array_equal(np.array(["Hi"], dtype='S'), d[1]) np.testing.assert_array_equal(np.array([], dtype=np.float64), d[2]) @@ -60,7 +60,7 @@ def test_tensor_empty_batch(): data = ds.GeneratorDataset(gen, column_names=["col1", "col2", "col3"]).batch(2) - for d in data.create_tuple_iterator(output_numpy=True): + for d in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(np.array([], dtype=np.int64).reshape([2, 0]), d[0]) np.testing.assert_array_equal(np.array([], dtype='S').reshape([2, 0, 4]), d[1]) np.testing.assert_array_equal(np.array([[1], [1]], dtype=np.float64), d[2]) diff --git a/tests/ut/python/dataset/test_tensor_string.py b/tests/ut/python/dataset/test_tensor_string.py index d9a2d2f7aa4..7b98f8f527f 100644 --- a/tests/ut/python/dataset/test_tensor_string.py +++ b/tests/ut/python/dataset/test_tensor_string.py @@ -35,7 +35,7 @@ def compare(strings, dtype='S'): data = ds.GeneratorDataset(gen, column_names=["col"]) - for d in data.create_tuple_iterator(output_numpy=True): + for d in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(d[0], arr.astype('S')) @@ -79,7 +79,7 @@ def test_batching_strings(): data = ds.GeneratorDataset(gen, column_names=["col"]) data = data.batch(2, drop_remainder=True) - for d in data.create_tuple_iterator(output_numpy=True): + for d in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(d[0], to_bytes(chinese[0:2])) @@ -96,7 +96,7 @@ def test_map(): data = data.map(operations=split, input_columns=["col"]) expected = np.array(["ab", "cde", "121"], dtype='S') - for d in data.create_tuple_iterator(output_numpy=True): + for d in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(d[0], expected) @@ -112,7 +112,7 @@ def test_map2(): data = data.map(operations=upper, input_columns=["col"]) expected = np.array(["AB CDE 121"], dtype='S') - for d in data.create_tuple_iterator(output_numpy=True): + for d in data.create_tuple_iterator(num_epochs=1, output_numpy=True): np.testing.assert_array_equal(d[0], expected)