mindspore-Ecosystem
/
mindspore
21
0
Fork 27
Code Issues Pull Requests 6 Projects Releases Wiki Activity

add try to minddataset python tests to remove files on fail 

delete unrelated file

change finally to except else

fix spacing alignment

fix indentation

fix indentation

fix indentation

add fix to new test case

add if main for pytest

fix spacing
This commit is contained in:
tony_liu2 2020-07-17 14:06:14 -04:00
parent 859fe6bc41
commit 1f4251a440
5 changed files with 1306 additions and 1083 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

210
tests/ut/python/dataset/test_minddataset.py
View File

@ -46,6 +46,7 @@ def add_and_remove_cv_file():
"""add/remove cv file"""
paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0'))
for x in range(FILES_NUM)]
try:
for x in paths:
if os.path.exists("{}".format(x)):
os.remove("{}".format(x))
@ -62,16 +63,22 @@ def add_and_remove_cv_file():
writer.write_raw_data(data)
writer.commit()
yield "yield_cv_data"
except Exception as error:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
raise error
else:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
@pytest.fixture
def add_and_remove_nlp_file():
"""add/remove nlp file"""
paths = ["{}{}".format(NLP_FILE_NAME, str(x).rjust(1, '0'))
for x in range(FILES_NUM)]
try:
for x in paths:
if os.path.exists("{}".format(x)):
os.remove("{}".format(x))
@ -95,6 +102,12 @@ def add_and_remove_nlp_file():
writer.write_raw_data(data)
writer.commit()
yield "yield_nlp_data"
except Exception as error:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
raise error
else:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
@ -105,6 +118,7 @@ def add_and_remove_nlp_compress_file():
"""add/remove nlp file"""
paths = ["{}{}".format(NLP_FILE_NAME, str(x).rjust(1, '0'))
for x in range(FILES_NUM)]
try:
for x in paths:
if os.path.exists("{}".format(x)):
os.remove("{}".format(x))
@ -140,6 +154,12 @@ def add_and_remove_nlp_compress_file():
writer.write_raw_data(data)
writer.commit()
yield "yield_nlp_data"
except Exception as error:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
raise error
else:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
@ -199,6 +219,7 @@ def test_cv_minddataset_writer_tutorial():
"""tutorial for cv dataset writer."""
paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0'))
for x in range(FILES_NUM)]
try:
for x in paths:
if os.path.exists("{}".format(x)):
os.remove("{}".format(x))
@ -212,6 +233,12 @@ def test_cv_minddataset_writer_tutorial():
writer.add_index(["file_name", "label"])
writer.write_raw_data(data)
writer.commit()
except Exception as error:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
raise error
else:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
@ -654,6 +681,7 @@ def test_cv_minddataset_reader_one_partition(add_and_remove_cv_file):
def test_cv_minddataset_reader_two_dataset(add_and_remove_cv_file):
"""tutorial for cv minderdataset."""
try:
if os.path.exists(CV1_FILE_NAME):
os.remove(CV1_FILE_NAME)
if os.path.exists("{}.db".format(CV1_FILE_NAME)):
@ -702,6 +730,17 @@ def test_cv_minddataset_reader_two_dataset(add_and_remove_cv_file):
"-------------- item[label]: {} ----------------------------".format(item["label"]))
num_iter += 1
assert num_iter == 30
except Exception as error:
if os.path.exists(CV1_FILE_NAME):
os.remove(CV1_FILE_NAME)
if os.path.exists("{}.db".format(CV1_FILE_NAME)):
os.remove("{}.db".format(CV1_FILE_NAME))
if os.path.exists(CV2_FILE_NAME):
os.remove(CV2_FILE_NAME)
if os.path.exists("{}.db".format(CV2_FILE_NAME)):
os.remove("{}.db".format(CV2_FILE_NAME))
raise error
else:
if os.path.exists(CV1_FILE_NAME):
os.remove(CV1_FILE_NAME)
if os.path.exists("{}.db".format(CV1_FILE_NAME)):
@ -711,10 +750,10 @@ def test_cv_minddataset_reader_two_dataset(add_and_remove_cv_file):
if os.path.exists("{}.db".format(CV2_FILE_NAME)):
os.remove("{}.db".format(CV2_FILE_NAME))
def test_cv_minddataset_reader_two_dataset_partition(add_and_remove_cv_file):
paths = ["{}{}".format(CV1_FILE_NAME, str(x).rjust(1, '0'))
for x in range(FILES_NUM)]
try:
for x in paths:
if os.path.exists("{}".format(x)):
os.remove("{}".format(x))
@ -733,7 +772,8 @@ def test_cv_minddataset_reader_two_dataset_partition(add_and_remove_cv_file):
columns_list = ["data", "file_name", "label"]
num_readers = 4
data_set = ds.MindDataset([CV_FILE_NAME + str(x) for x in range(2)] + [CV1_FILE_NAME + str(x) for x in range(2, 4)],
data_set = ds.MindDataset([CV_FILE_NAME + str(x) for x in range(2)] +
[CV1_FILE_NAME + str(x) for x in range(2, 4)],
columns_list, num_readers)
assert data_set.get_dataset_size() < 20
num_iter = 0
@ -750,10 +790,15 @@ def test_cv_minddataset_reader_two_dataset_partition(add_and_remove_cv_file):
"-------------- item[label]: {} ----------------------------".format(item["label"]))
num_iter += 1
assert num_iter < 20
except Exception as error:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
raise error
else:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
def test_cv_minddataset_reader_basic_tutorial(add_and_remove_cv_file):
"""tutorial for cv minderdataset."""
@ -1086,6 +1131,7 @@ def inputs(vectors, maxlen=50):
def test_write_with_multi_bytes_and_array_and_read_by_MindDataset():
mindrecord_file_name = "test.mindrecord"
try:
if os.path.exists("{}".format(mindrecord_file_name)):
os.remove("{}".format(mindrecord_file_name))
if os.path.exists("{}.db".format(mindrecord_file_name)):
@ -1238,8 +1284,7 @@ def test_write_with_multi_bytes_and_array_and_read_by_MindDataset():
num_readers = 1
data_set = ds.MindDataset(dataset_file=mindrecord_file_name,
columns_list=[
"image2", "source_sos_mask", "image3", "target_sos_ids"],
columns_list=["image2", "source_sos_mask", "image3", "target_sos_ids"],
num_parallel_workers=num_readers,
shuffle=False)
assert data_set.get_dataset_size() == 6
@ -1314,8 +1359,9 @@ def test_write_with_multi_bytes_and_array_and_read_by_MindDataset():
num_readers = 2
data_set = ds.MindDataset(dataset_file=mindrecord_file_name,
columns_list=["label", "target_eos_mask", "image1", "target_eos_ids", "source_sos_mask",
"image2", "image4", "image3", "source_sos_ids", "image5", "file_name"],
columns_list=["label", "target_eos_mask", "image1", "target_eos_ids",
"source_sos_mask", "image2", "image4", "image3",
"source_sos_ids", "image5", "file_name"],
num_parallel_workers=num_readers,
shuffle=False)
assert data_set.get_dataset_size() == 6
@ -1330,13 +1376,18 @@ def test_write_with_multi_bytes_and_array_and_read_by_MindDataset():
assert item[field] == data_value_to_list[num_iter][field]
num_iter += 1
assert num_iter == 6
except Exception as error:
os.remove("{}".format(mindrecord_file_name))
os.remove("{}.db".format(mindrecord_file_name))
raise error
else:
os.remove("{}".format(mindrecord_file_name))
os.remove("{}.db".format(mindrecord_file_name))
def test_write_with_multi_bytes_and_MindDataset():
mindrecord_file_name = "test.mindrecord"
try:
data = [{"file_name": "001.jpg", "label": 43,
"image1": bytes("image1 bytes abc", encoding='UTF-8'),
"image2": bytes("image1 bytes def", encoding='UTF-8'),
@ -1506,13 +1557,17 @@ def test_write_with_multi_bytes_and_MindDataset():
assert item[field] == data_value_to_list[num_iter][field]
num_iter += 1
assert num_iter == 6
except Exception as error:
os.remove("{}".format(mindrecord_file_name))
os.remove("{}.db".format(mindrecord_file_name))
raise error
else:
os.remove("{}".format(mindrecord_file_name))
os.remove("{}.db".format(mindrecord_file_name))
def test_write_with_multi_array_and_MindDataset():
mindrecord_file_name = "test.mindrecord"
try:
data = [{"source_sos_ids": np.array([1, 2, 3, 4, 5], dtype=np.int64),
"source_sos_mask": np.array([6, 7, 8, 9, 10, 11, 12], dtype=np.int64),
"source_eos_ids": np.array([13, 14, 15, 16, 17, 18], dtype=np.int64),
@ -1704,12 +1759,64 @@ def test_write_with_multi_array_and_MindDataset():
assert item[field] == data_value_to_list[num_iter][field]
num_iter += 1
assert num_iter == 6
except Exception as error:
os.remove("{}".format(mindrecord_file_name))
os.remove("{}.db".format(mindrecord_file_name))
raise error
else:
os.remove("{}".format(mindrecord_file_name))
os.remove("{}.db".format(mindrecord_file_name))
def test_numpy_generic():
paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0'))
for x in range(FILES_NUM)]
try:
for x in paths:
if os.path.exists("{}".format(x)):
os.remove("{}".format(x))
if os.path.exists("{}.db".format(x)):
os.remove("{}.db".format(x))
writer = FileWriter(CV_FILE_NAME, FILES_NUM)
cv_schema_json = {"label1": {"type": "int32"}, "label2": {"type": "int64"},
"label3": {"type": "float32"}, "label4": {"type": "float64"}}
data = []
for idx in range(10):
row = {}
row['label1'] = np.int32(idx)
row['label2'] = np.int64(idx*10)
row['label3'] = np.float32(idx+0.12345)
row['label4'] = np.float64(idx+0.12345789)
data.append(row)
writer.add_schema(cv_schema_json, "img_schema")
writer.write_raw_data(data)
writer.commit()
num_readers = 4
data_set = ds.MindDataset(CV_FILE_NAME + "0", None, num_readers, shuffle=False)
assert data_set.get_dataset_size() == 10
idx = 0
for item in data_set.create_dict_iterator():
assert item['label1'] == item['label1']
assert item['label2'] == item['label2']
assert item['label3'] == item['label3']
assert item['label4'] == item['label4']
idx += 1
assert idx == 10
except Exception as error:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
raise error
else:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
def test_write_with_float32_float64_float32_array_float64_array_and_MindDataset():
mindrecord_file_name = "test.mindrecord"
try:
data = [{"float32_array": np.array([1.2, 2.78, 3.1234, 4.9871, 5.12341], dtype=np.float32),
"float64_array": np.array([48.1234556789, 49.3251241431, 50.13514312414, 51.8971298471,
123414314.2141243, 87.1212122], dtype=np.float64),
@ -1842,7 +1949,8 @@ def test_write_with_float32_float64_float32_array_float64_array_and_MindDataset(
np.array(data_value_to_list[num_iter][field], np.float32)).all()
elif item[field].dtype == np.float64:
assert math.isclose(item[field],
np.array(data_value_to_list[num_iter][field], np.float64), rel_tol=1e-14)
np.array(data_value_to_list[num_iter][field], np.float64),
rel_tol=1e-14)
else:
assert (item[field] ==
data_value_to_list[num_iter][field]).all()
@ -1850,45 +1958,43 @@ def test_write_with_float32_float64_float32_array_float64_array_and_MindDataset(
assert item[field] == data_value_to_list[num_iter][field]
num_iter += 1
assert num_iter == 5
except Exception as error:
os.remove("{}".format(mindrecord_file_name))
os.remove("{}.db".format(mindrecord_file_name))
raise error
else:
os.remove("{}".format(mindrecord_file_name))
os.remove("{}.db".format(mindrecord_file_name))
def test_numpy_generic():
paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0'))
for x in range(FILES_NUM)]
for x in paths:
if os.path.exists("{}".format(x)):
os.remove("{}".format(x))
if os.path.exists("{}.db".format(x)):
os.remove("{}.db".format(x))
writer = FileWriter(CV_FILE_NAME, FILES_NUM)
cv_schema_json = {"label1": {"type": "int32"}, "label2": {"type": "int64"},
"label3": {"type": "float32"}, "label4": {"type": "float64"}}
data = []
for idx in range(10):
row = {}
row['label1'] = np.int32(idx)
row['label2'] = np.int64(idx*10)
row['label3'] = np.float32(idx+0.12345)
row['label4'] = np.float64(idx+0.12345789)
data.append(row)
writer.add_schema(cv_schema_json, "img_schema")
writer.write_raw_data(data)
writer.commit()
num_readers = 4
data_set = ds.MindDataset(CV_FILE_NAME + "0", None, num_readers, shuffle=False)
assert data_set.get_dataset_size() == 10
idx = 0
for item in data_set.create_dict_iterator():
assert item['label1'] == item['label1']
assert item['label2'] == item['label2']
assert item['label3'] == item['label3']
assert item['label4'] == item['label4']
idx += 1
assert idx == 10
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
if __name__ == '__main__':
test_nlp_compress_data(add_and_remove_nlp_compress_file)
test_nlp_compress_data_old_version(add_and_remove_nlp_compress_file)
test_cv_minddataset_writer_tutorial()
test_cv_minddataset_partition_tutorial(add_and_remove_cv_file)
test_cv_minddataset_partition_num_samples_0(add_and_remove_cv_file)
test_cv_minddataset_partition_num_samples_1(add_and_remove_cv_file)
test_cv_minddataset_partition_num_samples_2(add_and_remove_cv_file)
test_cv_minddataset_partition_tutorial_check_shuffle_result(add_and_remove_cv_file)
test_cv_minddataset_partition_tutorial_check_whole_reshuffle_result_per_epoch(add_and_remove_cv_file)
test_cv_minddataset_check_shuffle_result(add_and_remove_cv_file)
test_cv_minddataset_dataset_size(add_and_remove_cv_file)
test_cv_minddataset_repeat_reshuffle(add_and_remove_cv_file)
test_cv_minddataset_batch_size_larger_than_records(add_and_remove_cv_file)
test_cv_minddataset_issue_888(add_and_remove_cv_file)
test_cv_minddataset_blockreader_tutorial(add_and_remove_cv_file)
test_cv_minddataset_blockreader_some_field_not_in_index_tutorial(add_and_remove_cv_file)
test_cv_minddataset_reader_file_list(add_and_remove_cv_file)
test_cv_minddataset_reader_one_partition(add_and_remove_cv_file)
test_cv_minddataset_reader_two_dataset(add_and_remove_cv_file)
test_cv_minddataset_reader_two_dataset_partition(add_and_remove_cv_file)
test_cv_minddataset_reader_basic_tutorial(add_and_remove_cv_file)
test_nlp_minddataset_reader_basic_tutorial(add_and_remove_cv_file)
test_cv_minddataset_reader_basic_tutorial_5_epoch(add_and_remove_cv_file)
test_cv_minddataset_reader_basic_tutorial_5_epoch_with_batch(add_and_remove_cv_file)
test_cv_minddataset_reader_no_columns(add_and_remove_cv_file)
test_cv_minddataset_reader_repeat_tutorial(add_and_remove_cv_file)
test_write_with_multi_bytes_and_array_and_read_by_MindDataset()
test_write_with_multi_bytes_and_MindDataset()
test_write_with_multi_array_and_MindDataset()
test_numpy_generic()
test_write_with_float32_float64_float32_array_float64_array_and_MindDataset()

58
tests/ut/python/dataset/test_minddataset_exception.py
View File

@ -99,7 +99,12 @@ def test_invalid_mindrecord():
num_iter = 0
for _ in data_set.create_dict_iterator():
num_iter += 1
try:
assert num_iter == 0
except Exception as error:
os.remove('dummy.mindrecord')
raise error
else:
os.remove('dummy.mindrecord')
@ -113,7 +118,12 @@ def test_minddataset_lack_db():
num_iter = 0
for _ in data_set.create_dict_iterator():
num_iter += 1
try:
assert num_iter == 0
except Exception as error:
os.remove(CV_FILE_NAME)
raise error
else:
os.remove(CV_FILE_NAME)
@ -189,10 +199,16 @@ def test_minddataset_invalidate_num_shards():
num_iter = 0
for _ in data_set.create_dict_iterator():
num_iter += 1
try:
assert 'Input shard_id is not within the required interval of (0 to 0).' in str(error_info.value)
except Exception as error:
os.remove(CV_FILE_NAME)
os.remove("{}.db".format(CV_FILE_NAME))
raise error
else:
os.remove(CV_FILE_NAME)
os.remove("{}.db".format(CV_FILE_NAME))
def test_minddataset_invalidate_shard_id():
create_cv_mindrecord(1)
@ -203,7 +219,13 @@ def test_minddataset_invalidate_shard_id():
num_iter = 0
for _ in data_set.create_dict_iterator():
num_iter += 1
try:
assert 'Input shard_id is not within the required interval of (0 to 0).' in str(error_info.value)
except Exception as error:
os.remove(CV_FILE_NAME)
os.remove("{}.db".format(CV_FILE_NAME))
raise error
else:
os.remove(CV_FILE_NAME)
os.remove("{}.db".format(CV_FILE_NAME))
@ -217,17 +239,28 @@ def test_minddataset_shard_id_bigger_than_num_shard():
num_iter = 0
for _ in data_set.create_dict_iterator():
num_iter += 1
try:
assert 'Input shard_id is not within the required interval of (0 to 1).' in str(error_info.value)
except Exception as error:
os.remove(CV_FILE_NAME)
os.remove("{}.db".format(CV_FILE_NAME))
raise error
with pytest.raises(Exception) as error_info:
data_set = ds.MindDataset(CV_FILE_NAME, columns_list, num_readers, True, 2, 5)
num_iter = 0
for _ in data_set.create_dict_iterator():
num_iter += 1
try:
assert 'Input shard_id is not within the required interval of (0 to 1).' in str(error_info.value)
except Exception as error:
os.remove(CV_FILE_NAME)
os.remove("{}.db".format(CV_FILE_NAME))
raise error
else:
os.remove(CV_FILE_NAME)
os.remove("{}.db".format(CV_FILE_NAME))
def test_cv_minddataset_partition_num_samples_equals_0():
"""tutorial for cv minddataset."""
@ -245,7 +278,26 @@ def test_cv_minddataset_partition_num_samples_equals_0():
num_iter += 1
with pytest.raises(Exception) as error_info:
partitions(5)
try:
assert 'num_samples should be a positive integer value, but got num_samples=0' in str(error_info.value)
except Exception as error:
os.remove(CV_FILE_NAME)
os.remove("{}.db".format(CV_FILE_NAME))
raise error
else:
os.remove(CV_FILE_NAME)
os.remove("{}.db".format(CV_FILE_NAME))
if __name__ == '__main__':
test_cv_lack_json()
test_cv_lack_mindrecord()
test_invalid_mindrecord()
test_minddataset_lack_db()
test_cv_minddataset_pk_sample_error_class_column()
test_cv_minddataset_pk_sample_exclusive_shuffle()
test_cv_minddataset_reader_different_schema()
test_cv_minddataset_reader_different_page_size()
test_minddataset_invalidate_num_shards()
test_minddataset_invalidate_shard_id()
test_minddataset_shard_id_bigger_than_num_shard()
test_cv_minddataset_partition_num_samples_equals_0()

12
tests/ut/python/dataset/test_minddataset_multi_images_and_ndarray.py
View File

@ -27,6 +27,7 @@ CV_FILE_NAME = "./complex.mindrecord"
def test_cv_minddataset_reader_multi_image_and_ndarray_tutorial():
try:
writer = FileWriter(CV_FILE_NAME, FILES_NUM)
cv_schema_json = {"id": {"type": "int32"},
"image_0": {"type": "bytes"},
@ -73,8 +74,17 @@ def test_cv_minddataset_reader_multi_image_and_ndarray_tutorial():
assert item["segments"].dtype == np.float32
num_iter += 1
assert num_iter == 5
except Exception as error:
if os.path.exists("{}".format(CV_FILE_NAME + ".db")):
os.remove(CV_FILE_NAME + ".db")
if os.path.exists("{}".format(CV_FILE_NAME)):
os.remove(CV_FILE_NAME)
raise error
else:
if os.path.exists("{}".format(CV_FILE_NAME + ".db")):
os.remove(CV_FILE_NAME + ".db")
if os.path.exists("{}".format(CV_FILE_NAME)):
os.remove(CV_FILE_NAME)
if __name__ == '__main__':
test_cv_minddataset_reader_multi_image_and_ndarray_tutorial()

28
tests/ut/python/dataset/test_minddataset_padded.py
View File

@ -44,6 +44,7 @@ def add_and_remove_cv_file():
"""add/remove cv file"""
paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0'))
for x in range(FILES_NUM)]
try:
for x in paths:
os.remove("{}".format(x)) if os.path.exists("{}".format(x)) else None
os.remove("{}.db".format(x)) if os.path.exists(
@ -59,6 +60,12 @@ def add_and_remove_cv_file():
writer.write_raw_data(data)
writer.commit()
yield "yield_cv_data"
except Exception as error:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
raise error
else:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
@ -69,6 +76,7 @@ def add_and_remove_nlp_file():
"""add/remove nlp file"""
paths = ["{}{}".format(NLP_FILE_NAME, str(x).rjust(1, '0'))
for x in range(FILES_NUM)]
try:
for x in paths:
if os.path.exists("{}".format(x)):
os.remove("{}".format(x))
@ -92,6 +100,12 @@ def add_and_remove_nlp_file():
writer.write_raw_data(data)
writer.commit()
yield "yield_nlp_data"
except Exception as error:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
raise error
else:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
@ -636,3 +650,17 @@ def inputs(vectors, maxlen=50):
mask = [1] * length + [0] * (maxlen - length)
segment = [0] * maxlen
return input_, mask, segment
if __name__ == '__main__':
test_cv_minddataset_reader_basic_padded_samples(add_and_remove_cv_file)
test_cv_minddataset_partition_padded_samples(add_and_remove_cv_file)
test_cv_minddataset_partition_padded_samples_multi_epoch(add_and_remove_cv_file)
test_cv_minddataset_partition_padded_samples_no_dividsible(add_and_remove_cv_file)
test_cv_minddataset_partition_padded_samples_dataset_size_no_divisible(add_and_remove_cv_file)
test_cv_minddataset_partition_padded_samples_no_equal_column_list(add_and_remove_cv_file)
test_cv_minddataset_partition_padded_samples_no_column_list(add_and_remove_cv_file)
test_cv_minddataset_partition_padded_samples_no_num_padded(add_and_remove_cv_file)
test_cv_minddataset_partition_padded_samples_no_padded_samples(add_and_remove_cv_file)
test_nlp_minddataset_reader_basic_padded_samples(add_and_remove_nlp_file)
test_nlp_minddataset_reader_basic_padded_samples_multi_epoch(add_and_remove_nlp_file)
test_nlp_minddataset_reader_basic_padded_samples_check_whole_reshuffle_result_per_epoch(add_and_remove_nlp_file)

29
tests/ut/python/dataset/test_minddataset_sampler.py
View File

@ -34,6 +34,7 @@ def add_and_remove_cv_file():
"""add/remove cv file"""
paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0'))
for x in range(FILES_NUM)]
try:
for x in paths:
if os.path.exists("{}".format(x)):
os.remove("{}".format(x))
@ -50,10 +51,15 @@ def add_and_remove_cv_file():
writer.write_raw_data(data)
writer.commit()
yield "yield_cv_data"
except Exception as error:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
raise error
else:
for x in paths:
os.remove("{}".format(x))
os.remove("{}.db".format(x))
def test_cv_minddataset_pk_sample_no_column(add_and_remove_cv_file):
"""tutorial for cv minderdataset."""
@ -626,3 +632,24 @@ def get_data(dir_name, sampler=False):
except FileNotFoundError:
continue
return data_list
if __name__ == '__main__':
test_cv_minddataset_pk_sample_no_column(add_and_remove_cv_file)
test_cv_minddataset_pk_sample_basic(add_and_remove_cv_file)
test_cv_minddataset_pk_sample_shuffle(add_and_remove_cv_file)
test_cv_minddataset_pk_sample_out_of_range(add_and_remove_cv_file)
test_cv_minddataset_subset_random_sample_basic(add_and_remove_cv_file)
test_cv_minddataset_subset_random_sample_replica(add_and_remove_cv_file)
test_cv_minddataset_subset_random_sample_empty(add_and_remove_cv_file)
test_cv_minddataset_subset_random_sample_out_of_range(add_and_remove_cv_file)
test_cv_minddataset_subset_random_sample_negative(add_and_remove_cv_file)
test_cv_minddataset_random_sampler_basic(add_and_remove_cv_file)
test_cv_minddataset_random_sampler_repeat(add_and_remove_cv_file)
test_cv_minddataset_random_sampler_replacement(add_and_remove_cv_file)
test_cv_minddataset_sequential_sampler_basic(add_and_remove_cv_file)
test_cv_minddataset_sequential_sampler_exceed_size(add_and_remove_cv_file)
test_cv_minddataset_split_basic(add_and_remove_cv_file)
test_cv_minddataset_split_exact_percent(add_and_remove_cv_file)
test_cv_minddataset_split_fuzzy_percent(add_and_remove_cv_file)
test_cv_minddataset_split_deterministic(add_and_remove_cv_file)
test_cv_minddataset_split_sharding(add_and_remove_cv_file)