sfqcyy
/
mindspore
forked from OSSInnovation/mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'ql/fix_cache_core' of https://gitee.com/qianlong21st/mindspore into ql/fix_cache_core

This commit is contained in:
guansongsong 2020-07-28 11:37:06 +08:00
parent 16079e6356 bf87b1d1fb
commit 620d951e5e
2 changed files with 36 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc
View File

@ -387,6 +387,9 @@ uint32_t DatasetOp::GenerateCRC(const std::shared_ptr<DatasetOp> &op) {
ss_str = std::regex_replace(ss_str, std::regex("Num workers.*\n"), ""); ss_str = std::regex_replace(ss_str, std::regex("Num workers.*\n"), "");
ss_str = std::regex_replace(ss_str, std::regex("\\[workers.*\\]"), ""); ss_str = std::regex_replace(ss_str, std::regex("\\[workers.*\\]"), "");
// Filter out Number of rows when generating the check sum
ss_str = std::regex_replace(ss_str, std::regex("Number of rows.*\n"), "");
// Filter out the Operator control flags field when generating the check sum // Filter out the Operator control flags field when generating the check sum
ss_str = std::regex_replace(ss_str, std::regex("Operator control flags.*\n"), ""); ss_str = std::regex_replace(ss_str, std::regex("Operator control flags.*\n"), "");

39
tests/ut/python/dataset/test_cache_map.py
View File

@ -104,11 +104,11 @@ def test_cache_map_basic3():
decode_op = c_vision.Decode() decode_op = c_vision.Decode()
ds1 = ds1.repeat(4) ds1 = ds1.repeat(4)
ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache) ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache)
print("ds1.dataset_size is ", ds1.get_dataset_size()) logger.info("ds1.dataset_size is ", ds1.get_dataset_size())
num_iter = 0 num_iter = 0
for _ in ds1.create_dict_iterator(): for _ in ds1.create_dict_iterator():
print("get data from dataset") logger.info("get data from dataset")
num_iter += 1 num_iter += 1
logger.info("Number of data in ds1: {} ".format(num_iter)) logger.info("Number of data in ds1: {} ".format(num_iter))
@ -116,6 +116,31 @@ def test_cache_map_basic3():
logger.info('test_cache_basic3 Ended.\n') logger.info('test_cache_basic3 Ended.\n')
def test_cache_map_basic4():
"""
Test different rows result in core dump
"""
logger.info("Test cache basic 4")
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache)
decode_op = c_vision.Decode()
ds1 = ds1.repeat(4)
ds1 = ds1.map(input_columns=["image"], operations=decode_op)
logger.info("ds1.dataset_size is ", ds1.get_dataset_size())
shape = ds1.output_shapes()
num_iter = 0
for _ in ds1.create_dict_iterator():
logger.info("get data from dataset")
num_iter += 1
logger.info("Number of data in ds1: {} ".format(num_iter))
assert num_iter == 8
logger.info('test_cache_basic3 Ended.\n')
def test_cache_map_failure1(): def test_cache_map_failure1():
""" """
Test nested cache (failure) Test nested cache (failure)
@ -154,10 +179,12 @@ def test_cache_map_failure1():
if __name__ == '__main__': if __name__ == '__main__':
test_cache_map_basic1() test_cache_map_basic1()
print("test_cache_map_basic1 success.") logger.info("test_cache_map_basic1 success.")
test_cache_map_basic2() test_cache_map_basic2()
print("test_cache_map_basic2 success.") logger.info("test_cache_map_basic2 success.")
test_cache_map_basic3() test_cache_map_basic3()
print("test_cache_map_basic3 success.") logger.info("test_cache_map_basic3 success.")
test_cache_map_basic4()
logger.info("test_cache_map_basic3 success.")
test_cache_map_failure1() test_cache_map_failure1()
print("test_cache_map_failure1 success.") logger.info("test_cache_map_failure1 success.")