Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!32711 [MD] fix mindrecord rename issue 

Merge pull request !32711 from liyong126/fix_rename_mindrecord_issue
This commit is contained in:
i-robot 2022-04-13 06:58:28 +00:00 committed by Gitee
parent 38b4d950a4 830e76656a
commit 4442e0f654
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
8 changed files with 380 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
mindspore/ccsrc/minddata/mindrecord/common/shard_utils.cc
View File

@ -118,7 +118,7 @@ bool CheckIsValidUtf8(const std::string &str) {
return true;
}
bool IsLegalFile(const std::string &path) {
Status CheckFile(const std::string &path) {
struct stat s;
#if defined(_WIN32) || defined(_WIN64)
if (stat(FileUtils::UTF_8ToGB2312(path.data()).data(), &s) == 0) {
@ -126,11 +126,13 @@ bool IsLegalFile(const std::string &path) {
if (stat(common::SafeCStr(path), &s) == 0) {
#endif
if (S_ISDIR(s.st_mode)) {
return false;
RETURN_STATUS_UNEXPECTED("Invalid file, " + path + " is not a mindrecord file, but got directory.");
}
return true;
return Status::OK();
}
return false;
RETURN_STATUS_UNEXPECTED(
"Invalid file, mindrecord file: " + path +
" can not be found. Please check whether the mindrecord file exists and do not rename the mindrecord file.");
}
Status GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type, std::shared_ptr<uint64_t> *size_ptr) {

2
mindspore/ccsrc/minddata/mindrecord/include/common/shard_utils.h
View File

@ -182,7 +182,7 @@ bool CheckIsValidUtf8(const std::string &str);
/// \brief judge if a path is legal file
/// \param path file path
/// \return Whether the path is legal or not
bool IsLegalFile(const std::string &path);
Status CheckFile(const std::string &path);
enum DiskSizeType { kTotalSize = 0, kFreeSize };

4
mindspore/ccsrc/minddata/mindrecord/include/shard_reader.h
View File

@ -88,10 +88,6 @@ class API_PUBLIC ShardReader {
/// \return null
void Close();
/// \brief read the file, get schema meta,statistics and index, single-thread mode
/// \return MSRStatus the status of MSRStatus
Status Open();
/// \brief read the file, get schema meta,statistics and index, multiple-thread mode
/// \return MSRStatus the status of MSRStatus
Status Open(int n_consumer);

47
mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc
View File

@ -56,9 +56,7 @@ ShardReader::ShardReader()
Status ShardReader::GetMeta(const std::string &file_path, std::shared_ptr<json> meta_data_ptr,
std::shared_ptr<std::vector<std::string>> *addresses_ptr) {
RETURN_UNEXPECTED_IF_NULL(addresses_ptr);
CHECK_FAIL_RETURN_UNEXPECTED(
IsLegalFile(file_path),
"Invalid file, failed to verify files for reading mindrecord files. Please check file: " + file_path);
RETURN_IF_NOT_OK(CheckFile(file_path));
std::shared_ptr<json> header_ptr;
RETURN_IF_NOT_OK(ShardHeader::BuildSingleHeader(file_path, &header_ptr));
@ -77,9 +75,9 @@ Status ShardReader::Init(const std::vector<std::string> &file_paths, bool load_d
if (file_paths.size() == 1 && load_dataset == true) {
auto ds = std::make_shared<std::vector<std::string>>();
RETURN_IF_NOT_OK(GetDatasetFiles(file_path, *addresses_ptr, &ds));
file_paths_ = *ds;
file_paths_ = *ds; // load files according to shard_addresses
} else if (file_paths.size() >= 1 && load_dataset == false) {
file_paths_ = file_paths;
file_paths_ = file_paths; // load files according to the input
} else {
RETURN_STATUS_UNEXPECTED("[Internal ERROR] The values of 'load_dataset' and 'file_paths' are not as expected.");
}
@ -149,8 +147,8 @@ Status ShardReader::VerifyDataset(sqlite3 **db, const string &file) {
// sqlite3_open create a database if not found, use sqlite3_open_v2 instead of it
CHECK_FAIL_RETURN_UNEXPECTED(
sqlite3_open_v2(path_utf8.data(), db, SQLITE_OPEN_READONLY, nullptr) == SQLITE_OK,
"Invalid file, failed to open mindrecord meta files while verifying meta file. Please check the meta file: " +
file + ".db");
"Invalid file, failed to open mindrecord meta file. Please check whether the meta file: " + file +
".db exists and do not rename the mindrecord file and meta file.");
MS_LOG(DEBUG) << "Succeed to open meta file, path: " << file << ".db.";
string sql = "SELECT NAME from SHARD_NAME;";
@ -169,8 +167,8 @@ Status ShardReader::VerifyDataset(sqlite3 **db, const string &file) {
if (name.empty() || name[0][0] != *fn_ptr) {
sqlite3_free(errmsg);
sqlite3_close(*db);
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to verify while reading mindrecord file: " + *fn_ptr +
". Please make sure not rename mindrecord file or .db file.");
RETURN_STATUS_UNEXPECTED("Invalid file, mindrecord meta file: " + file + ".db and mindrecord file: " + file +
" can not match. Please do not rename the mindrecord file or meta file.");
}
}
return Status::OK();
@ -187,37 +185,6 @@ Status ShardReader::CheckColumnList(const std::vector<std::string> &selected_col
return Status::OK();
}
Status ShardReader::Open() {
file_streams_.clear();
for (const auto &file : file_paths_) {
std::optional<std::string> dir = "";
std::optional<std::string> local_file_name = "";
FileUtils::SplitDirAndFileName(file, &dir, &local_file_name);
if (!dir.has_value()) {
dir = ".";
}
auto realpath = FileUtils::GetRealPath(dir.value().c_str());
CHECK_FAIL_RETURN_UNEXPECTED(
realpath.has_value(), "Invalid file, failed to get the realpath of mindrecord files. Please check file: " + file);
std::optional<std::string> whole_path = "";
FileUtils::ConcatDirAndFileName(&realpath, &local_file_name, &whole_path);
std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>();
fs->open(whole_path.value(), std::ios::in | std::ios::binary);
if (!fs->good()) {
RETURN_STATUS_UNEXPECTED(
"Invalid file, failed to open files for reading mindrecord files. Please check file path, permission and open "
"files limit(ulimit -a): " +
file);
}
file_streams_.push_back(fs);
MS_LOG(INFO) << "Succeed to open file, path: " << file;
}
return Status::OK();
}
Status ShardReader::Open(int n_consumer) {
file_streams_random_ =
std::vector<std::vector<std::shared_ptr<std::fstream>>>(n_consumer, std::vector<std::shared_ptr<std::fstream>>());

3
mindspore/ccsrc/minddata/mindrecord/io/shard_writer.cc
View File

@ -183,8 +183,7 @@ Status ShardWriter::Open(const std::vector<std::string> &paths, bool append, boo
}
Status ShardWriter::OpenForAppend(const std::string &path) {
CHECK_FAIL_RETURN_UNEXPECTED(
IsLegalFile(path), "Invalid file, failed to verify files for append mindrecord files. Please check file: " + path);
RETURN_IF_NOT_OK(CheckFile(path));
std::shared_ptr<json> header_ptr;
RETURN_IF_NOT_OK(ShardHeader::BuildSingleHeader(path, &header_ptr));
auto ds = std::make_shared<std::vector<std::string>>();

345
tests/ut/python/dataset/test_minddataset_exception.py
View File

@ -114,8 +114,7 @@ def test_minddataset_lack_db():
os.remove("{}.db".format(file_name))
columns_list = ["data", "file_name", "label"]
num_readers = 4
with pytest.raises(RuntimeError, match="Invalid file, failed to open mindrecord meta files "
"while verifying meta file. Please check the meta file:"):
with pytest.raises(RuntimeError, match=".db exists and do not rename the mindrecord file and meta file."):
data_set = ds.MindDataset(file_name, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
@ -360,6 +359,339 @@ def test_shuffle_with_num_samples_exception():
"cannot be specified at the same time."):
_ = ds.MindDataset(MIND_DIR, shuffle=ds.Shuffle.INFILE, num_samples=5)
def test_rename_exception_01():
"""
Feature: rename mindrecord file
Description: dataset that contains single mindrecord file
Expectation: exception occurred
"""
file_name = os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]
create_cv_mindrecord(file_name, 1)
new_file_name = file_name + "_new"
os.rename(file_name, new_file_name)
columns_list = ["data", "file_name", "label"]
num_readers = 4
with pytest.raises(RuntimeError, match="can not be found. Please check whether the mindrecord file exists" \
" and do not rename the mindrecord file."):
data_set = ds.MindDataset(new_file_name, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
with pytest.raises(RuntimeError, match=".db exists and do not rename the mindrecord file and meta file."):
data_set = ds.MindDataset([new_file_name], columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
os.remove(new_file_name)
os.remove(file_name + ".db")
def test_rename_exception_02():
"""
Feature: rename mindrecord meta file
Description: dataset that contains single mindrecord file
Expectation: exception occurred
"""
file_name = os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]
create_cv_mindrecord(file_name, 1)
new_file_name = file_name + "_new"
os.rename(file_name + ".db", new_file_name + ".db")
columns_list = ["data", "file_name", "label"]
num_readers = 4
with pytest.raises(RuntimeError, match=".db exists and do not rename the mindrecord file and meta file."):
data_set = ds.MindDataset(file_name, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
with pytest.raises(RuntimeError, match=".db exists and do not rename the mindrecord file and meta file."):
data_set = ds.MindDataset([file_name], columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
os.remove(file_name)
os.remove(new_file_name + ".db")
def test_rename_exception_03():
"""
Feature: rename both mindrecord file and meta file
Description: dataset that contains single mindrecord file
Expectation: exception occurred
"""
file_name = os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]
create_cv_mindrecord(file_name, 1)
new_file_name = file_name + "_new"
os.rename(file_name, new_file_name)
os.rename(file_name + ".db", new_file_name + ".db")
columns_list = ["data", "file_name", "label"]
num_readers = 4
with pytest.raises(RuntimeError, match="can not be found. Please check whether the mindrecord file exists" \
" and do not rename the mindrecord file."):
data_set = ds.MindDataset(new_file_name, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
with pytest.raises(RuntimeError, match="can not match. Please do not rename the mindrecord file or meta file."):
data_set = ds.MindDataset([new_file_name], columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
os.remove(new_file_name)
os.remove(new_file_name + ".db")
def test_rename_exception_04():
"""
Feature: rename current mindrecord file
Description: dataset that contains multiple mindrecord files
Expectation: exception occurred
"""
ori_file_name = os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]
create_cv_mindrecord(ori_file_name, 4)
file_name = ori_file_name + '0'
new_file_name = file_name + "_new"
os.rename(file_name, new_file_name)
columns_list = ["data", "file_name", "label"]
num_readers = 4
with pytest.raises(RuntimeError, match="can not be found. Please check whether the mindrecord file exists" \
" and do not rename the mindrecord file."):
data_set = ds.MindDataset(new_file_name, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
file_list = [ori_file_name + str(x) for x in range(4)]
file_list[0] = new_file_name
with pytest.raises(RuntimeError, match=".db exists and do not rename the mindrecord file and meta file."):
data_set = ds.MindDataset(file_list, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
os.remove(new_file_name)
for x in range(4):
if os.path.exists(ori_file_name + str(x)):
os.remove(ori_file_name + str(x))
if os.path.exists(ori_file_name + str(x) + ".db"):
os.remove(ori_file_name + str(x) + ".db")
def test_rename_exception_05():
"""
Feature: rename other mindrecord file
Description: dataset that contains multiple mindrecord files
Expectation: exception occurred
"""
ori_file_name = os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]
create_cv_mindrecord(ori_file_name, 4)
other_file_name = ori_file_name + '2'
new_file_name = other_file_name + "_new"
os.rename(other_file_name, new_file_name)
columns_list = ["data", "file_name", "label"]
num_readers = 4
file_name = ori_file_name + '0'
with pytest.raises(RuntimeError, match="can not be found. Please check whether the mindrecord file exists" \
" and do not rename the mindrecord file."):
data_set = ds.MindDataset(file_name, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
file_list = [ori_file_name + str(x) for x in range(4)]
file_list[2] = new_file_name
with pytest.raises(RuntimeError, match=".db exists and do not rename the mindrecord file and meta file."):
data_set = ds.MindDataset(file_list, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
os.remove(new_file_name)
for x in range(4):
if os.path.exists(ori_file_name + str(x)):
os.remove(ori_file_name + str(x))
if os.path.exists(ori_file_name + str(x) + ".db"):
os.remove(ori_file_name + str(x) + ".db")
def test_rename_exception_06():
"""
Feature: rename current meta file
Description: dataset that contains multiple mindrecord files
Expectation: exception occurred
"""
ori_file_name = os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]
create_cv_mindrecord(ori_file_name, 4)
file_name = ori_file_name + '0'
new_file_name = file_name + "_new"
os.rename(file_name + ".db", new_file_name + ".db")
columns_list = ["data", "file_name", "label"]
num_readers = 4
with pytest.raises(RuntimeError, match=".db exists and do not rename the mindrecord file and meta file."):
data_set = ds.MindDataset(file_name, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
file_list = [ori_file_name + str(x) for x in range(4)]
with pytest.raises(RuntimeError, match=".db exists and do not rename the mindrecord file and meta file."):
data_set = ds.MindDataset(file_list, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
os.remove(new_file_name + ".db")
for x in range(4):
if os.path.exists(ori_file_name + str(x)):
os.remove(ori_file_name + str(x))
if os.path.exists(ori_file_name + str(x) + ".db"):
os.remove(ori_file_name + str(x) + ".db")
def test_rename_exception_07():
"""
Feature: rename other meta file
Description: dataset that contains multiple mindrecord files
Expectation: exception occurred
"""
ori_file_name = os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]
create_cv_mindrecord(ori_file_name, 4)
other_file_name = ori_file_name + '2'
new_file_name = other_file_name + "_new"
os.rename(other_file_name + ".db", new_file_name + ".db")
file_name = ori_file_name + '0'
columns_list = ["data", "file_name", "label"]
num_readers = 4
with pytest.raises(RuntimeError, match=".db exists and do not rename the mindrecord file and meta file."):
data_set = ds.MindDataset(file_name, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
file_list = [ori_file_name + str(x) for x in range(4)]
with pytest.raises(RuntimeError, match=".db exists and do not rename the mindrecord file and meta file."):
data_set = ds.MindDataset(file_list, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
os.remove(new_file_name + ".db")
for x in range(4):
if os.path.exists(ori_file_name + str(x)):
os.remove(ori_file_name + str(x))
if os.path.exists(ori_file_name + str(x) + ".db"):
os.remove(ori_file_name + str(x) + ".db")
def test_rename_exception_08():
"""
Feature: rename both current mindrecord file and meta file
Description: dataset that contains multiple mindrecord files
Expectation: exception occurred
"""
ori_file_name = os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]
create_cv_mindrecord(ori_file_name, 4)
file_name = ori_file_name + '0'
new_file_name = file_name + "_new"
os.rename(file_name, new_file_name)
os.rename(file_name + ".db", new_file_name + ".db")
columns_list = ["data", "file_name", "label"]
num_readers = 4
with pytest.raises(RuntimeError, match="can not be found. Please check whether the mindrecord file exists" \
" and do not rename the mindrecord file."):
data_set = ds.MindDataset(new_file_name, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
file_list = [ori_file_name + str(x) for x in range(4)]
file_list[0] = new_file_name
with pytest.raises(RuntimeError, match="can not match. Please do not rename the mindrecord file or meta file."):
data_set = ds.MindDataset(file_list, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
os.remove(new_file_name)
os.remove(new_file_name + ".db")
for x in range(4):
if os.path.exists(ori_file_name + str(x)):
os.remove(ori_file_name + str(x))
if os.path.exists(ori_file_name + str(x) + ".db"):
os.remove(ori_file_name + str(x) + ".db")
def test_rename_exception_09():
"""
Feature: rename both other mindrecord file and meta file
Description: dataset that contains multiple mindrecord files
Expectation: exception occurred
"""
ori_file_name = os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]
create_cv_mindrecord(ori_file_name, 4)
other_file_name = ori_file_name + '2'
new_file_name = other_file_name + "_new"
os.rename(other_file_name, new_file_name)
os.rename(other_file_name + ".db", new_file_name + ".db")
columns_list = ["data", "file_name", "label"]
num_readers = 4
file_name = ori_file_name + '0'
with pytest.raises(RuntimeError, match="can not be found. Please check whether the mindrecord file exists" \
" and do not rename the mindrecord file."):
data_set = ds.MindDataset(file_name, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
file_list = [ori_file_name + str(x) for x in range(4)]
file_list[2] = new_file_name
with pytest.raises(RuntimeError, match="can not match. Please do not rename the mindrecord file or meta file."):
data_set = ds.MindDataset(file_list, columns_list, num_readers)
num_iter = 0
for _ in data_set.create_dict_iterator(num_epochs=1, output_numpy=True):
num_iter += 1
os.remove(new_file_name)
os.remove(new_file_name + ".db")
for x in range(4):
if os.path.exists(ori_file_name + str(x)):
os.remove(ori_file_name + str(x))
if os.path.exists(ori_file_name + str(x) + ".db"):
os.remove(ori_file_name + str(x) + ".db")
if __name__ == '__main__':
test_cv_lack_json()
test_cv_lack_mindrecord()
@ -374,3 +706,12 @@ if __name__ == '__main__':
test_minddataset_shard_id_bigger_than_num_shard()
test_cv_minddataset_partition_num_samples_equals_0()
test_mindrecord_exception()
test_rename_exception_01()
test_rename_exception_02()
test_rename_exception_03()
test_rename_exception_04()
test_rename_exception_05()
test_rename_exception_06()
test_rename_exception_07()
test_rename_exception_08()
test_rename_exception_09()

30
tests/ut/python/mindrecord/test_mindrecord_exception.py
View File

@ -103,8 +103,8 @@ def test_lack_partition_and_db():
with pytest.raises(RuntimeError) as err:
reader = FileReader('dummy.mindrecord')
reader.close()
assert "Unexpected error. Invalid file, failed to verify files for reading mindrecord files. " \
"Please check file:" in str(err.value)
assert "can not be found. Please check whether the mindrecord file exists" \
" and do not rename the mindrecord file." in str(err.value)
def test_lack_db():
"""
@ -118,8 +118,7 @@ def test_lack_db():
with pytest.raises(RuntimeError) as err:
reader = FileReader(file_name)
reader.close()
assert "Unexpected error. Invalid file, failed to open mindrecord meta files while verifying meta file. " \
"Please check the meta file:" in str(err.value)
assert ".db exists and do not rename the mindrecord file and meta file." in str(err.value)
remove_file(file_name)
def test_lack_some_partition_and_db():
@ -137,8 +136,8 @@ def test_lack_some_partition_and_db():
with pytest.raises(RuntimeError) as err:
reader = FileReader(file_name + "0")
reader.close()
assert "Unexpected error. Invalid file, failed to verify files for reading mindrecord files. " \
"Please check file:" in str(err.value)
assert "can not be found. Please check whether the mindrecord file exists" \
" and do not rename the mindrecord file." in str(err.value)
remove_file(file_name)
def test_lack_some_partition_first():
@ -155,8 +154,8 @@ def test_lack_some_partition_first():
with pytest.raises(RuntimeError) as err:
reader = FileReader(file_name + "0")
reader.close()
assert "Unexpected error. Invalid file, failed to verify files for reading mindrecord files. " \
"Please check file:" in str(err.value)
assert "can not be found. Please check whether the mindrecord file exists" \
" and do not rename the mindrecord file." in str(err.value)
remove_file(file_name)
def test_lack_some_partition_middle():
@ -173,8 +172,8 @@ def test_lack_some_partition_middle():
with pytest.raises(RuntimeError) as err:
reader = FileReader(file_name + "0")
reader.close()
assert "Unexpected error. Invalid file, failed to verify files for reading mindrecord files. " \
"Please check file:" in str(err.value)
assert "can not be found. Please check whether the mindrecord file exists" \
" and do not rename the mindrecord file." in str(err.value)
remove_file(file_name)
def test_lack_some_partition_last():
@ -191,8 +190,8 @@ def test_lack_some_partition_last():
with pytest.raises(RuntimeError) as err:
reader = FileReader(file_name + "0")
reader.close()
assert "Unexpected error. Invalid file, failed to verify files for reading mindrecord files. " \
"Please check file:" in str(err.value)
assert "can not be found. Please check whether the mindrecord file exists" \
" and do not rename the mindrecord file." in str(err.value)
remove_file(file_name)
def test_mindpage_lack_some_partition():
@ -208,8 +207,8 @@ def test_mindpage_lack_some_partition():
os.remove("{}".format(paths[0]))
with pytest.raises(RuntimeError) as err:
MindPage(file_name + "0")
assert "Unexpected error. Invalid file, failed to verify files for reading mindrecord files. " \
"Please check file:" in str(err.value)
assert "can not be found. Please check whether the mindrecord file exists" \
" and do not rename the mindrecord file." in str(err.value)
remove_file(file_name)
def test_lack_some_db():
@ -226,8 +225,7 @@ def test_lack_some_db():
with pytest.raises(RuntimeError) as err:
reader = FileReader(file_name + "0")
reader.close()
assert "Unexpected error. Invalid file, failed to open mindrecord meta files while verifying meta file. " \
"Please check the meta file:" in str(err.value)
assert ".db exists and do not rename the mindrecord file and meta file." in str(err.value)
remove_file(file_name)
def test_invalid_mindrecord():

8
tests/ut/python/mindrecord/test_mindrecord_multi_images.py
View File

@ -102,6 +102,10 @@ def test_write_two_diff_shape_images_mindrecord():
assert os.path.exists(file_name)
assert os.path.exists(file_name + ".db")
read(file_name, bytes_num)
if os.path.exists("{}".format(file_name + ".db")):
os.remove(file_name + ".db")
if os.path.exists("{}".format(file_name)):
os.remove(file_name)
def test_write_multi_images_mindrecord():
@ -122,6 +126,10 @@ def test_write_multi_images_mindrecord():
assert os.path.exists(file_name)
assert os.path.exists(file_name + ".db")
read(file_name, bytes_num)
if os.path.exists("{}".format(file_name + ".db")):
os.remove(file_name + ".db")
if os.path.exists("{}".format(file_name)):
os.remove(file_name)
def test_write_two_images_and_array_mindrecord():