p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!277 fix: use correct permissions when opening mindrecord files 

Merge pull request !277 from guozhijian/fix_read_with_exact_option
This commit is contained in:
mindspore-ci-bot 2020-04-14 21:23:10 +08:00 committed by Gitee
parent 57aee805ce ea5da25d10
commit d9dd6aa0b8
3 changed files with 36 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mindspore/ccsrc/mindrecord/io/shard_index_generator.cc
View File

@ -512,6 +512,10 @@ MSRStatus ShardIndexGenerator::ExecuteTransaction(const int &shard_no, const std
std::fstream in; std::fstream in;
in.open(common::SafeCStr(shard_address), std::ios::in | std::ios::binary); in.open(common::SafeCStr(shard_address), std::ios::in | std::ios::binary);
if (!in.good()) {
MS_LOG(ERROR) << "File could not opened";
return FAILED;
}
(void)sqlite3_exec(db.second, "BEGIN TRANSACTION;", nullptr, nullptr, nullptr); (void)sqlite3_exec(db.second, "BEGIN TRANSACTION;", nullptr, nullptr, nullptr);
for (int raw_page_id : raw_page_ids) { for (int raw_page_id : raw_page_ids) {
auto sql = GenerateRawSQL(fields_); auto sql = GenerateRawSQL(fields_);

26
mindspore/ccsrc/mindrecord/io/shard_reader.cc
View File

@ -125,14 +125,11 @@ MSRStatus ShardReader::Open() {
for (const auto &file : file_paths_) { for (const auto &file : file_paths_) {
std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>(); std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>();
fs->open(common::SafeCStr(file), std::ios::in | std::ios::out | std::ios::binary); fs->open(common::SafeCStr(file), std::ios::in | std::ios::binary);
if (fs->fail()) { if (!fs->good()) {
fs->open(common::SafeCStr(file), std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary);
if (fs->fail()) {
MS_LOG(ERROR) << "File could not opened"; MS_LOG(ERROR) << "File could not opened";
return FAILED; return FAILED;
} }
}
MS_LOG(INFO) << "Open shard file successfully."; MS_LOG(INFO) << "Open shard file successfully.";
file_streams_.push_back(fs); file_streams_.push_back(fs);
} }
@ -146,14 +143,11 @@ MSRStatus ShardReader::Open(int n_consumer) {
for (const auto &file : file_paths_) { for (const auto &file : file_paths_) {
for (int j = 0; j < n_consumer; ++j) { for (int j = 0; j < n_consumer; ++j) {
std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>(); std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>();
fs->open(common::SafeCStr(file), std::ios::in | std::ios::out | std::ios::binary); fs->open(common::SafeCStr(file), std::ios::in | std::ios::binary);
if (fs->fail()) { if (!fs->good()) {
fs->open(common::SafeCStr(file), std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary);
if (fs->fail()) {
MS_LOG(ERROR) << "File could not opened"; MS_LOG(ERROR) << "File could not opened";
return FAILED; return FAILED;
} }
}
file_streams_random_[j].push_back(fs); file_streams_random_[j].push_back(fs);
} }
MS_LOG(INFO) << "Open shard file successfully."; MS_LOG(INFO) << "Open shard file successfully.";
@ -311,12 +305,10 @@ MSRStatus ShardReader::ReadAllRowsInShard(int shard_id, const std::string &sql,
std::string file_name = file_paths_[shard_id]; std::string file_name = file_paths_[shard_id];
std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>(); std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>();
if (!all_in_index_) { if (!all_in_index_) {
fs->open(common::SafeCStr(file_name), std::ios::in | std::ios::out | std::ios::binary); fs->open(common::SafeCStr(file_name), std::ios::in | std::ios::binary);
if (fs->fail()) { if (!fs->good()) {
fs->open(common::SafeCStr(file_name), std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary);
if (fs->fail()) {
MS_LOG(ERROR) << "File could not opened"; MS_LOG(ERROR) << "File could not opened";
} return FAILED;
} }
} }
sqlite3_free(errmsg); sqlite3_free(errmsg);
@ -520,8 +512,8 @@ std::pair<MSRStatus, std::vector<json>> ShardReader::GetLabelsFromBinaryFile(
std::string file_name = file_paths_[shard_id]; std::string file_name = file_paths_[shard_id];
std::vector<json> res; std::vector<json> res;
std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>(); std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>();
fs->open(common::SafeCStr(file_name), std::ios::in | std::ios::out | std::ios::binary); fs->open(common::SafeCStr(file_name), std::ios::in | std::ios::binary);
if (fs->fail()) { if (!fs->good()) {
MS_LOG(ERROR) << "File could not opened"; MS_LOG(ERROR) << "File could not opened";
return {FAILED, {}}; return {FAILED, {}};
} }

25
mindspore/ccsrc/mindrecord/io/shard_writer.cc
View File

@ -76,16 +76,27 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append)
// Open files // Open files
for (const auto &file : file_paths_) { for (const auto &file : file_paths_) {
std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>(); std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>();
fs->open(common::SafeCStr(file), std::ios::in | std::ios::out | std::ios::binary); if (!append) {
if (fs->fail()) { // if not append and mindrecord file exist, return FAILED
fs->open(common::SafeCStr(file), std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary); fs->open(common::SafeCStr(file), std::ios::in | std::ios::binary);
if (fs->fail()) { if (fs->good()) {
MS_LOG(ERROR) << "File could not opened"; MS_LOG(ERROR) << "MindRecord file already existed.";
fs->close();
return FAILED;
}
fs->close();
// open the mindrecord file to write
fs->open(common::SafeCStr(file), std::ios::out | std::ios::binary);
if (!fs->good()) {
MS_LOG(ERROR) << "MindRecord file could not opened.";
return FAILED; return FAILED;
} }
} else { } else {
if (!append) { // open the mindrecord file to append
MS_LOG(ERROR) << "MindRecord file already existed"; fs->open(common::SafeCStr(file), std::ios::out | std::ios::in | std::ios::binary);
if (!fs->good()) {
MS_LOG(ERROR) << "MindRecord file could not opened for append.";
return FAILED; return FAILED;
} }
} }