mindspore-Ecosystem
/
mindspore
21
0
Fork 27
Code Issues Pull Requests 6 Projects Releases Wiki Activity

!28378 fix test data path for chinese path 

Merge pull request !28378 from guozhijian/ensure_chinese_path
This commit is contained in:
i-robot 2022-01-05 01:15:54 +00:00 committed by Gitee
parent 15bc71dca2 fd6721d872
commit 089239f485
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
4 changed files with 19 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mindspore/ccsrc/minddata/mindrecord/io/shard_writer.cc
View File

@ -120,7 +120,7 @@ Status ShardWriter::OpenDataFiles(bool append, bool overwrite) {
fs_db.close();
}
// open the mindrecord file to write
fs->open(common::SafeCStr(file), std::ios::out | std::ios::in | std::ios::binary | std::ios::trunc);
fs->open(whole_path.value().data(), std::ios::out | std::ios::in | std::ios::binary | std::ios::trunc);
if (!fs->good()) {
RETURN_STATUS_UNEXPECTED(
"Invalid file, failed to open files for writing mindrecord files. Please check file path, permission and "
@ -129,7 +129,7 @@ Status ShardWriter::OpenDataFiles(bool append, bool overwrite) {
}
} else {
// open the mindrecord file to append
fs->open(common::SafeCStr(file), std::ios::out | std::ios::in | std::ios::binary);
fs->open(whole_path.value().data(), std::ios::out | std::ios::in | std::ios::binary);
if (!fs->good()) {
fs->close();
RETURN_STATUS_UNEXPECTED(

23
mindspore/core/utils/file_utils.cc
View File

@ -28,12 +28,15 @@
#if defined(_WIN32) || defined(_WIN64)
#include <windows.h>
#include <wchar.h>
#undef ERROR // which is in wingdi.h and conflict with log_adaptor.h
#endif
namespace mindspore {
#if defined(_WIN32) || defined(_WIN64)
int IncludeChinese(const char *str) {
if (str == nullptr) {
MS_LOG(ERROR) << "Input str is nullptr";
return 0;
}
@ -150,6 +153,7 @@ void UnicodeToGB2312(char *p_out, WCHAR u_data) {
std::string FileUtils::UTF_8ToGB2312(const char *text) {
if (text == nullptr) {
MS_LOG(ERROR) << "Input text is nullptr";
return "";
}
@ -167,8 +171,9 @@ std::string FileUtils::UTF_8ToGB2312(const char *text) {
int len = strlen(text);
char *new_text = const_cast<char *>(text);
std::unique_ptr<char[]> rst(new char[len + (len >> 2) + 2]);
auto ret2 = memset_s(rst.get(), len + (len >> 2) + 2, 0, len + (len >> 2) + 2);
if (ret2 != 0) {
auto ret = memset_s(rst.get(), len + (len >> 2) + 2, 0, len + (len >> 2) + 2);
if (ret != 0) {
MS_LOG(ERROR) << "memset_s error, error code: " << ret;
return "";
}
@ -200,6 +205,7 @@ std::string FileUtils::UTF_8ToGB2312(const char *text) {
// gb2312 to utf8
std::string FileUtils::GB2312ToUTF_8(const char *gb2312) {
if (gb2312 == nullptr) {
MS_LOG(ERROR) << "Input string gb2312 is nullptr";
return "";
}
@ -211,6 +217,7 @@ std::string FileUtils::GB2312ToUTF_8(const char *gb2312) {
std::unique_ptr<wchar_t[]> wstr(new wchar_t[len + 1]);
auto ret = memset_s(wstr.get(), len + 1, 0, len + 1);
if (ret != 0) {
MS_LOG(ERROR) << "memset_s error, error code: " << ret;
return "";
}
MultiByteToWideChar(CP_ACP, 0, gb2312, -1, wstr.get(), len);
@ -219,6 +226,7 @@ std::string FileUtils::GB2312ToUTF_8(const char *gb2312) {
std::unique_ptr<char[]> str(new char[len + 1]);
auto ret2 = memset_s(str.get(), len + 1, 0, len + 1);
if (ret2 != 0) {
MS_LOG(ERROR) << "memset_s error, error code: " << ret2;
return "";
}
WideCharToMultiByte(CP_UTF8, 0, wstr.get(), -1, str.get(), len, nullptr, nullptr);
@ -230,9 +238,7 @@ std::string FileUtils::GB2312ToUTF_8(const char *gb2312) {
std::optional<std::string> FileUtils::GetRealPath(const char *path) {
if (path == nullptr) {
#if !defined(_WIN32) && !defined(_WIN64)
MS_LOG(ERROR) << "Input path is nullptr";
#endif
return std::nullopt;
}
@ -240,6 +246,7 @@ std::optional<std::string> FileUtils::GetRealPath(const char *path) {
#if defined(_WIN32) || defined(_WIN64)
std::string new_path = FileUtils::UTF_8ToGB2312(path);
if (new_path.length() >= PATH_MAX || _fullpath(real_path, new_path.data(), PATH_MAX) == nullptr) {
MS_LOG(ERROR) << "Get realpath failed, path[" << path << "]";
return std::nullopt;
}
#else
@ -284,17 +291,13 @@ void FileUtils::ConcatDirAndFileName(const std::optional<std::string> *dir, cons
std::optional<std::string> FileUtils::CreateNotExistDirs(const std::string &path, const bool support_relative_path) {
if (path.size() >= PATH_MAX) {
#if !defined(_WIN32) && !defined(_WIN64)
MS_LOG(ERROR) << "The length of the path is greater than or equal to:" << PATH_MAX;
#endif
return std::nullopt;
}
if (!support_relative_path) {
auto dot_pos = path.find("..");
if (dot_pos != std::string::npos) {
#if !defined(_WIN32) && !defined(_WIN64)
MS_LOG(ERROR) << "Do not support relative path";
#endif
return std::nullopt;
}
}
@ -311,9 +314,7 @@ std::optional<std::string> FileUtils::CreateNotExistDirs(const std::string &path
std::string path_handle(temp_path);
if (!fs->FileExist(path_handle)) {
if (!fs->CreateDir(path_handle)) {
#if !defined(_WIN32) && !defined(_WIN64)
MS_LOG(ERROR) << "Create " << path_handle << " dir error";
#endif
return std::nullopt;
}
}
@ -324,9 +325,7 @@ std::optional<std::string> FileUtils::CreateNotExistDirs(const std::string &path
if (!fs->FileExist(path)) {
if (!fs->CreateDir(path)) {
#if !defined(_WIN32) && !defined(_WIN64)
MS_LOG(ERROR) << "Create " << path << " dir error";
#endif
return std::nullopt;
}
}

1
tests/st/data/数据集/train/训练集/Readme.txt
View File

@ -1 +0,0 @@
# test chinese path when platform is windows

9
tests/st/dataset/test_chinese_path_on_windows.py
View File

@ -87,8 +87,8 @@ def test_chinese_path_on_windows():
Expectation: raise axception
"""
mindrecord_file_name = os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]
cv_mindrecord_file = "../data/" + mindrecord_file_name
cv_dir_name_cn = "../data/数据集/train/"
cv_mindrecord_file = "./data/" + mindrecord_file_name
cv_dir_name_cn = "./data/数据集/train/"
file_name = mindrecord_file_name
file_name2 = "./训练集/" + mindrecord_file_name
@ -97,6 +97,9 @@ def test_chinese_path_on_windows():
current_pwd = os.getcwd()
# create chinese path for test
os.makedirs("data/数据集/train/训练集")
# current dir in english, mindrecord path in english
dir_path = "./"
mindrecord_path = cv_mindrecord_file
@ -111,7 +114,7 @@ def test_chinese_path_on_windows():
# current dir in english, mindrecord path in chinese
dir_path = "./"
mindrecord_path = cv_dir_name_cn + "/" + file_name
mindrecord_path = cv_dir_name_cn + file_name
add_and_remove_cv_file(dir_path + mindrecord_path)