From fd6721d872f9bb3b8bd564fa06c6bb199f864c95 Mon Sep 17 00:00:00 2001 From: jonyguo Date: Wed, 29 Dec 2021 17:28:03 +0800 Subject: [PATCH] fix: ERROR conflict with windows.h and testcase path --- .../minddata/mindrecord/io/shard_writer.cc | 4 ++-- mindspore/core/utils/file_utils.cc | 23 +++++++++---------- .../data/数据集/train/训练集/Readme.txt | 1 - .../dataset/test_chinese_path_on_windows.py | 9 +++++--- 4 files changed, 19 insertions(+), 18 deletions(-) delete mode 100644 tests/st/data/数据集/train/训练集/Readme.txt diff --git a/mindspore/ccsrc/minddata/mindrecord/io/shard_writer.cc b/mindspore/ccsrc/minddata/mindrecord/io/shard_writer.cc index d1b93248670..c4a4be449e3 100644 --- a/mindspore/ccsrc/minddata/mindrecord/io/shard_writer.cc +++ b/mindspore/ccsrc/minddata/mindrecord/io/shard_writer.cc @@ -120,7 +120,7 @@ Status ShardWriter::OpenDataFiles(bool append, bool overwrite) { fs_db.close(); } // open the mindrecord file to write - fs->open(common::SafeCStr(file), std::ios::out | std::ios::in | std::ios::binary | std::ios::trunc); + fs->open(whole_path.value().data(), std::ios::out | std::ios::in | std::ios::binary | std::ios::trunc); if (!fs->good()) { RETURN_STATUS_UNEXPECTED( "Invalid file, failed to open files for writing mindrecord files. Please check file path, permission and " @@ -129,7 +129,7 @@ Status ShardWriter::OpenDataFiles(bool append, bool overwrite) { } } else { // open the mindrecord file to append - fs->open(common::SafeCStr(file), std::ios::out | std::ios::in | std::ios::binary); + fs->open(whole_path.value().data(), std::ios::out | std::ios::in | std::ios::binary); if (!fs->good()) { fs->close(); RETURN_STATUS_UNEXPECTED( diff --git a/mindspore/core/utils/file_utils.cc b/mindspore/core/utils/file_utils.cc index bdfd8d8a9c0..2e3f4eda7b3 100644 --- a/mindspore/core/utils/file_utils.cc +++ b/mindspore/core/utils/file_utils.cc @@ -28,12 +28,15 @@ #if defined(_WIN32) || defined(_WIN64) #include #include + +#undef ERROR // which is in wingdi.h and conflict with log_adaptor.h #endif namespace mindspore { #if defined(_WIN32) || defined(_WIN64) int IncludeChinese(const char *str) { if (str == nullptr) { + MS_LOG(ERROR) << "Input str is nullptr"; return 0; } @@ -150,6 +153,7 @@ void UnicodeToGB2312(char *p_out, WCHAR u_data) { std::string FileUtils::UTF_8ToGB2312(const char *text) { if (text == nullptr) { + MS_LOG(ERROR) << "Input text is nullptr"; return ""; } @@ -167,8 +171,9 @@ std::string FileUtils::UTF_8ToGB2312(const char *text) { int len = strlen(text); char *new_text = const_cast(text); std::unique_ptr rst(new char[len + (len >> 2) + 2]); - auto ret2 = memset_s(rst.get(), len + (len >> 2) + 2, 0, len + (len >> 2) + 2); - if (ret2 != 0) { + auto ret = memset_s(rst.get(), len + (len >> 2) + 2, 0, len + (len >> 2) + 2); + if (ret != 0) { + MS_LOG(ERROR) << "memset_s error, error code: " << ret; return ""; } @@ -200,6 +205,7 @@ std::string FileUtils::UTF_8ToGB2312(const char *text) { // gb2312 to utf8 std::string FileUtils::GB2312ToUTF_8(const char *gb2312) { if (gb2312 == nullptr) { + MS_LOG(ERROR) << "Input string gb2312 is nullptr"; return ""; } @@ -211,6 +217,7 @@ std::string FileUtils::GB2312ToUTF_8(const char *gb2312) { std::unique_ptr wstr(new wchar_t[len + 1]); auto ret = memset_s(wstr.get(), len + 1, 0, len + 1); if (ret != 0) { + MS_LOG(ERROR) << "memset_s error, error code: " << ret; return ""; } MultiByteToWideChar(CP_ACP, 0, gb2312, -1, wstr.get(), len); @@ -219,6 +226,7 @@ std::string FileUtils::GB2312ToUTF_8(const char *gb2312) { std::unique_ptr str(new char[len + 1]); auto ret2 = memset_s(str.get(), len + 1, 0, len + 1); if (ret2 != 0) { + MS_LOG(ERROR) << "memset_s error, error code: " << ret2; return ""; } WideCharToMultiByte(CP_UTF8, 0, wstr.get(), -1, str.get(), len, nullptr, nullptr); @@ -230,9 +238,7 @@ std::string FileUtils::GB2312ToUTF_8(const char *gb2312) { std::optional FileUtils::GetRealPath(const char *path) { if (path == nullptr) { -#if !defined(_WIN32) && !defined(_WIN64) MS_LOG(ERROR) << "Input path is nullptr"; -#endif return std::nullopt; } @@ -240,6 +246,7 @@ std::optional FileUtils::GetRealPath(const char *path) { #if defined(_WIN32) || defined(_WIN64) std::string new_path = FileUtils::UTF_8ToGB2312(path); if (new_path.length() >= PATH_MAX || _fullpath(real_path, new_path.data(), PATH_MAX) == nullptr) { + MS_LOG(ERROR) << "Get realpath failed, path[" << path << "]"; return std::nullopt; } #else @@ -284,17 +291,13 @@ void FileUtils::ConcatDirAndFileName(const std::optional *dir, cons std::optional FileUtils::CreateNotExistDirs(const std::string &path, const bool support_relative_path) { if (path.size() >= PATH_MAX) { -#if !defined(_WIN32) && !defined(_WIN64) MS_LOG(ERROR) << "The length of the path is greater than or equal to:" << PATH_MAX; -#endif return std::nullopt; } if (!support_relative_path) { auto dot_pos = path.find(".."); if (dot_pos != std::string::npos) { -#if !defined(_WIN32) && !defined(_WIN64) MS_LOG(ERROR) << "Do not support relative path"; -#endif return std::nullopt; } } @@ -311,9 +314,7 @@ std::optional FileUtils::CreateNotExistDirs(const std::string &path std::string path_handle(temp_path); if (!fs->FileExist(path_handle)) { if (!fs->CreateDir(path_handle)) { -#if !defined(_WIN32) && !defined(_WIN64) MS_LOG(ERROR) << "Create " << path_handle << " dir error"; -#endif return std::nullopt; } } @@ -324,9 +325,7 @@ std::optional FileUtils::CreateNotExistDirs(const std::string &path if (!fs->FileExist(path)) { if (!fs->CreateDir(path)) { -#if !defined(_WIN32) && !defined(_WIN64) MS_LOG(ERROR) << "Create " << path << " dir error"; -#endif return std::nullopt; } } diff --git a/tests/st/data/数据集/train/训练集/Readme.txt b/tests/st/data/数据集/train/训练集/Readme.txt deleted file mode 100644 index c8ff853fe9f..00000000000 --- a/tests/st/data/数据集/train/训练集/Readme.txt +++ /dev/null @@ -1 +0,0 @@ -# test chinese path when platform is windows diff --git a/tests/st/dataset/test_chinese_path_on_windows.py b/tests/st/dataset/test_chinese_path_on_windows.py index 0a408d3db6b..06e102f4f07 100644 --- a/tests/st/dataset/test_chinese_path_on_windows.py +++ b/tests/st/dataset/test_chinese_path_on_windows.py @@ -87,8 +87,8 @@ def test_chinese_path_on_windows(): Expectation: raise axception """ mindrecord_file_name = os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0] - cv_mindrecord_file = "../data/" + mindrecord_file_name - cv_dir_name_cn = "../data/数据集/train/" + cv_mindrecord_file = "./data/" + mindrecord_file_name + cv_dir_name_cn = "./data/数据集/train/" file_name = mindrecord_file_name file_name2 = "./训练集/" + mindrecord_file_name @@ -97,6 +97,9 @@ def test_chinese_path_on_windows(): current_pwd = os.getcwd() + # create chinese path for test + os.makedirs("data/数据集/train/训练集") + # current dir in english, mindrecord path in english dir_path = "./" mindrecord_path = cv_mindrecord_file @@ -111,7 +114,7 @@ def test_chinese_path_on_windows(): # current dir in english, mindrecord path in chinese dir_path = "./" - mindrecord_path = cv_dir_name_cn + "/" + file_name + mindrecord_path = cv_dir_name_cn + file_name add_and_remove_cv_file(dir_path + mindrecord_path)