From 20d1b6444353e806ad37989f7a7c8b2c10418536 Mon Sep 17 00:00:00 2001 From: jonyguo Date: Wed, 8 Apr 2020 15:52:53 +0800 Subject: [PATCH] fix: error info is not exactly when column list invalid --- .../engine/datasetops/source/mindrecord_op.cc | 3 +- .../engine/datasetops/source/mindrecord_op.h | 1 + .../ccsrc/mindrecord/common/shard_error.cc | 178 ++++++++++++++++++ .../ccsrc/mindrecord/include/shard_error.h | 102 +++++----- mindspore/ccsrc/mindrecord/io/shard_reader.cc | 2 +- tests/ut/cpp/dataset/mind_record_op_test.cc | 36 ++++ .../ut/cpp/mindrecord/ut_shard_reader_test.cc | 2 +- 7 files changed, 271 insertions(+), 53 deletions(-) create mode 100644 mindspore/ccsrc/mindrecord/common/shard_error.cc diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc index b5bea5416ca..cb0f135a0dd 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc @@ -103,7 +103,8 @@ Status MindRecordOp::Init() { shard_reader_ = mindspore::make_unique(); auto rc = shard_reader_->Open(dataset_file_, num_mind_record_workers_, columns_to_load_, operators_, block_reader_); - CHECK_FAIL_RETURN_UNEXPECTED(rc != MSRStatus::FAILED, "MindRecordOp init failed."); + CHECK_FAIL_RETURN_UNEXPECTED(rc != MSRStatus::FAILED, + "MindRecordOp init failed. Error message: " + ErrnoToMessage(rc)); data_schema_ = mindspore::make_unique(); diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h index 0b16391b205..aca5c86c2ce 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h @@ -32,6 +32,7 @@ #include "dataset/engine/datasetops/source/io_block.h" #include "dataset/util/queue.h" #include "dataset/util/status.h" +#include "mindrecord/include/shard_error.h" #include "mindrecord/include/shard_reader.h" #include "mindrecord/include/common/shard_utils.h" #include "dataset/util/wait_post.h" diff --git a/mindspore/ccsrc/mindrecord/common/shard_error.cc b/mindspore/ccsrc/mindrecord/common/shard_error.cc new file mode 100644 index 00000000000..cf43dcb315e --- /dev/null +++ b/mindspore/ccsrc/mindrecord/common/shard_error.cc @@ -0,0 +1,178 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/shard_error.h" + +namespace mindspore { +namespace mindrecord { +std::string ErrnoToMessage(MSRStatus status) { + switch (status) { + case FAILED: + return "operator failed"; + break; + case SUCCESS: + return "operator success"; + break; + case OPEN_FILE_FAILED: + return "open file failed"; + break; + case CLOSE_FILE_FAILED: + return "close file failed"; + break; + case WRITE_METADATA_FAILED: + return "write metadata failed"; + break; + case WRITE_RAWDATA_FAILED: + return "write rawdata failed"; + break; + case GET_SCHEMA_FAILED: + return "get schema failed"; + break; + case ILLEGAL_RAWDATA: + return "illegal raw data"; + break; + case PYTHON_TO_JSON_FAILED: + return "pybind: python object to json failed"; + break; + case DIR_CREATE_FAILED: + return "directory create failed"; + break; + case OPEN_DIR_FAILED: + return "open directory failed"; + break; + case INVALID_STATISTICS: + return "invalid statistics object"; + break; + case OPEN_DATABASE_FAILED: + return "open database failed"; + break; + case CLOSE_DATABASE_FAILED: + return "close database failed"; + break; + case DATABASE_OPERATE_FAILED: + return "database operate failed"; + break; + case BUILD_SCHEMA_FAILED: + return "build schema failed"; + break; + case DIVISOR_IS_ILLEGAL: + return "divisor is illegal"; + break; + case INVALID_FILE_PATH: + return "file path is invalid"; + break; + case SECURE_FUNC_FAILED: + return "secure function failed"; + break; + case ALLOCATE_MEM_FAILED: + return "allocate memory failed"; + break; + case ILLEGAL_FIELD_NAME: + return "illegal field name"; + break; + case ILLEGAL_FIELD_TYPE: + return "illegal field type"; + break; + case SET_METADATA_FAILED: + return "set metadata failed"; + break; + case ILLEGAL_SCHEMA_DEFINITION: + return "illegal schema definition"; + break; + case ILLEGAL_COLUMN_LIST: + return "illegal column list"; + break; + case SQL_ERROR: + return "sql error"; + break; + case ILLEGAL_SHARD_COUNT: + return "illegal shard count"; + break; + case ILLEGAL_SCHEMA_COUNT: + return "illegal schema count"; + break; + case VERSION_ERROR: + return "data version is not matched"; + break; + case ADD_SCHEMA_FAILED: + return "add schema failed"; + break; + case ILLEGAL_Header_SIZE: + return "illegal header size"; + break; + case ILLEGAL_Page_SIZE: + return "illegal page size"; + break; + case ILLEGAL_SIZE_VALUE: + return "illegal size value"; + break; + case INDEX_FIELD_ERROR: + return "add index fields failed"; + break; + case GET_CANDIDATE_CATEGORYFIELDS_FAILED: + return "get candidate category fields failed"; + break; + case GET_CATEGORY_INFO_FAILED: + return "get category information failed"; + break; + case ILLEGAL_CATEGORY_ID: + return "illegal category id"; + break; + case ILLEGAL_ROWNUMBER_OF_PAGE: + return "illegal row number of page"; + break; + case ILLEGAL_SCHEMA_ID: + return "illegal schema id"; + break; + case DESERIALIZE_SCHEMA_FAILED: + return "deserialize schema failed"; + break; + case DESERIALIZE_STATISTICS_FAILED: + return "deserialize statistics failed"; + break; + case ILLEGAL_DB_FILE: + return "illegal db file"; + break; + case OVERWRITE_DB_FILE: + return "overwrite db file"; + break; + case OVERWRITE_MINDRECORD_FILE: + return "overwrite mindrecord file"; + break; + case ILLEGAL_MINDRECORD_FILE: + return "illegal mindrecord file"; + break; + case PARSE_JSON_FAILED: + return "parse json failed"; + break; + case ILLEGAL_PARAMETERS: + return "illegal parameters"; + break; + case GET_PAGE_BY_GROUP_ID_FAILED: + return "get page by group id failed"; + break; + case GET_SYSTEM_STATE_FAILED: + return "get system state failed"; + break; + case IO_FAILED: + return "io operate failed"; + break; + default: + return "invalid error no"; + } +} +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/include/shard_error.h b/mindspore/ccsrc/mindrecord/include/shard_error.h index 026ee836e35..b85eeb71c0b 100644 --- a/mindspore/ccsrc/mindrecord/include/shard_error.h +++ b/mindspore/ccsrc/mindrecord/include/shard_error.h @@ -18,63 +18,65 @@ #define MINDRECORD_INCLUDE_SHARD_ERROR_H_ #include -#include "utils/error_code.h" +#include namespace mindspore { namespace mindrecord { -DE_ERRORNO_MINDRECORD(OPEN_FILE_FAILED, 0, "open file failed"); -DE_ERRORNO_MINDRECORD(CLOSE_FILE_FAILED, 1, "close file failed"); -DE_ERRORNO_MINDRECORD(WRITE_METADATA_FAILED, 2, "write metadata failed"); -DE_ERRORNO_MINDRECORD(WRITE_RAWDATA_FAILED, 3, "write rawdata failed"); -DE_ERRORNO_MINDRECORD(GET_SCHEMA_FAILED, 4, "get schema failed"); -DE_ERRORNO_MINDRECORD(ILLEGAL_RAWDATA, 5, "illegal raw data"); -DE_ERRORNO_MINDRECORD(PYTHON_TO_JSON_FAILED, 6, "pybind: python object to json failed"); -DE_ERRORNO_MINDRECORD(DIR_CREATE_FAILED, 7, "directory create failed"); -DE_ERRORNO_MINDRECORD(OPEN_DIR_FAILED, 8, "open directory failed"); -DE_ERRORNO_MINDRECORD(INVALID_STATISTICS, 9, "invalid statistics object"); -DE_ERRORNO_MINDRECORD(OPEN_DATABASE_FAILED, 10, "open database failed"); -DE_ERRORNO_MINDRECORD(CLOSE_DATABASE_FAILED, 11, "close database failed"); -DE_ERRORNO_MINDRECORD(DATABASE_OPERATE_FAILED, 12, "database operate failed"); -DE_ERRORNO_MINDRECORD(BUILD_SCHEMA_FAILED, 13, "build schema failed"); -DE_ERRORNO_MINDRECORD(DIVISOR_IS_ILLEGAL, 14, "divisor is illegal"); -DE_ERRORNO_MINDRECORD(INVALID_FILE_PATH, 15, "file path is invalid"); -DE_ERRORNO_MINDRECORD(SECURE_FUNC_FAILED, 16, "secure function failed"); -DE_ERRORNO_MINDRECORD(ALLOCATE_MEM_FAILED, 17, "allocate memory failed"); -DE_ERRORNO_MINDRECORD(ILLEGAL_FIELD_NAME, 18, "illegal field name"); -DE_ERRORNO_MINDRECORD(ILLEGAL_FIELD_TYPE, 19, "illegal field type"); -DE_ERRORNO_MINDRECORD(SET_METADATA_FAILED, 20, "set metadata failed"); -DE_ERRORNO_MINDRECORD(ILLEGAL_SCHEMA_DEFINITION, 21, "illegal schema definition"); -DE_ERRORNO_MINDRECORD(ILLEGAL_COLUMN_LIST, 22, "illegal column list"); -DE_ERRORNO_MINDRECORD(SQL_ERROR, 23, "sql error"); -DE_ERRORNO_MINDRECORD(ILLEGAL_SHARD_COUNT, 24, "illegal shard count"); -DE_ERRORNO_MINDRECORD(ILLEGAL_SCHEMA_COUNT, 25, "illegal schema count"); -DE_ERRORNO_MINDRECORD(VERSION_ERROR, 26, "data version is not matched"); -DE_ERRORNO_MINDRECORD(ADD_SCHEMA_FAILED, 27, "add schema failed"); -DE_ERRORNO_MINDRECORD(ILLEGAL_Header_SIZE, 28, "illegal header size"); -DE_ERRORNO_MINDRECORD(ILLEGAL_Page_SIZE, 29, "illegal page size"); -DE_ERRORNO_MINDRECORD(ILLEGAL_SIZE_VALUE, 30, "illegal size value"); -DE_ERRORNO_MINDRECORD(INDEX_FIELD_FAILED, 31, "add index fields failed"); -DE_ERRORNO_MINDRECORD(GET_CANDIDATE_CATEGORYFIELDS_FAILED, 32, "get candidate categoryFields failed"); -DE_ERRORNO_MINDRECORD(GET_CATEGORY_INFO, 33, "get category information failed"); -DE_ERRORNO_MINDRECORD(ILLEGAL_CATEGORY_ID, 34, "illegal category id"); -DE_ERRORNO_MINDRECORD(ILLEGAL_ROWNUMBER_OF_PAGE, 35, "illegal row number of page"); -DE_ERRORNO_MINDRECORD(ILLEGAL_SCHEMA_ID, 36, "illegal schema id"); -DE_ERRORNO_MINDRECORD(DESERIALIZE_SCHEMA_FAILED, 37, "deserialize schema failed"); -DE_ERRORNO_MINDRECORD(DESERIALIZE_STATISTICS_FAILED, 38, "deserialize statistics failed"); -DE_ERRORNO_MINDRECORD(ILLEGAL_DB_FILE, 39, "illegal db file."); -DE_ERRORNO_MINDRECORD(OVERWRITE_DB_FILE, 40, "overwrite db file."); -DE_ERRORNO_MINDRECORD(OVERWRITE_MINDRECORD_FILE, 41, "overwrite mindrecord file."); -DE_ERRORNO_MINDRECORD(ILLEGAL_MINDRECORD_FILE, 42, "illegal mindrecord file."); -DE_ERRORNO_MINDRECORD(PARSE_JSON_FAILED, 43, "parse json failed."); -DE_ERRORNO_MINDRECORD(ILLEGAL_PARAMETERS, 44, "illegal parameters."); -DE_ERRORNO_MINDRECORD(GET_PAGE_BY_GROUP_ID_FAILED, 46, "get page by group id failed."); -DE_ERRORNO_MINDRECORD(GET_SYSTEM_STATE_FAILED, 47, "get system state failed."); -DE_ERRORNO_MINDRECORD(IO_FAILED, 48, "io operate failed."); - enum MSRStatus { SUCCESS = 0, FAILED = 1, + OPEN_FILE_FAILED, + CLOSE_FILE_FAILED, + WRITE_METADATA_FAILED, + WRITE_RAWDATA_FAILED, + GET_SCHEMA_FAILED, + ILLEGAL_RAWDATA, + PYTHON_TO_JSON_FAILED, + DIR_CREATE_FAILED, + OPEN_DIR_FAILED, + INVALID_STATISTICS, + OPEN_DATABASE_FAILED, + CLOSE_DATABASE_FAILED, + DATABASE_OPERATE_FAILED, + BUILD_SCHEMA_FAILED, + DIVISOR_IS_ILLEGAL, + INVALID_FILE_PATH, + SECURE_FUNC_FAILED, + ALLOCATE_MEM_FAILED, + ILLEGAL_FIELD_NAME, + ILLEGAL_FIELD_TYPE, + SET_METADATA_FAILED, + ILLEGAL_SCHEMA_DEFINITION, + ILLEGAL_COLUMN_LIST, + SQL_ERROR, + ILLEGAL_SHARD_COUNT, + ILLEGAL_SCHEMA_COUNT, + VERSION_ERROR, + ADD_SCHEMA_FAILED, + ILLEGAL_Header_SIZE, + ILLEGAL_Page_SIZE, + ILLEGAL_SIZE_VALUE, + INDEX_FIELD_ERROR, + GET_CANDIDATE_CATEGORYFIELDS_FAILED, + GET_CATEGORY_INFO_FAILED, + ILLEGAL_CATEGORY_ID, + ILLEGAL_ROWNUMBER_OF_PAGE, + ILLEGAL_SCHEMA_ID, + DESERIALIZE_SCHEMA_FAILED, + DESERIALIZE_STATISTICS_FAILED, + ILLEGAL_DB_FILE, + OVERWRITE_DB_FILE, + OVERWRITE_MINDRECORD_FILE, + ILLEGAL_MINDRECORD_FILE, + PARSE_JSON_FAILED, + ILLEGAL_PARAMETERS, + GET_PAGE_BY_GROUP_ID_FAILED, + GET_SYSTEM_STATE_FAILED, + IO_FAILED }; + +// convert error no to string message +std::string ErrnoToMessage(MSRStatus status); } // namespace mindrecord } // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/io/shard_reader.cc b/mindspore/ccsrc/mindrecord/io/shard_reader.cc index 32825fd9dfa..f91d28544e8 100644 --- a/mindspore/ccsrc/mindrecord/io/shard_reader.cc +++ b/mindspore/ccsrc/mindrecord/io/shard_reader.cc @@ -676,7 +676,7 @@ MSRStatus ShardReader::Open(const std::string &file_path, int n_consumer, if (CheckColumnList(selected_columns_) == FAILED) { MS_LOG(ERROR) << "Illegal column list"; - return FAILED; + return ILLEGAL_COLUMN_LIST; } // Initialize argument diff --git a/tests/ut/cpp/dataset/mind_record_op_test.cc b/tests/ut/cpp/dataset/mind_record_op_test.cc index 3d5c80b3f43..90f41fdeb9a 100644 --- a/tests/ut/cpp/dataset/mind_record_op_test.cc +++ b/tests/ut/cpp/dataset/mind_record_op_test.cc @@ -21,6 +21,7 @@ #include "common/utils.h" #include "gtest/gtest.h" #include "mindrecord/include/shard_category.h" +#include "mindrecord/include/shard_error.h" #include "mindrecord/include/shard_sample.h" #include "mindrecord/include/shard_shuffle.h" #include "utils/log_adapter.h" @@ -479,3 +480,38 @@ TEST_F(MindDataTestMindRecordOp, TestMindRecordBlockReaderRepeat) { row_count++; } } + +TEST_F(MindDataTestMindRecordOp, TestMindRecordInvalidColumnList) { + // single MindRecord op and nothing else + // + // MindRecordOp + + MS_LOG(INFO) << "UT test TestMindRecordInvalidColumnList"; + + Status rc; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + // Test info: + // Dataset from testDataset1 has 10 rows, 2 columns. + // RowsPerBuffer buffer setting of 3 yields 4 buffers with the last buffer having single row + // only. 2 workers. + // Test a column selection instead of all columns as well. + + std::vector column_list; + std::string label_col_name("file_name_2"); + column_list.push_back(label_col_name); + label_col_name = "label"; + column_list.push_back(label_col_name); + + std::shared_ptr my_mindrecord_op; + MindRecordOp::Builder builder; + builder.SetDatasetFile(mindrecord_root_path_ + "/testMindDataSet/testImageNetData/imagenet.mindrecord0") + .SetRowsPerBuffer(3) + .SetNumMindRecordWorkers(4) + .SetColumnsToLoad(column_list); + rc = builder.Build(&my_mindrecord_op); + ASSERT_TRUE(rc.IsError()); + ASSERT_TRUE(rc.ToString().find_first_of("illegal column list") != std::string::npos); +} diff --git a/tests/ut/cpp/mindrecord/ut_shard_reader_test.cc b/tests/ut/cpp/mindrecord/ut_shard_reader_test.cc index fd63373e20e..f7ed39a0061 100644 --- a/tests/ut/cpp/mindrecord/ut_shard_reader_test.cc +++ b/tests/ut/cpp/mindrecord/ut_shard_reader_test.cc @@ -155,7 +155,7 @@ TEST_F(TestShardReader, TestShardReaderColumnNotInSchema) { auto column_list = std::vector{"file_namex"}; ShardReader dataset; MSRStatus ret = dataset.Open(file_name, 4, column_list); - ASSERT_EQ(ret, FAILED); + ASSERT_EQ(ret, ILLEGAL_COLUMN_LIST); } TEST_F(TestShardReader, TestShardVersion) {