p12346870
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix: error info is not exactly when column list invalid

This commit is contained in:
jonyguo 2020-04-08 15:52:53 +08:00
parent 606310d9c3
commit 20d1b64443
7 changed files with 271 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc
View File

@ -103,7 +103,8 @@ Status MindRecordOp::Init() {
shard_reader_ = mindspore::make_unique<ShardReader>();
auto rc = shard_reader_->Open(dataset_file_, num_mind_record_workers_, columns_to_load_, operators_, block_reader_);
CHECK_FAIL_RETURN_UNEXPECTED(rc != MSRStatus::FAILED, "MindRecordOp init failed.");
CHECK_FAIL_RETURN_UNEXPECTED(rc != MSRStatus::FAILED,
"MindRecordOp init failed. Error message: " + ErrnoToMessage(rc));
data_schema_ = mindspore::make_unique<DataSchema>();

1
mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h
View File

@ -32,6 +32,7 @@
#include "dataset/engine/datasetops/source/io_block.h"
#include "dataset/util/queue.h"
#include "dataset/util/status.h"
#include "mindrecord/include/shard_error.h"
#include "mindrecord/include/shard_reader.h"
#include "mindrecord/include/common/shard_utils.h"
#include "dataset/util/wait_post.h"

178
mindspore/ccsrc/mindrecord/common/shard_error.cc Normal file
View File

@ -0,0 +1,178 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "mindrecord/include/shard_error.h"
namespace mindspore {
namespace mindrecord {
std::string ErrnoToMessage(MSRStatus status) {
switch (status) {
case FAILED:
return "operator failed";
break;
case SUCCESS:
return "operator success";
break;
case OPEN_FILE_FAILED:
return "open file failed";
break;
case CLOSE_FILE_FAILED:
return "close file failed";
break;
case WRITE_METADATA_FAILED:
return "write metadata failed";
break;
case WRITE_RAWDATA_FAILED:
return "write rawdata failed";
break;
case GET_SCHEMA_FAILED:
return "get schema failed";
break;
case ILLEGAL_RAWDATA:
return "illegal raw data";
break;
case PYTHON_TO_JSON_FAILED:
return "pybind: python object to json failed";
break;
case DIR_CREATE_FAILED:
return "directory create failed";
break;
case OPEN_DIR_FAILED:
return "open directory failed";
break;
case INVALID_STATISTICS:
return "invalid statistics object";
break;
case OPEN_DATABASE_FAILED:
return "open database failed";
break;
case CLOSE_DATABASE_FAILED:
return "close database failed";
break;
case DATABASE_OPERATE_FAILED:
return "database operate failed";
break;
case BUILD_SCHEMA_FAILED:
return "build schema failed";
break;
case DIVISOR_IS_ILLEGAL:
return "divisor is illegal";
break;
case INVALID_FILE_PATH:
return "file path is invalid";
break;
case SECURE_FUNC_FAILED:
return "secure function failed";
break;
case ALLOCATE_MEM_FAILED:
return "allocate memory failed";
break;
case ILLEGAL_FIELD_NAME:
return "illegal field name";
break;
case ILLEGAL_FIELD_TYPE:
return "illegal field type";
break;
case SET_METADATA_FAILED:
return "set metadata failed";
break;
case ILLEGAL_SCHEMA_DEFINITION:
return "illegal schema definition";
break;
case ILLEGAL_COLUMN_LIST:
return "illegal column list";
break;
case SQL_ERROR:
return "sql error";
break;
case ILLEGAL_SHARD_COUNT:
return "illegal shard count";
break;
case ILLEGAL_SCHEMA_COUNT:
return "illegal schema count";
break;
case VERSION_ERROR:
return "data version is not matched";
break;
case ADD_SCHEMA_FAILED:
return "add schema failed";
break;
case ILLEGAL_Header_SIZE:
return "illegal header size";
break;
case ILLEGAL_Page_SIZE:
return "illegal page size";
break;
case ILLEGAL_SIZE_VALUE:
return "illegal size value";
break;
case INDEX_FIELD_ERROR:
return "add index fields failed";
break;
case GET_CANDIDATE_CATEGORYFIELDS_FAILED:
return "get candidate category fields failed";
break;
case GET_CATEGORY_INFO_FAILED:
return "get category information failed";
break;
case ILLEGAL_CATEGORY_ID:
return "illegal category id";
break;
case ILLEGAL_ROWNUMBER_OF_PAGE:
return "illegal row number of page";
break;
case ILLEGAL_SCHEMA_ID:
return "illegal schema id";
break;
case DESERIALIZE_SCHEMA_FAILED:
return "deserialize schema failed";
break;
case DESERIALIZE_STATISTICS_FAILED:
return "deserialize statistics failed";
break;
case ILLEGAL_DB_FILE:
return "illegal db file";
break;
case OVERWRITE_DB_FILE:
return "overwrite db file";
break;
case OVERWRITE_MINDRECORD_FILE:
return "overwrite mindrecord file";
break;
case ILLEGAL_MINDRECORD_FILE:
return "illegal mindrecord file";
break;
case PARSE_JSON_FAILED:
return "parse json failed";
break;
case ILLEGAL_PARAMETERS:
return "illegal parameters";
break;
case GET_PAGE_BY_GROUP_ID_FAILED:
return "get page by group id failed";
break;
case GET_SYSTEM_STATE_FAILED:
return "get system state failed";
break;
case IO_FAILED:
return "io operate failed";
break;
default:
return "invalid error no";
}
}
} // namespace mindrecord
} // namespace mindspore

102
mindspore/ccsrc/mindrecord/include/shard_error.h
View File

@ -18,63 +18,65 @@
#define MINDRECORD_INCLUDE_SHARD_ERROR_H_
#include <map>
#include "utils/error_code.h"
#include <string>
namespace mindspore {
namespace mindrecord {
DE_ERRORNO_MINDRECORD(OPEN_FILE_FAILED, 0, "open file failed");
DE_ERRORNO_MINDRECORD(CLOSE_FILE_FAILED, 1, "close file failed");
DE_ERRORNO_MINDRECORD(WRITE_METADATA_FAILED, 2, "write metadata failed");
DE_ERRORNO_MINDRECORD(WRITE_RAWDATA_FAILED, 3, "write rawdata failed");
DE_ERRORNO_MINDRECORD(GET_SCHEMA_FAILED, 4, "get schema failed");
DE_ERRORNO_MINDRECORD(ILLEGAL_RAWDATA, 5, "illegal raw data");
DE_ERRORNO_MINDRECORD(PYTHON_TO_JSON_FAILED, 6, "pybind: python object to json failed");
DE_ERRORNO_MINDRECORD(DIR_CREATE_FAILED, 7, "directory create failed");
DE_ERRORNO_MINDRECORD(OPEN_DIR_FAILED, 8, "open directory failed");
DE_ERRORNO_MINDRECORD(INVALID_STATISTICS, 9, "invalid statistics object");
DE_ERRORNO_MINDRECORD(OPEN_DATABASE_FAILED, 10, "open database failed");
DE_ERRORNO_MINDRECORD(CLOSE_DATABASE_FAILED, 11, "close database failed");
DE_ERRORNO_MINDRECORD(DATABASE_OPERATE_FAILED, 12, "database operate failed");
DE_ERRORNO_MINDRECORD(BUILD_SCHEMA_FAILED, 13, "build schema failed");
DE_ERRORNO_MINDRECORD(DIVISOR_IS_ILLEGAL, 14, "divisor is illegal");
DE_ERRORNO_MINDRECORD(INVALID_FILE_PATH, 15, "file path is invalid");
DE_ERRORNO_MINDRECORD(SECURE_FUNC_FAILED, 16, "secure function failed");
DE_ERRORNO_MINDRECORD(ALLOCATE_MEM_FAILED, 17, "allocate memory failed");
DE_ERRORNO_MINDRECORD(ILLEGAL_FIELD_NAME, 18, "illegal field name");
DE_ERRORNO_MINDRECORD(ILLEGAL_FIELD_TYPE, 19, "illegal field type");
DE_ERRORNO_MINDRECORD(SET_METADATA_FAILED, 20, "set metadata failed");
DE_ERRORNO_MINDRECORD(ILLEGAL_SCHEMA_DEFINITION, 21, "illegal schema definition");
DE_ERRORNO_MINDRECORD(ILLEGAL_COLUMN_LIST, 22, "illegal column list");
DE_ERRORNO_MINDRECORD(SQL_ERROR, 23, "sql error");
DE_ERRORNO_MINDRECORD(ILLEGAL_SHARD_COUNT, 24, "illegal shard count");
DE_ERRORNO_MINDRECORD(ILLEGAL_SCHEMA_COUNT, 25, "illegal schema count");
DE_ERRORNO_MINDRECORD(VERSION_ERROR, 26, "data version is not matched");
DE_ERRORNO_MINDRECORD(ADD_SCHEMA_FAILED, 27, "add schema failed");
DE_ERRORNO_MINDRECORD(ILLEGAL_Header_SIZE, 28, "illegal header size");
DE_ERRORNO_MINDRECORD(ILLEGAL_Page_SIZE, 29, "illegal page size");
DE_ERRORNO_MINDRECORD(ILLEGAL_SIZE_VALUE, 30, "illegal size value");
DE_ERRORNO_MINDRECORD(INDEX_FIELD_FAILED, 31, "add index fields failed");
DE_ERRORNO_MINDRECORD(GET_CANDIDATE_CATEGORYFIELDS_FAILED, 32, "get candidate categoryFields failed");
DE_ERRORNO_MINDRECORD(GET_CATEGORY_INFO, 33, "get category information failed");
DE_ERRORNO_MINDRECORD(ILLEGAL_CATEGORY_ID, 34, "illegal category id");
DE_ERRORNO_MINDRECORD(ILLEGAL_ROWNUMBER_OF_PAGE, 35, "illegal row number of page");
DE_ERRORNO_MINDRECORD(ILLEGAL_SCHEMA_ID, 36, "illegal schema id");
DE_ERRORNO_MINDRECORD(DESERIALIZE_SCHEMA_FAILED, 37, "deserialize schema failed");
DE_ERRORNO_MINDRECORD(DESERIALIZE_STATISTICS_FAILED, 38, "deserialize statistics failed");
DE_ERRORNO_MINDRECORD(ILLEGAL_DB_FILE, 39, "illegal db file.");
DE_ERRORNO_MINDRECORD(OVERWRITE_DB_FILE, 40, "overwrite db file.");
DE_ERRORNO_MINDRECORD(OVERWRITE_MINDRECORD_FILE, 41, "overwrite mindrecord file.");
DE_ERRORNO_MINDRECORD(ILLEGAL_MINDRECORD_FILE, 42, "illegal mindrecord file.");
DE_ERRORNO_MINDRECORD(PARSE_JSON_FAILED, 43, "parse json failed.");
DE_ERRORNO_MINDRECORD(ILLEGAL_PARAMETERS, 44, "illegal parameters.");
DE_ERRORNO_MINDRECORD(GET_PAGE_BY_GROUP_ID_FAILED, 46, "get page by group id failed.");
DE_ERRORNO_MINDRECORD(GET_SYSTEM_STATE_FAILED, 47, "get system state failed.");
DE_ERRORNO_MINDRECORD(IO_FAILED, 48, "io operate failed.");
enum MSRStatus {
SUCCESS = 0,
FAILED = 1,
OPEN_FILE_FAILED,
CLOSE_FILE_FAILED,
WRITE_METADATA_FAILED,
WRITE_RAWDATA_FAILED,
GET_SCHEMA_FAILED,
ILLEGAL_RAWDATA,
PYTHON_TO_JSON_FAILED,
DIR_CREATE_FAILED,
OPEN_DIR_FAILED,
INVALID_STATISTICS,
OPEN_DATABASE_FAILED,
CLOSE_DATABASE_FAILED,
DATABASE_OPERATE_FAILED,
BUILD_SCHEMA_FAILED,
DIVISOR_IS_ILLEGAL,
INVALID_FILE_PATH,
SECURE_FUNC_FAILED,
ALLOCATE_MEM_FAILED,
ILLEGAL_FIELD_NAME,
ILLEGAL_FIELD_TYPE,
SET_METADATA_FAILED,
ILLEGAL_SCHEMA_DEFINITION,
ILLEGAL_COLUMN_LIST,
SQL_ERROR,
ILLEGAL_SHARD_COUNT,
ILLEGAL_SCHEMA_COUNT,
VERSION_ERROR,
ADD_SCHEMA_FAILED,
ILLEGAL_Header_SIZE,
ILLEGAL_Page_SIZE,
ILLEGAL_SIZE_VALUE,
INDEX_FIELD_ERROR,
GET_CANDIDATE_CATEGORYFIELDS_FAILED,
GET_CATEGORY_INFO_FAILED,
ILLEGAL_CATEGORY_ID,
ILLEGAL_ROWNUMBER_OF_PAGE,
ILLEGAL_SCHEMA_ID,
DESERIALIZE_SCHEMA_FAILED,
DESERIALIZE_STATISTICS_FAILED,
ILLEGAL_DB_FILE,
OVERWRITE_DB_FILE,
OVERWRITE_MINDRECORD_FILE,
ILLEGAL_MINDRECORD_FILE,
PARSE_JSON_FAILED,
ILLEGAL_PARAMETERS,
GET_PAGE_BY_GROUP_ID_FAILED,
GET_SYSTEM_STATE_FAILED,
IO_FAILED
};
// convert error no to string message
std::string ErrnoToMessage(MSRStatus status);
} // namespace mindrecord
} // namespace mindspore

2
mindspore/ccsrc/mindrecord/io/shard_reader.cc
View File

@ -676,7 +676,7 @@ MSRStatus ShardReader::Open(const std::string &file_path, int n_consumer,
if (CheckColumnList(selected_columns_) == FAILED) {
MS_LOG(ERROR) << "Illegal column list";
return FAILED;
return ILLEGAL_COLUMN_LIST;
}
// Initialize argument

36
tests/ut/cpp/dataset/mind_record_op_test.cc
View File

@ -21,6 +21,7 @@
#include "common/utils.h"
#include "gtest/gtest.h"
#include "mindrecord/include/shard_category.h"
#include "mindrecord/include/shard_error.h"
#include "mindrecord/include/shard_sample.h"
#include "mindrecord/include/shard_shuffle.h"
#include "utils/log_adapter.h"
@ -479,3 +480,38 @@ TEST_F(MindDataTestMindRecordOp, TestMindRecordBlockReaderRepeat) {
row_count++;
}
}
TEST_F(MindDataTestMindRecordOp, TestMindRecordInvalidColumnList) {
// single MindRecord op and nothing else
//
// MindRecordOp
MS_LOG(INFO) << "UT test TestMindRecordInvalidColumnList";
Status rc;
// Start with an empty execution tree
auto my_tree = std::make_shared<ExecutionTree>();
// Test info:
// Dataset from testDataset1 has 10 rows, 2 columns.
// RowsPerBuffer buffer setting of 3 yields 4 buffers with the last buffer having single row
// only. 2 workers.
// Test a column selection instead of all columns as well.
std::vector<std::string> column_list;
std::string label_col_name("file_name_2");
column_list.push_back(label_col_name);
label_col_name = "label";
column_list.push_back(label_col_name);
std::shared_ptr<MindRecordOp> my_mindrecord_op;
MindRecordOp::Builder builder;
builder.SetDatasetFile(mindrecord_root_path_ + "/testMindDataSet/testImageNetData/imagenet.mindrecord0")
.SetRowsPerBuffer(3)
.SetNumMindRecordWorkers(4)
.SetColumnsToLoad(column_list);
rc = builder.Build(&my_mindrecord_op);
ASSERT_TRUE(rc.IsError());
ASSERT_TRUE(rc.ToString().find_first_of("illegal column list") != std::string::npos);
}

2
tests/ut/cpp/mindrecord/ut_shard_reader_test.cc
View File

@ -155,7 +155,7 @@ TEST_F(TestShardReader, TestShardReaderColumnNotInSchema) {
auto column_list = std::vector<std::string>{"file_namex"};
ShardReader dataset;
MSRStatus ret = dataset.Open(file_name, 4, column_list);
ASSERT_EQ(ret, FAILED);
ASSERT_EQ(ret, ILLEGAL_COLUMN_LIST);
}
TEST_F(TestShardReader, TestShardVersion) {