p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix csv shard

This commit is contained in:
jiangzhiwen 2020-07-22 17:14:46 +08:00
parent 74f2c89d01
commit 58f739e983
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc
View File

@ -130,6 +130,7 @@ int CsvOp::CsvParser::put_row(char c) {
}
if (total_rows_ >= end_offset_) {
cur_col_ = 0;
return 0;
}
@ -353,7 +354,9 @@ Status CsvOp::CsvParser::initCsvParser() {
{{State::END_OF_LINE, Message::MS_NORMAL},
{State::UNQUOTE,
[this](CsvParser &, char c) -> int {
this->tensor_table_->push_back(TensorRow(column_default_.size(), nullptr));
if (this->total_rows_ > this->start_offset_ && this->total_rows_ <= this->end_offset_) {
this->tensor_table_->push_back(TensorRow(column_default_.size(), nullptr));
}
this->str_buf_[0] = c;
this->pos_ = 1;
return 0;
@ -361,14 +364,18 @@ Status CsvOp::CsvParser::initCsvParser() {
{{State::END_OF_LINE, Message::MS_DELIM},
{State::DELIM,
[this](CsvParser &, char c) -> int {
this->tensor_table_->push_back(TensorRow(column_default_.size(), nullptr));
if (this->total_rows_ > this->start_offset_ && this->total_rows_ <= this->end_offset_) {
this->tensor_table_->push_back(TensorRow(column_default_.size(), nullptr));
}
this->put_record(c);
return 0;
}}},
{{State::END_OF_LINE, Message::MS_QUOTE},
{State::QUOTE,
[this](CsvParser &, char c) -> int {
this->tensor_table_->push_back(TensorRow(column_default_.size(), nullptr));
if (this->total_rows_ > this->start_offset_ && this->total_rows_ <= this->end_offset_) {
this->tensor_table_->push_back(TensorRow(column_default_.size(), nullptr));
}
return 0;
}}},
{{State::END_OF_LINE, Message::MS_END_OF_LINE}, {State::END_OF_LINE, &CsvParser::null_func}},