p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!1834 sorts column names for tfreader during schema creation 

Merge pull request !1834 from Peilin/tf-reader-column-order-fix
This commit is contained in:
mindspore-ci-bot 2020-06-04 14:57:18 +08:00 committed by Gitee
parent 18ecafcf0e 9c8ea3c1af
commit cd90dfa12e
2 changed files with 11 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc
View File

@ -15,14 +15,14 @@
*/
#include "dataset/engine/datasetops/source/tf_reader_op.h"
#include <cmath>
#include <condition_variable>
#include <algorithm>
#include <future>
#include <iomanip>
#include <memory>
#include <mutex>
#include <string>
#include <utility>
#include <unordered_map>
#include <vector>
#include "proto/example.pb.h"
#include "./securec.h"
@ -905,7 +905,7 @@ Status TFReaderOp::LoadIntList(const ColDescriptor &current_col, const dataengin
return Status::OK();
}
Status TFReaderOp::CreateSchema(const std::string tf_file, const std::vector<std::string> &columns_to_load) {
Status TFReaderOp::CreateSchema(const std::string tf_file, std::vector<std::string> columns_to_load) {
std::ifstream reader;
reader.open(tf_file);
@ -926,12 +926,14 @@ Status TFReaderOp::CreateSchema(const std::string tf_file, const std::vector<std
const dataengine::Features &example_features = example.features();
const google::protobuf::Map<std::string, dataengine::Feature> &feature_map = example_features.feature();
std::vector<std::string> columns = columns_to_load;
if (columns_to_load.empty())
(void)std::transform(feature_map.begin(), feature_map.end(), std::back_inserter(columns),
if (columns_to_load.empty()) {
(void)std::transform(feature_map.begin(), feature_map.end(), std::back_inserter(columns_to_load),
[](const auto &it) -> std::string { return it.first; });
for (const auto &curr_col_name : columns) {
std::sort(columns_to_load.begin(), columns_to_load.end());
}
for (const auto &curr_col_name : columns_to_load) {
auto it = feature_map.find(curr_col_name);
if (it == feature_map.end()) {
RETURN_STATUS_UNEXPECTED("Failed to find column " + curr_col_name);

2
mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h
View File

@ -335,7 +335,7 @@ class TFReaderOp : public ParallelOp {
// Reads one row of data from a tf file and creates a schema based on that row
// @return Status - the error code returned.
Status CreateSchema(const std::string tf_file, const std::vector<std::string> &columns_to_load);
Status CreateSchema(const std::string tf_file, std::vector<std::string> columns_to_load);
// Meant to be called async. Will read files in the range [begin, end) and return the total rows
// @param filenames - a list of tf data filenames.