Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Address 3 code issues: 

1- access out of bound memory in toFlatIndex
2- CreateFromNPArray function is 50>
3- duplicate variable init
This commit is contained in:
hesham 2021-04-06 14:57:26 -04:00
parent 3d85930cfe
commit 97cf4e516e
14 changed files with 106 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
mindspore/ccsrc/minddata/dataset/core/tensor.cc
View File

@ -59,7 +59,28 @@ namespace dataset {
out << std::hex << std::setw(2) << std::setfill('0') << o << std::dec << std::setfill(' '); \
break; \
}
/// Copy memory with no max limit since memcpy_s will fail when byte_size > 2^31 - 1 (SECUREC_MEM_MAX_LEN).
/// \param dest Destination buffer.
/// \param destMax Size of the destination buffer.
/// \param src Buffer to copy from.
/// \param count Number of characters to copy
/// \return Error number. Returns 0 for succuss copying.
errno_t memcpy_ss(uchar *dest, size_t destMax, const uchar *src, size_t count) {
uint32_t step = 0;
while (count >= SECUREC_MEM_MAX_LEN) {
int ret_code = memcpy_s(dest + step * SECUREC_MEM_MAX_LEN, SECUREC_MEM_MAX_LEN, src + step * SECUREC_MEM_MAX_LEN,
SECUREC_MEM_MAX_LEN);
if (ret_code != 0) {
return ret_code;
}
count -= SECUREC_MEM_MAX_LEN;
step++;
}
if (count > 0) {
return memcpy_s(dest + step * SECUREC_MEM_MAX_LEN, count, src + step * SECUREC_MEM_MAX_LEN, count);
}
return 0;
}
Tensor::Tensor(const TensorShape &shape, const DataType &type) : shape_(shape), type_(type), data_(nullptr) {
// grab the mem pool from global context and create the allocator for char data area
std::shared_ptr<MemoryPool> global_pool = GlobalContext::Instance()->mem_pool();
@ -111,7 +132,7 @@ Status Tensor::CreateFromMemory(const TensorShape &shape, const DataType &type,
if (src != nullptr) {
// Given the shape/type of this tensor, compute the data size and copy in the input bytes.
int64_t byte_size = (*out)->SizeInBytes();
int ret_code = memcpy_s((*out)->data_, byte_size, src, byte_size);
int ret_code = memcpy_ss((*out)->data_, byte_size, src, byte_size);
CHECK_FAIL_RETURN_UNEXPECTED(ret_code == 0, "Failed to copy data into tensor.");
}
return Status::OK();
@ -163,71 +184,32 @@ Status Tensor::CreateFromNpArray(const py::array &arr, std::shared_ptr<Tensor> *
if (DataType::FromNpArray(arr) == DataType::DE_STRING) {
return CreateFromNpString(arr, out);
}
const TensorAlloc *alloc = GlobalContext::Instance()->tensor_allocator();
*out = std::allocate_shared<Tensor>(*alloc, TensorShape::CreateScalar(), DataType(DataType::DE_UNKNOWN));
std::vector<dsize_t> shape;
for (dsize_t i = 0; i < arr.ndim(); i++) {
shape.push_back(static_cast<dsize_t>(arr.shape()[i]));
}
(*out)->shape_ = TensorShape(shape);
(*out)->type_ = DataType::FromNpArray(arr);
if (!(*out)->shape_.known()) RETURN_STATUS_UNEXPECTED("Invalid shape.");
if ((*out)->type_ == DataType::DE_UNKNOWN) RETURN_STATUS_UNEXPECTED("Invalid data type.");
std::shared_ptr<MemoryPool> global_pool = GlobalContext::Instance()->mem_pool();
(*out)->data_allocator_ = std::make_unique<Allocator<unsigned char>>(global_pool);
int64_t byte_size = (*out)->SizeInBytes();
if (byte_size == 0) {
return Status::OK();
}
RETURN_IF_NOT_OK((*out)->AllocateBuffer(byte_size));
unsigned char *data = static_cast<unsigned char *>(arr.request().ptr);
if ((*out)->data_ == nullptr) {
RETURN_STATUS_UNEXPECTED("Failed to create memory for Tensor.");
}
std::vector<dsize_t> strides;
for (dsize_t i = 0; i < arr.ndim(); i++) {
strides.push_back(static_cast<dsize_t>(arr.strides()[i]));
}
// check if strides are contiguous
bool is_strided = false;
dsize_t count = (*out)->shape_.NumOfElements();
for (size_t i = 0; i < shape.size(); i++) {
count /= shape[i];
if (strides[i] != (*out)->type_.SizeInBytes() * count) {
is_strided = true;
break;
dsize_t count = arr.size();
for (dsize_t i = 0; i < arr.ndim(); i++) {
shape.push_back(static_cast<dsize_t>(arr.shape()[i]));
strides.push_back(static_cast<dsize_t>(arr.strides()[i]));
// in case of empty array num_items=0
if (count != 0) {
count /= shape[i];
if (strides[i] != arr.itemsize() * count) {
is_strided = true;
}
}
}
unsigned char *data = static_cast<unsigned char *>(arr.request().ptr);
if (is_strided) {
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape(shape), DataType::FromNpArray(arr), out));
RETURN_IF_NOT_OK(CopyStridedArray((*out)->data_, data, shape, strides, (*out)->type_.SizeInBytes()));
} else {
// fix: memcpy_s will fail when byte_size > 2^31 - 1
uint32_t step = 0;
while (byte_size >= kDeMaxDim) {
int ret_code = memcpy_s((*out)->data_ + step * kDeMaxDim, kDeMaxDim, data + step * kDeMaxDim, kDeMaxDim);
if (ret_code != 0) {
RETURN_STATUS_UNEXPECTED("Failed to copy data into Tensor.");
}
byte_size -= kDeMaxDim;
step++;
}
if (byte_size > 0) {
int ret_code = memcpy_s((*out)->data_ + step * kDeMaxDim, byte_size, data + step * kDeMaxDim, byte_size);
if (ret_code != 0) {
RETURN_STATUS_UNEXPECTED("Failed to copy data into Tensor.");
}
}
RETURN_IF_NOT_OK(Tensor::CreateFromMemory(TensorShape(shape), DataType::FromNpArray(arr), data, out));
}
return Status::OK();
}
#endif
@ -943,8 +925,8 @@ Status Tensor::Slice(std::shared_ptr<Tensor> *out, const std::vector<SliceOption
converted_slice_objects.emplace_back(slice_option_item);
}
// if a string with partial slices, pass in the rest
if (slice_options_.size() != Rank() && type() == DataType::DE_STRING) {
// partial slices, pass in the rest
if (slice_options_.size() != Rank()) {
for (int i = slice_options_.size(); i < Rank(); i++) {
mindspore::dataset::Slice slice = mindspore::dataset::Slice(0, shape_[i]);
converted_slice_objects.emplace_back(SliceOption(slice));

2
mindspore/ccsrc/minddata/dataset/core/tensor_row.h
View File

@ -141,7 +141,7 @@ class TensorRow {
if (input.at(0)->shape() != TensorShape({})) {
RETURN_STATUS_UNEXPECTED("ConvertFromTensorRow: The input tensors must be a scalar tensor.");
}
return input.at(0)->GetItemAt(o, {0});
return input.at(0)->GetItemAt(o, {});
}
/// Convert a TensorRow consisting of one 1-D tensor to a vector of size n.

5
mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc
View File

@ -230,6 +230,11 @@ std::vector<dsize_t> TensorShape::Strides() const { return std::vector<dsize_t>{
// Name: ToFlatIndex()
// Description: convert a vector style index to number, used to access memory internal use only
Status TensorShape::ToFlatIndex(const std::vector<dsize_t> &index, dsize_t *flat_index) const {
if (index.size() != raw_shape_.size()) {
std::stringstream ss;
ss << "Index size (" << index.size() << ") does not match the shape size (" << raw_shape_.size() << ").";
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, ss.str());
}
*flat_index = 0;
for (size_t k = 0; k < index.size(); k++) {
*flat_index +=

2
mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.cc
View File

@ -405,7 +405,7 @@ Status SaveToDisk::FetchItemData(std::shared_ptr<Tensor> tensor, std::string col
RETURN_IF_NOT_OK(s);
} else if (column_type == DataType::DE_STRING) {
std::string_view sv;
RETURN_IF_NOT_OK(tensor->GetItemAt(&sv, {0})); // assume scalar string tensor
RETURN_IF_NOT_OK(tensor->GetItemAt(&sv, {})); // assume scalar string tensor
std::string ss(sv);
(*row_raw_data)[column_name] = std::move(ss);
return Status::OK();

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc
View File

@ -81,7 +81,7 @@ Status GeneratorOp::InitSampler() {
// Invoke the generatorFunction to get generator object
Status GeneratorOp::CreateGeneratorObject() {
Status ret = Status::OK();
Status ret;
{
// Acquire Python GIL
py::gil_scoped_acquire gil_acquire;

2
mindspore/ccsrc/minddata/dataset/text/kernels/wordpiece_tokenizer_op.cc
View File

@ -128,7 +128,7 @@ Status WordpieceTokenizerOp::Compute(const TensorRow &input, TensorRow *output)
uint32_t basic_start = 0;
std::vector<std::string> temp_tokens;
if (with_offsets_ && input.size() == 3) {
RETURN_IF_NOT_OK(input[1]->GetItemAt<uint32_t>(&basic_start, {count, 0}));
RETURN_IF_NOT_OK(input[1]->GetItemAt<uint32_t>(&basic_start, {count}));
}
RETURN_IF_NOT_OK(GetTokens(std::string(*iter), basic_start, &temp_tokens, &offsets_start, &offsets_limit));
out_tokens.insert(out_tokens.end(), temp_tokens.begin(), temp_tokens.end());

14
tests/ut/cpp/dataset/album_op_test.cc
View File

@ -92,7 +92,7 @@ TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchema) {
uint64_t i = 0;
std::string_view label = 0;
while (tensor_map.size() != 0) {
EXPECT_TRUE(tensor_map["label"]->GetItemAt(&label, {}));
EXPECT_TRUE(tensor_map["label"]->GetItemAt(&label, {0}));
MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape"
<< tensor_map["label"] << "\n";
i++;
@ -118,7 +118,7 @@ TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchemaNoOrder) {
uint64_t i = 0;
std::string_view label;
while (tensor_map.size() != 0) {
EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {}));
EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {0}));
MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape"
<< tensor_map["label"] << "\n";
i++;
@ -146,10 +146,10 @@ TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchemaFloat) {
std::string_view label;
double priority = 0;
while (tensor_map.size() != 0) {
EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {}));
EXPECT_OK(tensor_map["_priority"]->GetItemAt<double>(&priority, {}));
EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {0}));
EXPECT_OK(tensor_map["_priority"]->GetItemAt<double>(&priority, {0}));
MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape"
<< tensor_map["label"] << "priority: " << priority << "\n";
<< tensor_map["label"] << "priority: " << priority << "\n";
i++;
di.GetNextAsMap(&tensor_map);
}
@ -176,8 +176,8 @@ TEST_F(MindDataTestAlbum, TestSequentialAlbumWithFullSchema) {
double priority = 0;
int64_t id = 0;
while (tensor_map.size() != 0) {
EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {}));
EXPECT_OK(tensor_map["_priority"]->GetItemAt<double>(&priority, {}));
EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {0}));
EXPECT_OK(tensor_map["_priority"]->GetItemAt<double>(&priority, {0}));
EXPECT_OK(tensor_map["id"]->GetItemAt<int64_t>(&id, {}));
MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape"
<< tensor_map["label"] << "priority: " << priority

22
tests/ut/cpp/dataset/c_api_dataset_clue_test.cc
View File

@ -57,7 +57,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetAFQMC) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
@ -86,7 +86,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetAFQMC) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
iter->GetNextRow(&row);
@ -109,7 +109,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetAFQMC) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
iter->GetNextRow(&row);
@ -252,7 +252,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetCMNLI) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
@ -295,7 +295,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetCSL) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
@ -438,7 +438,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetIFLYTEK) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
@ -500,7 +500,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleFilesA) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
@ -567,7 +567,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleFilesB) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
// Compare against expected result
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
@ -622,7 +622,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleGlobal) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
@ -669,7 +669,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetTNEWS) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
@ -713,7 +713,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetWSC) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();

4
tests/ut/cpp/dataset/c_api_dataset_config_test.cc
View File

@ -141,7 +141,7 @@ TEST_F(MindDataTestPipeline, TestShuffleWithSeed) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
@ -206,7 +206,7 @@ TEST_F(MindDataTestPipeline, TestCallShuffleTwice) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);

22
tests/ut/cpp/dataset/c_api_dataset_csv_test.cc
View File

@ -57,7 +57,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetBasic) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str());
}
@ -123,7 +123,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetMultiFiles) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str());
}
@ -169,7 +169,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetNumSamples) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str());
}
@ -211,7 +211,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetDistribution) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str());
}
@ -273,15 +273,15 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetType) {
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
if (colum_type[j]->type == CsvType::INT) {
int val;
de_text->GetItemAt(&val, {0});
de_text->GetItemAt(&val, {});
EXPECT_EQ(val, std::dynamic_pointer_cast<CsvRecord<int>>(expected[i][j])->value);
} else if (colum_type[j]->type == CsvType::FLOAT) {
float val;
de_text->GetItemAt(&val, {0});
de_text->GetItemAt(&val, {});
EXPECT_EQ(val, std::dynamic_pointer_cast<CsvRecord<float>>(expected[i][j])->value);
} else if (colum_type[j]->type == CsvType::STRING) {
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), std::dynamic_pointer_cast<CsvRecord<std::string>>(expected[i][j])->value.c_str());
}
@ -326,7 +326,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetHeader) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str());
}
@ -435,7 +435,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetShuffleFilesA) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str());
}
@ -492,7 +492,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetShuffleFilesB) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str());
@ -548,7 +548,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetShuffleGlobal) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str());
}

24
tests/ut/cpp/dataset/c_api_dataset_ops_test.cc
View File

@ -66,9 +66,9 @@ MSTensorVec Predicate1(MSTensorVec in) {
// Convert from boolean to TensorRow
TensorRow output;
std::shared_ptr<Tensor> out;
Tensor::CreateEmpty(mindspore::dataset::TensorShape({1}),
Tensor::CreateEmpty(mindspore::dataset::TensorShape({}),
mindspore::dataset::DataType(mindspore::dataset::DataType::Type::DE_BOOL), &out);
out->SetItemAt({0}, result);
out->SetItemAt({}, result);
output.push_back(out);
return RowToVec(output);
@ -85,9 +85,9 @@ MSTensorVec Predicate2(MSTensorVec in) {
// Convert from boolean to TensorRow
TensorRow output;
std::shared_ptr<Tensor> out;
Tensor::CreateEmpty(mindspore::dataset::TensorShape({1}),
Tensor::CreateEmpty(mindspore::dataset::TensorShape({}),
mindspore::dataset::DataType(mindspore::dataset::DataType::Type::DE_BOOL), &out);
out->SetItemAt({0}, result);
out->SetItemAt({}, result);
output.push_back(out);
return RowToVec(output);
@ -191,14 +191,14 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess2) {
// Iterate over the dataset and get each row
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
ASSERT_OK(iter->GetNextRow(&row));
}
// With 2 boundaries, 3 buckets are created
EXPECT_EQ(i, 3);
@ -630,7 +630,7 @@ TEST_F(MindDataTestPipeline, TestFilterSuccess1) {
// iterate over the dataset and get each row
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);
ASSERT_OK(iter->GetNextRow(&row));
std::vector<uint64_t> label_list;
uint64_t i = 0;
@ -641,10 +641,10 @@ TEST_F(MindDataTestPipeline, TestFilterSuccess1) {
std::shared_ptr<Tensor> de_label;
uint64_t label_value;
ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
de_label->GetItemAt(&label_value, {0});
ASSERT_OK(de_label->GetItemAt(&label_value, {0}));
label_list.push_back(label_value);
iter->GetNextRow(&row);
ASSERT_OK(iter->GetNextRow(&row));
}
// Only 1 column whose label is equal to 3
@ -676,7 +676,7 @@ TEST_F(MindDataTestPipeline, TestFilterSuccess2) {
// iterate over the dataset and get each row
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);
ASSERT_OK(iter->GetNextRow(&row));
std::vector<uint64_t> label_list;
uint64_t i = 0;
@ -687,10 +687,10 @@ TEST_F(MindDataTestPipeline, TestFilterSuccess2) {
std::shared_ptr<Tensor> de_label;
uint64_t label_value;
ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
de_label->GetItemAt(&label_value, {0});
ASSERT_OK(de_label->GetItemAt(&label_value, {0}));
label_list.push_back(label_value);
iter->GetNextRow(&row);
ASSERT_OK(iter->GetNextRow(&row));
}
// There are 2 columns whose label is more than 1

20
tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc
View File

@ -63,7 +63,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetBasic) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
@ -323,7 +323,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse1A) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
@ -384,7 +384,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse1B) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
@ -444,7 +444,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse4Shard) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
@ -507,7 +507,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1A) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
@ -570,7 +570,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1B) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
@ -632,7 +632,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles4) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
@ -690,7 +690,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
@ -751,7 +751,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1B) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
@ -812,7 +812,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) {
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
de_text->GetItemAt(&sv, {});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result

8
tests/ut/cpp/dataset/c_api_epoch_ctrl_test.cc
View File

@ -54,7 +54,7 @@ TEST_F(MindDataTestEpochCtrl, TestAutoInjectEpoch) {
std::shared_ptr<Tensor> de_label;
int64_t label_value;
ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
de_label->GetItemAt(&label_value, {0});
de_label->GetItemAt(&label_value, {});
EXPECT_TRUE(img_class[(i % sampler_size) / class_size] == label_value);
iter->GetNextRow(&row);
@ -100,7 +100,7 @@ TEST_F(MindDataTestEpochCtrl, TestEpoch) {
std::shared_ptr<Tensor> de_label;
int64_t label_value;
ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
de_label->GetItemAt(&label_value, {0});
de_label->GetItemAt(&label_value, {});
EXPECT_TRUE(label_value >= 0 && label_value <= 3);
iter->GetNextRow(&row);
@ -149,7 +149,7 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatEpoch) {
std::shared_ptr<Tensor> de_label;
int64_t label_value;
ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
de_label->GetItemAt(&label_value, {0});
de_label->GetItemAt(&label_value, {});
EXPECT_TRUE(label_value >= 0 && label_value <= 3);
iter->GetNextRow(&row);
@ -198,7 +198,7 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatRepeatEpoch) {
std::shared_ptr<Tensor> de_label;
int64_t label_value;
ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
de_label->GetItemAt(&label_value, {0});
de_label->GetItemAt(&label_value, {});
EXPECT_TRUE(label_value >= 0 && label_value <= 3);
iter->GetNextRow(&row);

2
tests/ut/cpp/dataset/pad_end_op_test.cc
View File

@ -32,7 +32,7 @@ TEST_F(MindDataTestPadEndOp, TestOp) {
// first set of testunits for numeric values
TensorShape pad_data_shape({1});
TensorShape pad_data_shape({});
// prepare input tensor
std::vector<float> orig1 = {1, 1, 1, 1};