p12346870
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix minddata issues

This commit is contained in:
YangLuo 2021-05-10 11:12:00 +08:00
parent 5482a7a855
commit 0a4da64e13
7 changed files with 56 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmake/package.cmake
View File

@ -366,12 +366,14 @@ install(
## Public header files for minddata
install(
FILES ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/constants.h
FILES ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/config.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/constants.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/execute.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/text.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/transforms.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision_lite.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision_ascend.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/execute.h
DESTINATION ${INSTALL_BASE_DIR}/include/dataset
COMPONENT mindspore
)

1
mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt
View File

@ -7,7 +7,6 @@ if(ENABLE_PYTHON)
python/bindings/dataset/core/bindings.cc
python/bindings/dataset/engine/cache/bindings.cc
python/bindings/dataset/engine/datasetops/bindings.cc
python/bindings/dataset/engine/datasetops/source/bindings.cc
python/bindings/dataset/engine/gnn/bindings.cc
python/bindings/dataset/engine/ir/consumer/bindings.cc
python/bindings/dataset/engine/ir/datasetops/bindings.cc

2
mindspore/ccsrc/minddata/dataset/api/audio.cc
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
#include "minddata/dataset/include/audio.h"
#include "minddata/dataset/include/dataset/audio.h"
#include "minddata/dataset/audio/ir/kernels/audio_ir.h"

171
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/source/bindings.cc
View File

@ -1,171 +0,0 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "minddata/dataset/api/python/pybind_register.h"
#include "pybind11/pybind11.h"
#include "pybind11/stl_bind.h"
#include "minddata/dataset/engine/datasetops/dataset_op.h"
#include "minddata/dataset/engine/datasetops/source/cifar_op.h"
#include "minddata/dataset/engine/datasetops/source/clue_op.h"
#include "minddata/dataset/engine/datasetops/source/csv_op.h"
#include "minddata/dataset/engine/datasetops/source/coco_op.h"
#include "minddata/dataset/engine/datasetops/source/image_folder_op.h"
#include "minddata/dataset/engine/datasetops/source/io_block.h"
#include "minddata/dataset/engine/datasetops/source/manifest_op.h"
#include "minddata/dataset/engine/datasetops/source/mindrecord_op.h"
#include "minddata/dataset/engine/datasetops/source/mnist_op.h"
#include "minddata/dataset/engine/datasetops/source/text_file_op.h"
#include "minddata/dataset/engine/datasetops/source/tf_reader_op.h"
#include "minddata/dataset/engine/datasetops/source/voc_op.h"
namespace mindspore {
namespace dataset {
PYBIND_REGISTER(CifarOp, 1, ([](const py::module *m) {
(void)py::class_<CifarOp, DatasetOp, std::shared_ptr<CifarOp>>(*m, "CifarOp")
.def_static("get_num_rows", [](const std::string &dir, const std::string &usage, bool isCifar10) {
int64_t count = 0;
THROW_IF_ERROR(CifarOp::CountTotalRows(dir, usage, isCifar10, &count));
return count;
});
}));
PYBIND_REGISTER(ClueOp, 1, ([](const py::module *m) {
(void)py::class_<ClueOp, DatasetOp, std::shared_ptr<ClueOp>>(*m, "ClueOp")
.def_static("get_num_rows", [](const py::list &files) {
int64_t count = 0;
std::vector<std::string> filenames;
for (auto file : files) {
file.is_none() ? (void)filenames.emplace_back("") : filenames.push_back(py::str(file));
}
THROW_IF_ERROR(ClueOp::CountAllFileRows(filenames, &count));
return count;
});
}));
PYBIND_REGISTER(CsvOp, 1, ([](const py::module *m) {
(void)py::class_<CsvOp, DatasetOp, std::shared_ptr<CsvOp>>(*m, "CsvOp")
.def_static("get_num_rows", [](const py::list &files, bool csv_header) {
int64_t count = 0;
std::vector<std::string> filenames;
for (auto file : files) {
file.is_none() ? (void)filenames.emplace_back("") : filenames.push_back(py::str(file));
}
THROW_IF_ERROR(CsvOp::CountAllFileRows(filenames, csv_header, &count));
return count;
});
}));
PYBIND_REGISTER(CocoOp, 1, ([](const py::module *m) {
(void)py::class_<CocoOp, DatasetOp, std::shared_ptr<CocoOp>>(*m, "CocoOp")
.def_static("get_class_indexing",
[](const std::string &dir, const std::string &file, const std::string &task) {
std::vector<std::pair<std::string, std::vector<int32_t>>> output_class_indexing;
THROW_IF_ERROR(CocoOp::GetClassIndexing(dir, file, task, &output_class_indexing));
return output_class_indexing;
})
.def_static("get_num_rows",
[](const std::string &dir, const std::string &file, const std::string &task) {
int64_t count = 0;
THROW_IF_ERROR(CocoOp::CountTotalRows(dir, file, task, &count));
return count;
});
}));
PYBIND_REGISTER(ImageFolderOp, 1, ([](const py::module *m) {
(void)py::class_<ImageFolderOp, DatasetOp, std::shared_ptr<ImageFolderOp>>(*m, "ImageFolderOp")
.def_static("get_num_rows",
[](const std::string &path) {
int64_t count = 0;
THROW_IF_ERROR(ImageFolderOp::CountRowsAndClasses(path, {}, &count, nullptr, {}));
return count;
})
.def_static("get_num_classes", [](const std::string &path,
const std::map<std::string, int32_t> class_index) {
int64_t num_classes = 0;
THROW_IF_ERROR(ImageFolderOp::CountRowsAndClasses(path, {}, nullptr, &num_classes, class_index));
return num_classes;
});
}));
PYBIND_REGISTER(ManifestOp, 1, ([](const py::module *m) {
(void)py::class_<ManifestOp, DatasetOp, std::shared_ptr<ManifestOp>>(*m, "ManifestOp");
}));
PYBIND_REGISTER(MindRecordOp, 1, ([](const py::module *m) {
(void)py::class_<MindRecordOp, DatasetOp, std::shared_ptr<MindRecordOp>>(*m, "MindRecordOp")
.def_static("get_num_rows", [](const std::vector<std::string> &paths, bool load_dataset,
const py::object &sampler, const int64_t num_padded) {
int64_t count = 0;
std::shared_ptr<mindrecord::ShardOperator> op;
if (py::hasattr(sampler, "create_for_minddataset")) {
auto create = sampler.attr("create_for_minddataset");
op = create().cast<std::shared_ptr<mindrecord::ShardOperator>>();
}
THROW_IF_ERROR(MindRecordOp::CountTotalRows(paths, load_dataset, op, &count, num_padded));
return count;
});
}));
PYBIND_REGISTER(MnistOp, 1, ([](const py::module *m) {
(void)py::class_<MnistOp, DatasetOp, std::shared_ptr<MnistOp>>(*m, "MnistOp")
.def_static("get_num_rows", [](const std::string &dir, const std::string &usage) {
int64_t count = 0;
THROW_IF_ERROR(MnistOp::CountTotalRows(dir, usage, &count));
return count;
});
}));
PYBIND_REGISTER(TextFileOp, 1, ([](const py::module *m) {
(void)py::class_<TextFileOp, DatasetOp, std::shared_ptr<TextFileOp>>(*m, "TextFileOp")
.def_static("get_num_rows", [](const py::list &files) {
int64_t count = 0;
std::vector<std::string> filenames;
for (auto file : files) {
!file.is_none() ? filenames.push_back(py::str(file)) : (void)filenames.emplace_back("");
}
THROW_IF_ERROR(TextFileOp::CountAllFileRows(filenames, &count));
return count;
});
}));
PYBIND_REGISTER(TFReaderOp, 1, ([](const py::module *m) {
(void)py::class_<TFReaderOp, DatasetOp, std::shared_ptr<TFReaderOp>>(*m, "TFReaderOp")
.def_static(
"get_num_rows", [](const py::list &files, int64_t numParallelWorkers, bool estimate = false) {
int64_t count = 0;
std::vector<std::string> filenames;
for (auto l : files) {
!l.is_none() ? filenames.push_back(py::str(l)) : (void)filenames.emplace_back("");
}
THROW_IF_ERROR(TFReaderOp::CountTotalRows(&count, filenames, numParallelWorkers, estimate));
return count;
});
}));
PYBIND_REGISTER(VOCOp, 1, ([](const py::module *m) {
(void)py::class_<VOCOp, DatasetOp, std::shared_ptr<VOCOp>>(*m, "VOCOp")
.def_static("get_class_indexing", [](const std::string &dir, const std::string &task_type,
const std::string &task_mode, const py::dict &dict) {
std::map<std::string, int32_t> output_class_indexing;
THROW_IF_ERROR(VOCOp::GetClassIndexing(dir, task_type, task_mode, dict, &output_class_indexing));
return output_class_indexing;
});
}));
} // namespace dataset
} // namespace mindspore

6
mindspore/ccsrc/minddata/dataset/include/audio.h → mindspore/ccsrc/minddata/dataset/include/dataset/audio.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_AUDIO_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_AUDIO_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_AUDIO_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_AUDIO_H_
namespace mindspore {
namespace dataset {
@ -24,4 +24,4 @@ namespace audio {} // namespace audio
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_AUDIO_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_AUDIO_H_

16
mindspore/dataset/vision/c_transforms.py
View File

@ -723,9 +723,9 @@ class RandomCropDecodeResize(ImageTensorOperation):
size (Union[int, sequence]): The size of the output image.
If size is an integer, a square crop of size (size, size) is returned.
If size is a sequence of length 2, it should be (height, width).
scale (tuple, optional): Range [min, max) of respective size of the
scale (list, tuple, optional): Range [min, max) of respective size of the
original size to be cropped (default=(0.08, 1.0)).
ratio (tuple, optional): Range [min, max) of aspect ratio to be
ratio (list, tuple, optional): Range [min, max) of aspect ratio to be
cropped (default=(3. / 4., 4. / 3.)).
interpolation (Inter mode, optional): Image interpolation mode (default=Inter.BILINEAR).
It can be any of [Inter.BILINEAR, Inter.NEAREST, Inter.BICUBIC].
@ -918,9 +918,9 @@ class RandomResizedCrop(ImageTensorOperation):
size (Union[int, sequence]): The size of the output image.
If size is an integer, a square crop of size (size, size) is returned.
If size is a sequence of length 2, it should be (height, width).
scale (tuple, optional): Range [min, max) of respective size of the original
scale (list, tuple, optional): Range [min, max) of respective size of the original
size to be cropped (default=(0.08, 1.0)).
ratio (tuple, optional): Range [min, max) of aspect ratio to be cropped
ratio (list, tuple, optional): Range [min, max) of aspect ratio to be cropped
(default=(3. / 4., 4. / 3.)).
interpolation (Inter mode, optional): Image interpolation mode (default=Inter.BILINEAR).
It can be any of [Inter.BILINEAR, Inter.NEAREST, Inter.BICUBIC].
@ -972,9 +972,9 @@ class RandomResizedCropWithBBox(ImageTensorOperation):
size (Union[int, sequence]): The size of the output image.
If size is an integer, a square crop of size (size, size) is returned.
If size is a sequence of length 2, it should be (height, width).
scale (tuple, optional): Range (min, max) of respective size of the original
scale (list, tuple, optional): Range (min, max) of respective size of the original
size to be cropped (default=(0.08, 1.0)).
ratio (tuple, optional): Range (min, max) of aspect ratio to be cropped
ratio (list, tuple, optional): Range (min, max) of aspect ratio to be cropped
(default=(3. / 4., 4. / 3.)).
interpolation (Inter mode, optional): Image interpolation mode (default=Inter.BILINEAR).
It can be any of [Inter.BILINEAR, Inter.NEAREST, Inter.BICUBIC].
@ -1394,9 +1394,9 @@ class SoftDvppDecodeRandomCropResizeJpeg(ImageTensorOperation):
size (Union[int, sequence]): The size of the output image.
If size is an integer, a square crop of size (size, size) is returned.
If size is a sequence of length 2, it should be (height, width).
scale (tuple, optional): Range [min, max) of respective size of the
scale (list, tuple, optional): Range [min, max) of respective size of the
original size to be cropped (default=(0.08, 1.0)).
ratio (tuple, optional): Range [min, max) of aspect ratio to be
ratio (list, tuple, optional): Range [min, max) of aspect ratio to be
cropped (default=(3. / 4., 4. / 3.)).
max_attempts (int, optional): The maximum number of attempts to propose a valid crop_area (default=10).
If exceeded, fall back to use center_crop instead.

40
tests/ut/cpp/dataset/execute_test.cc
View File

@ -19,6 +19,7 @@
#include "minddata/dataset/include/dataset/execute.h"
#include "minddata/dataset/include/dataset/transforms.h"
#include "minddata/dataset/include/dataset/vision.h"
#include "minddata/dataset/include/dataset/text.h"
#include "utils/log_adapter.h"
using namespace mindspore::dataset;
@ -206,3 +207,42 @@ TEST_F(MindDataTestExecute, TestTransformDecodeResizeCenterCrop1) {
ASSERT_EQ(image.Shape()[1], 224);
ASSERT_EQ(image.Shape()[2], 224);
}
TEST_F(MindDataTestExecute, TestUniformAugment) {
// Read images
auto image = ReadFileToTensor("data/dataset/apple.jpg");
std::vector<mindspore::MSTensor> image2;
// Transform params
std::shared_ptr<TensorTransform> decode = std::make_shared<vision::Decode>();
std::shared_ptr<TensorTransform> resize_op(new vision::Resize({16, 16}));
std::shared_ptr<TensorTransform> vertical = std::make_shared<vision::RandomVerticalFlip>();
std::shared_ptr<TensorTransform> horizontal = std::make_shared<vision::RandomHorizontalFlip>();
std::shared_ptr<TensorTransform> uniform_op(new vision::UniformAugment({resize_op, vertical, horizontal}, 3));
auto transform1 = Execute({decode});
Status rc = transform1(image, &image);
ASSERT_TRUE(rc.IsOk());
auto transform2 = Execute({uniform_op});
rc = transform2({image}, &image2);
ASSERT_TRUE(rc.IsOk());
}
TEST_F(MindDataTestExecute, TestBasicTokenizer) {
std::shared_ptr<Tensor> de_tensor;
Tensor::CreateScalar<std::string>("Welcome to China.", &de_tensor);
auto txt = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor));
std::vector<mindspore::MSTensor> txt_result;
// Transform params
std::shared_ptr<TensorTransform> tokenizer =
std::make_shared<text::BasicTokenizer>(false, false, NormalizeForm::kNone, false, true);
// BasicTokenizer has 3 outputs so we need a vector to receive its result
auto transform1 = Execute({tokenizer});
Status rc = transform1({txt}, &txt_result);
ASSERT_EQ(txt_result.size(), 3);
ASSERT_TRUE(rc.IsOk());
}