From 0a4da64e1347f421de429796c5c33a9c914624a5 Mon Sep 17 00:00:00 2001 From: YangLuo Date: Mon, 10 May 2021 11:12:00 +0800 Subject: [PATCH] Fix minddata issues --- cmake/package.cmake | 6 +- .../ccsrc/minddata/dataset/api/CMakeLists.txt | 1 - mindspore/ccsrc/minddata/dataset/api/audio.cc | 2 +- .../engine/datasetops/source/bindings.cc | 171 ------------------ .../dataset/include/{ => dataset}/audio.h | 6 +- mindspore/dataset/vision/c_transforms.py | 16 +- tests/ut/cpp/dataset/execute_test.cc | 40 ++++ 7 files changed, 56 insertions(+), 186 deletions(-) delete mode 100644 mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/source/bindings.cc rename mindspore/ccsrc/minddata/dataset/include/{ => dataset}/audio.h (78%) diff --git a/cmake/package.cmake b/cmake/package.cmake index 32451a63a92..eb9291a5de2 100644 --- a/cmake/package.cmake +++ b/cmake/package.cmake @@ -366,12 +366,14 @@ install( ## Public header files for minddata install( - FILES ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/constants.h + FILES ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/config.h + ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/constants.h + ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/execute.h + ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/text.h ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/transforms.h ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision.h ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision_lite.h ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision_ascend.h - ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/execute.h DESTINATION ${INSTALL_BASE_DIR}/include/dataset COMPONENT mindspore ) diff --git a/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt index 617381e81a3..7c467102d7c 100644 --- a/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt @@ -7,7 +7,6 @@ if(ENABLE_PYTHON) python/bindings/dataset/core/bindings.cc python/bindings/dataset/engine/cache/bindings.cc python/bindings/dataset/engine/datasetops/bindings.cc - python/bindings/dataset/engine/datasetops/source/bindings.cc python/bindings/dataset/engine/gnn/bindings.cc python/bindings/dataset/engine/ir/consumer/bindings.cc python/bindings/dataset/engine/ir/datasetops/bindings.cc diff --git a/mindspore/ccsrc/minddata/dataset/api/audio.cc b/mindspore/ccsrc/minddata/dataset/api/audio.cc index 748b178f68f..ca3544ce3b5 100644 --- a/mindspore/ccsrc/minddata/dataset/api/audio.cc +++ b/mindspore/ccsrc/minddata/dataset/api/audio.cc @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "minddata/dataset/include/audio.h" +#include "minddata/dataset/include/dataset/audio.h" #include "minddata/dataset/audio/ir/kernels/audio_ir.h" diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/source/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/source/bindings.cc deleted file mode 100644 index 7cacb66497c..00000000000 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/source/bindings.cc +++ /dev/null @@ -1,171 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/dataset/api/python/pybind_register.h" - -#include "pybind11/pybind11.h" -#include "pybind11/stl_bind.h" - -#include "minddata/dataset/engine/datasetops/dataset_op.h" - -#include "minddata/dataset/engine/datasetops/source/cifar_op.h" -#include "minddata/dataset/engine/datasetops/source/clue_op.h" -#include "minddata/dataset/engine/datasetops/source/csv_op.h" -#include "minddata/dataset/engine/datasetops/source/coco_op.h" -#include "minddata/dataset/engine/datasetops/source/image_folder_op.h" -#include "minddata/dataset/engine/datasetops/source/io_block.h" -#include "minddata/dataset/engine/datasetops/source/manifest_op.h" -#include "minddata/dataset/engine/datasetops/source/mindrecord_op.h" -#include "minddata/dataset/engine/datasetops/source/mnist_op.h" -#include "minddata/dataset/engine/datasetops/source/text_file_op.h" -#include "minddata/dataset/engine/datasetops/source/tf_reader_op.h" -#include "minddata/dataset/engine/datasetops/source/voc_op.h" - -namespace mindspore { -namespace dataset { - -PYBIND_REGISTER(CifarOp, 1, ([](const py::module *m) { - (void)py::class_>(*m, "CifarOp") - .def_static("get_num_rows", [](const std::string &dir, const std::string &usage, bool isCifar10) { - int64_t count = 0; - THROW_IF_ERROR(CifarOp::CountTotalRows(dir, usage, isCifar10, &count)); - return count; - }); - })); - -PYBIND_REGISTER(ClueOp, 1, ([](const py::module *m) { - (void)py::class_>(*m, "ClueOp") - .def_static("get_num_rows", [](const py::list &files) { - int64_t count = 0; - std::vector filenames; - for (auto file : files) { - file.is_none() ? (void)filenames.emplace_back("") : filenames.push_back(py::str(file)); - } - THROW_IF_ERROR(ClueOp::CountAllFileRows(filenames, &count)); - return count; - }); - })); - -PYBIND_REGISTER(CsvOp, 1, ([](const py::module *m) { - (void)py::class_>(*m, "CsvOp") - .def_static("get_num_rows", [](const py::list &files, bool csv_header) { - int64_t count = 0; - std::vector filenames; - for (auto file : files) { - file.is_none() ? (void)filenames.emplace_back("") : filenames.push_back(py::str(file)); - } - THROW_IF_ERROR(CsvOp::CountAllFileRows(filenames, csv_header, &count)); - return count; - }); - })); -PYBIND_REGISTER(CocoOp, 1, ([](const py::module *m) { - (void)py::class_>(*m, "CocoOp") - .def_static("get_class_indexing", - [](const std::string &dir, const std::string &file, const std::string &task) { - std::vector>> output_class_indexing; - THROW_IF_ERROR(CocoOp::GetClassIndexing(dir, file, task, &output_class_indexing)); - return output_class_indexing; - }) - .def_static("get_num_rows", - [](const std::string &dir, const std::string &file, const std::string &task) { - int64_t count = 0; - THROW_IF_ERROR(CocoOp::CountTotalRows(dir, file, task, &count)); - return count; - }); - })); - -PYBIND_REGISTER(ImageFolderOp, 1, ([](const py::module *m) { - (void)py::class_>(*m, "ImageFolderOp") - .def_static("get_num_rows", - [](const std::string &path) { - int64_t count = 0; - THROW_IF_ERROR(ImageFolderOp::CountRowsAndClasses(path, {}, &count, nullptr, {})); - return count; - }) - .def_static("get_num_classes", [](const std::string &path, - const std::map class_index) { - int64_t num_classes = 0; - THROW_IF_ERROR(ImageFolderOp::CountRowsAndClasses(path, {}, nullptr, &num_classes, class_index)); - return num_classes; - }); - })); - -PYBIND_REGISTER(ManifestOp, 1, ([](const py::module *m) { - (void)py::class_>(*m, "ManifestOp"); - })); -PYBIND_REGISTER(MindRecordOp, 1, ([](const py::module *m) { - (void)py::class_>(*m, "MindRecordOp") - .def_static("get_num_rows", [](const std::vector &paths, bool load_dataset, - const py::object &sampler, const int64_t num_padded) { - int64_t count = 0; - std::shared_ptr op; - if (py::hasattr(sampler, "create_for_minddataset")) { - auto create = sampler.attr("create_for_minddataset"); - op = create().cast>(); - } - THROW_IF_ERROR(MindRecordOp::CountTotalRows(paths, load_dataset, op, &count, num_padded)); - return count; - }); - })); - -PYBIND_REGISTER(MnistOp, 1, ([](const py::module *m) { - (void)py::class_>(*m, "MnistOp") - .def_static("get_num_rows", [](const std::string &dir, const std::string &usage) { - int64_t count = 0; - THROW_IF_ERROR(MnistOp::CountTotalRows(dir, usage, &count)); - return count; - }); - })); - -PYBIND_REGISTER(TextFileOp, 1, ([](const py::module *m) { - (void)py::class_>(*m, "TextFileOp") - .def_static("get_num_rows", [](const py::list &files) { - int64_t count = 0; - std::vector filenames; - for (auto file : files) { - !file.is_none() ? filenames.push_back(py::str(file)) : (void)filenames.emplace_back(""); - } - THROW_IF_ERROR(TextFileOp::CountAllFileRows(filenames, &count)); - return count; - }); - })); - -PYBIND_REGISTER(TFReaderOp, 1, ([](const py::module *m) { - (void)py::class_>(*m, "TFReaderOp") - .def_static( - "get_num_rows", [](const py::list &files, int64_t numParallelWorkers, bool estimate = false) { - int64_t count = 0; - std::vector filenames; - for (auto l : files) { - !l.is_none() ? filenames.push_back(py::str(l)) : (void)filenames.emplace_back(""); - } - THROW_IF_ERROR(TFReaderOp::CountTotalRows(&count, filenames, numParallelWorkers, estimate)); - return count; - }); - })); - -PYBIND_REGISTER(VOCOp, 1, ([](const py::module *m) { - (void)py::class_>(*m, "VOCOp") - .def_static("get_class_indexing", [](const std::string &dir, const std::string &task_type, - const std::string &task_mode, const py::dict &dict) { - std::map output_class_indexing; - THROW_IF_ERROR(VOCOp::GetClassIndexing(dir, task_type, task_mode, dict, &output_class_indexing)); - return output_class_indexing; - }); - })); - -} // namespace dataset -} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/include/audio.h b/mindspore/ccsrc/minddata/dataset/include/dataset/audio.h similarity index 78% rename from mindspore/ccsrc/minddata/dataset/include/audio.h rename to mindspore/ccsrc/minddata/dataset/include/dataset/audio.h index e75628fab06..ff62080ffc1 100644 --- a/mindspore/ccsrc/minddata/dataset/include/audio.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/audio.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_AUDIO_H_ -#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_AUDIO_H_ +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_AUDIO_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_AUDIO_H_ namespace mindspore { namespace dataset { @@ -24,4 +24,4 @@ namespace audio {} // namespace audio } // namespace dataset } // namespace mindspore -#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_AUDIO_H_ +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_AUDIO_H_ diff --git a/mindspore/dataset/vision/c_transforms.py b/mindspore/dataset/vision/c_transforms.py index bd04d873cb7..34b2381b283 100644 --- a/mindspore/dataset/vision/c_transforms.py +++ b/mindspore/dataset/vision/c_transforms.py @@ -723,9 +723,9 @@ class RandomCropDecodeResize(ImageTensorOperation): size (Union[int, sequence]): The size of the output image. If size is an integer, a square crop of size (size, size) is returned. If size is a sequence of length 2, it should be (height, width). - scale (tuple, optional): Range [min, max) of respective size of the + scale (list, tuple, optional): Range [min, max) of respective size of the original size to be cropped (default=(0.08, 1.0)). - ratio (tuple, optional): Range [min, max) of aspect ratio to be + ratio (list, tuple, optional): Range [min, max) of aspect ratio to be cropped (default=(3. / 4., 4. / 3.)). interpolation (Inter mode, optional): Image interpolation mode (default=Inter.BILINEAR). It can be any of [Inter.BILINEAR, Inter.NEAREST, Inter.BICUBIC]. @@ -918,9 +918,9 @@ class RandomResizedCrop(ImageTensorOperation): size (Union[int, sequence]): The size of the output image. If size is an integer, a square crop of size (size, size) is returned. If size is a sequence of length 2, it should be (height, width). - scale (tuple, optional): Range [min, max) of respective size of the original + scale (list, tuple, optional): Range [min, max) of respective size of the original size to be cropped (default=(0.08, 1.0)). - ratio (tuple, optional): Range [min, max) of aspect ratio to be cropped + ratio (list, tuple, optional): Range [min, max) of aspect ratio to be cropped (default=(3. / 4., 4. / 3.)). interpolation (Inter mode, optional): Image interpolation mode (default=Inter.BILINEAR). It can be any of [Inter.BILINEAR, Inter.NEAREST, Inter.BICUBIC]. @@ -972,9 +972,9 @@ class RandomResizedCropWithBBox(ImageTensorOperation): size (Union[int, sequence]): The size of the output image. If size is an integer, a square crop of size (size, size) is returned. If size is a sequence of length 2, it should be (height, width). - scale (tuple, optional): Range (min, max) of respective size of the original + scale (list, tuple, optional): Range (min, max) of respective size of the original size to be cropped (default=(0.08, 1.0)). - ratio (tuple, optional): Range (min, max) of aspect ratio to be cropped + ratio (list, tuple, optional): Range (min, max) of aspect ratio to be cropped (default=(3. / 4., 4. / 3.)). interpolation (Inter mode, optional): Image interpolation mode (default=Inter.BILINEAR). It can be any of [Inter.BILINEAR, Inter.NEAREST, Inter.BICUBIC]. @@ -1394,9 +1394,9 @@ class SoftDvppDecodeRandomCropResizeJpeg(ImageTensorOperation): size (Union[int, sequence]): The size of the output image. If size is an integer, a square crop of size (size, size) is returned. If size is a sequence of length 2, it should be (height, width). - scale (tuple, optional): Range [min, max) of respective size of the + scale (list, tuple, optional): Range [min, max) of respective size of the original size to be cropped (default=(0.08, 1.0)). - ratio (tuple, optional): Range [min, max) of aspect ratio to be + ratio (list, tuple, optional): Range [min, max) of aspect ratio to be cropped (default=(3. / 4., 4. / 3.)). max_attempts (int, optional): The maximum number of attempts to propose a valid crop_area (default=10). If exceeded, fall back to use center_crop instead. diff --git a/tests/ut/cpp/dataset/execute_test.cc b/tests/ut/cpp/dataset/execute_test.cc index d6f677a56aa..2007dd9562b 100644 --- a/tests/ut/cpp/dataset/execute_test.cc +++ b/tests/ut/cpp/dataset/execute_test.cc @@ -19,6 +19,7 @@ #include "minddata/dataset/include/dataset/execute.h" #include "minddata/dataset/include/dataset/transforms.h" #include "minddata/dataset/include/dataset/vision.h" +#include "minddata/dataset/include/dataset/text.h" #include "utils/log_adapter.h" using namespace mindspore::dataset; @@ -206,3 +207,42 @@ TEST_F(MindDataTestExecute, TestTransformDecodeResizeCenterCrop1) { ASSERT_EQ(image.Shape()[1], 224); ASSERT_EQ(image.Shape()[2], 224); } + +TEST_F(MindDataTestExecute, TestUniformAugment) { + // Read images + auto image = ReadFileToTensor("data/dataset/apple.jpg"); + std::vector image2; + + // Transform params + std::shared_ptr decode = std::make_shared(); + std::shared_ptr resize_op(new vision::Resize({16, 16})); + std::shared_ptr vertical = std::make_shared(); + std::shared_ptr horizontal = std::make_shared(); + + std::shared_ptr uniform_op(new vision::UniformAugment({resize_op, vertical, horizontal}, 3)); + + auto transform1 = Execute({decode}); + Status rc = transform1(image, &image); + ASSERT_TRUE(rc.IsOk()); + + auto transform2 = Execute({uniform_op}); + rc = transform2({image}, &image2); + ASSERT_TRUE(rc.IsOk()); +} + +TEST_F(MindDataTestExecute, TestBasicTokenizer) { + std::shared_ptr de_tensor; + Tensor::CreateScalar("Welcome to China.", &de_tensor); + auto txt = mindspore::MSTensor(std::make_shared(de_tensor)); + std::vector txt_result; + + // Transform params + std::shared_ptr tokenizer = + std::make_shared(false, false, NormalizeForm::kNone, false, true); + + // BasicTokenizer has 3 outputs so we need a vector to receive its result + auto transform1 = Execute({tokenizer}); + Status rc = transform1({txt}, &txt_result); + ASSERT_EQ(txt_result.size(), 3); + ASSERT_TRUE(rc.IsOk()); +}