From b65125b0a111670c6e67651abf1a235d00924e0d Mon Sep 17 00:00:00 2001 From: nhussain Date: Mon, 27 Jul 2020 11:31:00 -0400 Subject: [PATCH] add more ifdef and include statments for pybind --- .../ccsrc/minddata/dataset/core/tensor.h | 14 +++-- .../build_sentence_piece_vocab_op.h | 1 + .../engine/datasetops/source/generator_op.h | 3 + .../ccsrc/minddata/dataset/engine/opt/pass.cc | 20 +++---- .../ccsrc/minddata/dataset/engine/opt/pass.h | 8 +-- .../engine/opt/pre/cache_transform_pass.cc | 55 ++++++++++--------- .../engine/opt/pre/cache_transform_pass.h | 6 +- .../ccsrc/minddata/dataset/include/tensor.h | 3 +- tests/ut/cpp/CMakeLists.txt | 1 + tests/ut/cpp/dataset/CMakeLists.txt | 18 ++++-- 10 files changed, 75 insertions(+), 54 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor.h b/mindspore/ccsrc/minddata/dataset/core/tensor.h index 888c542cf9..29d1785ad1 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor.h +++ b/mindspore/ccsrc/minddata/dataset/core/tensor.h @@ -649,12 +649,6 @@ class Tensor { unsigned char *data_end_ = nullptr; private: - /// Helper function to create a tensor from Numpy array of strings - /// \param[in] arr Numpy array - /// \param[out] out Created Tensor - /// \return Status - static Status CreateFromNpString(py::array arr, TensorPtr *out); - /// Copy raw data of a array based on shape and strides to the destination pointer /// \param dst [out] Pointer to the destination array where the content is to be copied /// \param[in] src Pointer to the source of strided array to be copied @@ -667,6 +661,14 @@ class Tensor { /// const of the size of the offset variable static constexpr uint8_t kOffsetSize = sizeof(offset_t); + +#ifdef ENABLE_PYTHON + /// Helper function to create a tensor from Numpy array of strings + /// \param[in] arr Numpy array + /// \param[out] out Created Tensor + /// \return Status + static Status CreateFromNpString(py::array arr, TensorPtr *out); +#endif }; template <> inline Tensor::TensorIterator Tensor::end() { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h index fc7c7d987f..c868585344 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h @@ -31,6 +31,7 @@ #include "minddata/dataset/util/status.h" #include "minddata/dataset/util/queue.h" #include "minddata/dataset/text/sentence_piece_vocab.h" +#include "pybind11/pybind11.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.h index ca001c01d6..1d7f2b97f3 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.h @@ -27,6 +27,9 @@ #include "minddata/dataset/engine/data_schema.h" #include "minddata/dataset/engine/datasetops/pipeline_op.h" #include "minddata/dataset/util/wait_post.h" +#include "pybind11/pybind11.h" + +namespace py = pybind11; namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc b/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc index 717fd92973..ca4c87f67b 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc @@ -156,6 +156,16 @@ Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { // Fallback to base class visitor by default return RunOnNode(std::static_pointer_cast(node), modified); } + +Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { + // Fallback to base class visitor by default + return RunOnNode(std::static_pointer_cast(node), modified); +} + +Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { + // Fallback to base class visitor by default + return RunOnNode(std::static_pointer_cast(node), modified); +} #endif Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { @@ -193,21 +203,11 @@ Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { return RunOnNode(std::static_pointer_cast(node), modified); } -Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { // Fallback to base class visitor by default return RunOnNode(std::static_pointer_cast(node), modified); } -Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { - // Fallback to base class visitor by default - return RunOnNode(std::static_pointer_cast(node), modified); -} - Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { // Fallback to base class visitor by default return RunOnNode(std::static_pointer_cast(node), modified); diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pass.h b/mindspore/ccsrc/minddata/dataset/engine/opt/pass.h index ed64055b21..f736c46229 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pass.h +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pass.h @@ -165,7 +165,11 @@ class NodePass : public Pass { #ifdef ENABLE_PYTHON virtual Status RunOnNode(std::shared_ptr node, bool *modified); + virtual Status RunOnNode(std::shared_ptr node, bool *modified); + virtual Status RunOnNode(std::shared_ptr node, bool *modified); + + virtual Status RunOnNode(std::shared_ptr node, bool *modified); #endif virtual Status RunOnNode(std::shared_ptr node, bool *modified); @@ -182,12 +186,8 @@ class NodePass : public Pass { virtual Status RunOnNode(std::shared_ptr node, bool *modified); - virtual Status RunOnNode(std::shared_ptr node, bool *modified); - virtual Status RunOnNode(std::shared_ptr node, bool *modified); - virtual Status RunOnNode(std::shared_ptr node, bool *modified); - virtual Status RunOnNode(std::shared_ptr node, bool *modified); virtual Status RunOnNode(std::shared_ptr node, bool *modified); diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_transform_pass.cc b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_transform_pass.cc index f305cbedb1..1667be4e96 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_transform_pass.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_transform_pass.cc @@ -24,14 +24,17 @@ #include "minddata/dataset/engine/datasetops/source/celeba_op.h" #include "minddata/dataset/engine/datasetops/source/cifar_op.h" #include "minddata/dataset/engine/datasetops/source/coco_op.h" -#include "minddata/dataset/engine/datasetops/source/generator_op.h" #include "minddata/dataset/engine/datasetops/source/image_folder_op.h" -#include "minddata/dataset/engine/datasetops/source/manifest_op.h" #include "minddata/dataset/engine/datasetops/source/mindrecord_op.h" #include "minddata/dataset/engine/datasetops/source/mnist_op.h" #include "minddata/dataset/engine/datasetops/source/random_data_op.h" #include "minddata/dataset/engine/datasetops/source/tf_reader_op.h" + +#ifdef ENABLE_PYTHON +#include "minddata/dataset/engine/datasetops/source/generator_op.h" +#include "minddata/dataset/engine/datasetops/source/manifest_op.h" #include "minddata/dataset/engine/datasetops/source/voc_op.h" +#endif namespace mindspore { namespace dataset { @@ -120,7 +123,7 @@ Status CacheTransformPass::CachePass::NonMappableCacheLeafSetup(std::shared_ptr< return Status::OK(); } -// Perform leaf node cache tranform identifications +// Perform leaf node cache transform identification Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node, bool *modified) { if (is_caching_) { // If we are a TF Reader in a caching tree, then change our config so that it becomes a basic @@ -130,56 +133,58 @@ Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node return NonMappableCacheLeafSetup(std::static_pointer_cast(node)); } -// Perform leaf node cache tranform identifications +// Perform leaf node cache transform identification Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node, bool *modified) { return NonMappableCacheLeafSetup(std::static_pointer_cast(node)); } -// Perform leaf node cache tranform identifications +// Perform leaf node cache transform identification Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node, bool *modified) { return MappableCacheLeafSetup(std::static_pointer_cast(node)); } -// Perform leaf node cache tranform identifications +// Perform leaf node cache transform identification Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node, bool *modified) { return MappableCacheLeafSetup(std::static_pointer_cast(node)); } -// Perform leaf node cache tranform identifications -Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node, bool *modified) { - return MappableCacheLeafSetup(std::static_pointer_cast(node)); -} - -// Perform leaf node cache tranform identifications -Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node, bool *modified) { - return MappableCacheLeafSetup(std::static_pointer_cast(node)); -} - -// Perform leaf node cache tranform identifications +// Perform leaf node cache transform identification Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node, bool *modified) { return MappableCacheLeafSetup(std::static_pointer_cast(node)); } -// Perform leaf node cache tranform identifications -Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node, bool *modified) { - return MappableCacheLeafSetup(std::static_pointer_cast(node)); -} - -// Perform leaf node cache tranform identifications +// Perform leaf node cache transform identification Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node, bool *modified) { return MappableCacheLeafSetup(std::static_pointer_cast(node)); } -// Perform leaf node cache tranform identifications +// Perform leaf node cache transform identification Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node, bool *modified) { return MappableCacheLeafSetup(std::static_pointer_cast(node)); } -// Perform leaf node cache tranform identifications +// Perform leaf node cache transform identification Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node, bool *modified) { return MappableCacheLeafSetup(std::static_pointer_cast(node)); } +#ifdef ENABLE_PYTHON +// Perform leaf node cache transform identification +Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node, bool *modified) { + return MappableCacheLeafSetup(std::static_pointer_cast(node)); +} + +// Perform leaf node cache transform identification +Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node, bool *modified) { + return MappableCacheLeafSetup(std::static_pointer_cast(node)); +} + +// Perform leaf node cache transform identification +Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr node, bool *modified) { + return MappableCacheLeafSetup(std::static_pointer_cast(node)); +} +#endif + // Assigns the leaf and cache operators that are involved in a cache transformation void CacheTransformPass::CachePass::AddMappableCacheOperators(std::shared_ptr leaf_op, std::shared_ptr cache_op) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_transform_pass.h b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_transform_pass.h index 942ea5fb5c..d16334f3bb 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_transform_pass.h +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_transform_pass.h @@ -82,6 +82,7 @@ class CacheTransformPass : public TreePass { /// \return Status The error code return Status RunOnNode(std::shared_ptr node, bool *modified) override; +#ifdef ENABLE_PYTHON /// \brief Perform leaf node cache tranform identifications /// \param[in] node The node being visited /// \param[inout] modified Indicator if the node was changed at all @@ -98,13 +99,14 @@ class CacheTransformPass : public TreePass { /// \param[in] node The node being visited /// \param[inout] modified Indicator if the node was changed at all /// \return Status The error code return - Status RunOnNode(std::shared_ptr node, bool *modified) override; + Status RunOnNode(std::shared_ptr node, bool *modified) override; +#endif /// \brief Perform leaf node cache tranform identifications /// \param[in] node The node being visited /// \param[inout] modified Indicator if the node was changed at all /// \return Status The error code return - Status RunOnNode(std::shared_ptr node, bool *modified) override; + Status RunOnNode(std::shared_ptr node, bool *modified) override; /// \brief Perform leaf node cache tranform identifications /// \param[in] node The node being visited diff --git a/mindspore/ccsrc/minddata/dataset/include/tensor.h b/mindspore/ccsrc/minddata/dataset/include/tensor.h index 888c542cf9..c40f8346c7 100644 --- a/mindspore/ccsrc/minddata/dataset/include/tensor.h +++ b/mindspore/ccsrc/minddata/dataset/include/tensor.h @@ -649,12 +649,13 @@ class Tensor { unsigned char *data_end_ = nullptr; private: +#ifdef ENABLE_PYTHON /// Helper function to create a tensor from Numpy array of strings /// \param[in] arr Numpy array /// \param[out] out Created Tensor /// \return Status static Status CreateFromNpString(py::array arr, TensorPtr *out); - +#endif /// Copy raw data of a array based on shape and strides to the destination pointer /// \param dst [out] Pointer to the destination array where the content is to be copied /// \param[in] src Pointer to the source of strided array to be copied diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index b4af09dee7..f746b323ea 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -39,6 +39,7 @@ if(ENABLE_MINDDATA) dataset/filter_op_test.cc dataset/voc_op_test.cc dataset/manifest_op_test.cc + dataset/sentence_piece_vocab_op_test.cc ) list(REMOVE_ITEM UT_SRCS ${PYTHON_RELATED_SRCS}) endif() diff --git a/tests/ut/cpp/dataset/CMakeLists.txt b/tests/ut/cpp/dataset/CMakeLists.txt index 9e166d110c..ca3981b25b 100644 --- a/tests/ut/cpp/dataset/CMakeLists.txt +++ b/tests/ut/cpp/dataset/CMakeLists.txt @@ -55,7 +55,7 @@ SET(DE_UT_SRCS resize_bilinear_op_test.cc resize_op_test.cc resize_with_bbox_op_test.cc - schema_test.cc + schema_test.cc shuffle_op_test.cc stand_alone_samplers_test.cc status_test.cc @@ -71,15 +71,12 @@ SET(DE_UT_SRCS subset_random_sampler_test.cc weighted_random_sampler_test.cc mnist_op_test.cc - manifest_op_test.cc - voc_op_test.cc cifar_op_test.cc celeba_op_test.cc take_op_test.cc clue_op_test.cc csv_op_test.cc text_file_op_test.cc - filter_op_test.cc concat_op_test.cc jieba_tokenizer_op_test.cc tokenizer_op_test.cc @@ -95,10 +92,19 @@ SET(DE_UT_SRCS tensor_op_fusion_pass_test.cc sliding_window_op_test.cc epoch_ctrl_op_test.cc - sentence_piece_vocab_op_test.cc - swap_red_blue_test.cc + swap_red_blue_test.cc ) +if (ENABLE_PYTHON) + set(DE_UT_SRCS + ${DE_UT_SRCS} + filter_op_test.cc + manifest_op_test.cc + voc_op_test.cc + sentence_piece_vocab_op_test.cc + ) +endif () + add_executable(de_ut_tests ${DE_UT_SRCS}) set_target_properties(de_ut_tests PROPERTIES INSTALL_RPATH "$ORIGIN/../lib:$ORIGIN/../lib64")