OSSInnovation
/
mindspore
1
0
Fork 26
Code Issues Pull Requests 7 Releases Wiki Activity

add more ifdef and include statments for pybind

This commit is contained in:
nhussain 2020-07-27 11:31:00 -04:00
parent 79225e044a
commit b65125b0a1
10 changed files with 75 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
mindspore/ccsrc/minddata/dataset/core/tensor.h
View File

@ -649,12 +649,6 @@ class Tensor {
unsigned char *data_end_ = nullptr; unsigned char *data_end_ = nullptr;
private: private:
/// Helper function to create a tensor from Numpy array of strings
/// \param[in] arr Numpy array
/// \param[out] out Created Tensor
/// \return Status
static Status CreateFromNpString(py::array arr, TensorPtr *out);
/// Copy raw data of a array based on shape and strides to the destination pointer /// Copy raw data of a array based on shape and strides to the destination pointer
/// \param dst [out] Pointer to the destination array where the content is to be copied /// \param dst [out] Pointer to the destination array where the content is to be copied
/// \param[in] src Pointer to the source of strided array to be copied /// \param[in] src Pointer to the source of strided array to be copied
@ -667,6 +661,14 @@ class Tensor {
/// const of the size of the offset variable /// const of the size of the offset variable
static constexpr uint8_t kOffsetSize = sizeof(offset_t); static constexpr uint8_t kOffsetSize = sizeof(offset_t);
#ifdef ENABLE_PYTHON
/// Helper function to create a tensor from Numpy array of strings
/// \param[in] arr Numpy array
/// \param[out] out Created Tensor
/// \return Status
static Status CreateFromNpString(py::array arr, TensorPtr *out);
#endif
}; };
template <> template <>
inline Tensor::TensorIterator<std::string_view> Tensor::end<std::string_view>() { inline Tensor::TensorIterator<std::string_view> Tensor::end<std::string_view>() {

1
mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h
View File

@ -31,6 +31,7 @@
#include "minddata/dataset/util/status.h" #include "minddata/dataset/util/status.h"
#include "minddata/dataset/util/queue.h" #include "minddata/dataset/util/queue.h"
#include "minddata/dataset/text/sentence_piece_vocab.h" #include "minddata/dataset/text/sentence_piece_vocab.h"
#include "pybind11/pybind11.h"
namespace mindspore { namespace mindspore {
namespace dataset { namespace dataset {

3
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.h
View File

@ -27,6 +27,9 @@
#include "minddata/dataset/engine/data_schema.h" #include "minddata/dataset/engine/data_schema.h"
#include "minddata/dataset/engine/datasetops/pipeline_op.h" #include "minddata/dataset/engine/datasetops/pipeline_op.h"
#include "minddata/dataset/util/wait_post.h" #include "minddata/dataset/util/wait_post.h"
#include "pybind11/pybind11.h"
namespace py = pybind11;
namespace mindspore { namespace mindspore {
namespace dataset { namespace dataset {

20
mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc
View File

@ -156,6 +156,16 @@ Status NodePass::RunOnNode(std::shared_ptr<GeneratorOp> node, bool *modified) {
// Fallback to base class visitor by default // Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified); return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
} }
Status NodePass::RunOnNode(std::shared_ptr<ManifestOp> node, bool *modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<VOCOp> node, bool *modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
#endif #endif
Status NodePass::RunOnNode(std::shared_ptr<RandomDataOp> node, bool *modified) { Status NodePass::RunOnNode(std::shared_ptr<RandomDataOp> node, bool *modified) {
@ -193,21 +203,11 @@ Status NodePass::RunOnNode(std::shared_ptr<MnistOp> node, bool *modified) {
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified); return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
} }
Status NodePass::RunOnNode(std::shared_ptr<ManifestOp> node, bool *modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<CifarOp> node, bool *modified) { Status NodePass::RunOnNode(std::shared_ptr<CifarOp> node, bool *modified) {
// Fallback to base class visitor by default // Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified); return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
} }
Status NodePass::RunOnNode(std::shared_ptr<VOCOp> node, bool *modified) {
// Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
}
Status NodePass::RunOnNode(std::shared_ptr<CelebAOp> node, bool *modified) { Status NodePass::RunOnNode(std::shared_ptr<CelebAOp> node, bool *modified) {
// Fallback to base class visitor by default // Fallback to base class visitor by default
return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified); return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);

8
mindspore/ccsrc/minddata/dataset/engine/opt/pass.h
View File

@ -165,7 +165,11 @@ class NodePass : public Pass {
#ifdef ENABLE_PYTHON #ifdef ENABLE_PYTHON
virtual Status RunOnNode(std::shared_ptr<FilterOp> node, bool *modified); virtual Status RunOnNode(std::shared_ptr<FilterOp> node, bool *modified);
virtual Status RunOnNode(std::shared_ptr<ManifestOp> node, bool *modified);
virtual Status RunOnNode(std::shared_ptr<GeneratorOp> node, bool *modified); virtual Status RunOnNode(std::shared_ptr<GeneratorOp> node, bool *modified);
virtual Status RunOnNode(std::shared_ptr<VOCOp> node, bool *modified);
#endif #endif
virtual Status RunOnNode(std::shared_ptr<RandomDataOp> node, bool *modified); virtual Status RunOnNode(std::shared_ptr<RandomDataOp> node, bool *modified);
@ -182,12 +186,8 @@ class NodePass : public Pass {
virtual Status RunOnNode(std::shared_ptr<MnistOp> node, bool *modified); virtual Status RunOnNode(std::shared_ptr<MnistOp> node, bool *modified);
virtual Status RunOnNode(std::shared_ptr<ManifestOp> node, bool *modified);
virtual Status RunOnNode(std::shared_ptr<CifarOp> node, bool *modified); virtual Status RunOnNode(std::shared_ptr<CifarOp> node, bool *modified);
virtual Status RunOnNode(std::shared_ptr<VOCOp> node, bool *modified);
virtual Status RunOnNode(std::shared_ptr<CocoOp> node, bool *modified); virtual Status RunOnNode(std::shared_ptr<CocoOp> node, bool *modified);
virtual Status RunOnNode(std::shared_ptr<CelebAOp> node, bool *modified); virtual Status RunOnNode(std::shared_ptr<CelebAOp> node, bool *modified);

55
mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_transform_pass.cc
View File

@ -24,14 +24,17 @@
#include "minddata/dataset/engine/datasetops/source/celeba_op.h" #include "minddata/dataset/engine/datasetops/source/celeba_op.h"
#include "minddata/dataset/engine/datasetops/source/cifar_op.h" #include "minddata/dataset/engine/datasetops/source/cifar_op.h"
#include "minddata/dataset/engine/datasetops/source/coco_op.h" #include "minddata/dataset/engine/datasetops/source/coco_op.h"
#include "minddata/dataset/engine/datasetops/source/generator_op.h"
#include "minddata/dataset/engine/datasetops/source/image_folder_op.h" #include "minddata/dataset/engine/datasetops/source/image_folder_op.h"
#include "minddata/dataset/engine/datasetops/source/manifest_op.h"
#include "minddata/dataset/engine/datasetops/source/mindrecord_op.h" #include "minddata/dataset/engine/datasetops/source/mindrecord_op.h"
#include "minddata/dataset/engine/datasetops/source/mnist_op.h" #include "minddata/dataset/engine/datasetops/source/mnist_op.h"
#include "minddata/dataset/engine/datasetops/source/random_data_op.h" #include "minddata/dataset/engine/datasetops/source/random_data_op.h"
#include "minddata/dataset/engine/datasetops/source/tf_reader_op.h" #include "minddata/dataset/engine/datasetops/source/tf_reader_op.h"
#ifdef ENABLE_PYTHON
#include "minddata/dataset/engine/datasetops/source/generator_op.h"
#include "minddata/dataset/engine/datasetops/source/manifest_op.h"
#include "minddata/dataset/engine/datasetops/source/voc_op.h" #include "minddata/dataset/engine/datasetops/source/voc_op.h"
#endif
namespace mindspore { namespace mindspore {
namespace dataset { namespace dataset {
@ -120,7 +123,7 @@ Status CacheTransformPass::CachePass::NonMappableCacheLeafSetup(std::shared_ptr<
return Status::OK(); return Status::OK();
} }
// Perform leaf node cache tranform identifications // Perform leaf node cache transform identification
Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<TFReaderOp> node, bool *modified) { Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<TFReaderOp> node, bool *modified) {
if (is_caching_) { if (is_caching_) {
// If we are a TF Reader in a caching tree, then change our config so that it becomes a basic // If we are a TF Reader in a caching tree, then change our config so that it becomes a basic
@ -130,56 +133,58 @@ Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<TFReaderOp> node
return NonMappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node)); return NonMappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node));
} }
// Perform leaf node cache tranform identifications // Perform leaf node cache transform identification
Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<RandomDataOp> node, bool *modified) { Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<RandomDataOp> node, bool *modified) {
return NonMappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node)); return NonMappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node));
} }
// Perform leaf node cache tranform identifications // Perform leaf node cache transform identification
Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<ImageFolderOp> node, bool *modified) { Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<ImageFolderOp> node, bool *modified) {
return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node)); return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node));
} }
// Perform leaf node cache tranform identifications // Perform leaf node cache transform identification
Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<MnistOp> node, bool *modified) { Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<MnistOp> node, bool *modified) {
return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node)); return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node));
} }
// Perform leaf node cache tranform identifications // Perform leaf node cache transform identification
Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<GeneratorOp> node, bool *modified) {
return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node));
}
// Perform leaf node cache tranform identifications
Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<ManifestOp> node, bool *modified) {
return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node));
}
// Perform leaf node cache tranform identifications
Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<CifarOp> node, bool *modified) { Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<CifarOp> node, bool *modified) {
return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node)); return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node));
} }
// Perform leaf node cache tranform identifications // Perform leaf node cache transform identification
Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<VOCOp> node, bool *modified) {
return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node));
}
// Perform leaf node cache tranform identifications
Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<CocoOp> node, bool *modified) { Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<CocoOp> node, bool *modified) {
return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node)); return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node));
} }
// Perform leaf node cache tranform identifications // Perform leaf node cache transform identification
Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<CelebAOp> node, bool *modified) { Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<CelebAOp> node, bool *modified) {
return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node)); return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node));
} }
// Perform leaf node cache tranform identifications // Perform leaf node cache transform identification
Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<MindRecordOp> node, bool *modified) { Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<MindRecordOp> node, bool *modified) {
return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node)); return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node));
} }
#ifdef ENABLE_PYTHON
// Perform leaf node cache transform identification
Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<GeneratorOp> node, bool *modified) {
return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node));
}
// Perform leaf node cache transform identification
Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<ManifestOp> node, bool *modified) {
return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node));
}
// Perform leaf node cache transform identification
Status CacheTransformPass::CachePass::RunOnNode(std::shared_ptr<VOCOp> node, bool *modified) {
return MappableCacheLeafSetup(std::static_pointer_cast<DatasetOp>(node));
}
#endif
// Assigns the leaf and cache operators that are involved in a cache transformation // Assigns the leaf and cache operators that are involved in a cache transformation
void CacheTransformPass::CachePass::AddMappableCacheOperators(std::shared_ptr<DatasetOp> leaf_op, void CacheTransformPass::CachePass::AddMappableCacheOperators(std::shared_ptr<DatasetOp> leaf_op,
std::shared_ptr<CacheOp> cache_op) { std::shared_ptr<CacheOp> cache_op) {

6
mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_transform_pass.h
View File

@ -82,6 +82,7 @@ class CacheTransformPass : public TreePass {
/// \return Status The error code return /// \return Status The error code return
Status RunOnNode(std::shared_ptr<MnistOp> node, bool *modified) override; Status RunOnNode(std::shared_ptr<MnistOp> node, bool *modified) override;
#ifdef ENABLE_PYTHON
/// \brief Perform leaf node cache tranform identifications /// \brief Perform leaf node cache tranform identifications
/// \param[in] node The node being visited /// \param[in] node The node being visited
/// \param[inout] modified Indicator if the node was changed at all /// \param[inout] modified Indicator if the node was changed at all
@ -98,13 +99,14 @@ class CacheTransformPass : public TreePass {
/// \param[in] node The node being visited /// \param[in] node The node being visited
/// \param[inout] modified Indicator if the node was changed at all /// \param[inout] modified Indicator if the node was changed at all
/// \return Status The error code return /// \return Status The error code return
Status RunOnNode(std::shared_ptr<CifarOp> node, bool *modified) override; Status RunOnNode(std::shared_ptr<VOCOp> node, bool *modified) override;
#endif
/// \brief Perform leaf node cache tranform identifications /// \brief Perform leaf node cache tranform identifications
/// \param[in] node The node being visited /// \param[in] node The node being visited
/// \param[inout] modified Indicator if the node was changed at all /// \param[inout] modified Indicator if the node was changed at all
/// \return Status The error code return /// \return Status The error code return
Status RunOnNode(std::shared_ptr<VOCOp> node, bool *modified) override; Status RunOnNode(std::shared_ptr<CifarOp> node, bool *modified) override;
/// \brief Perform leaf node cache tranform identifications /// \brief Perform leaf node cache tranform identifications
/// \param[in] node The node being visited /// \param[in] node The node being visited

3
mindspore/ccsrc/minddata/dataset/include/tensor.h
View File

@ -649,12 +649,13 @@ class Tensor {
unsigned char *data_end_ = nullptr; unsigned char *data_end_ = nullptr;
private: private:
#ifdef ENABLE_PYTHON
/// Helper function to create a tensor from Numpy array of strings /// Helper function to create a tensor from Numpy array of strings
/// \param[in] arr Numpy array /// \param[in] arr Numpy array
/// \param[out] out Created Tensor /// \param[out] out Created Tensor
/// \return Status /// \return Status
static Status CreateFromNpString(py::array arr, TensorPtr *out); static Status CreateFromNpString(py::array arr, TensorPtr *out);
#endif
/// Copy raw data of a array based on shape and strides to the destination pointer /// Copy raw data of a array based on shape and strides to the destination pointer
/// \param dst [out] Pointer to the destination array where the content is to be copied /// \param dst [out] Pointer to the destination array where the content is to be copied
/// \param[in] src Pointer to the source of strided array to be copied /// \param[in] src Pointer to the source of strided array to be copied

1
tests/ut/cpp/CMakeLists.txt
View File

@ -39,6 +39,7 @@ if(ENABLE_MINDDATA)
dataset/filter_op_test.cc dataset/filter_op_test.cc
dataset/voc_op_test.cc dataset/voc_op_test.cc
dataset/manifest_op_test.cc dataset/manifest_op_test.cc
dataset/sentence_piece_vocab_op_test.cc
) )
list(REMOVE_ITEM UT_SRCS ${PYTHON_RELATED_SRCS}) list(REMOVE_ITEM UT_SRCS ${PYTHON_RELATED_SRCS})
endif() endif()

18
tests/ut/cpp/dataset/CMakeLists.txt
View File

@ -55,7 +55,7 @@ SET(DE_UT_SRCS
resize_bilinear_op_test.cc resize_bilinear_op_test.cc
resize_op_test.cc resize_op_test.cc
resize_with_bbox_op_test.cc resize_with_bbox_op_test.cc
schema_test.cc schema_test.cc
shuffle_op_test.cc shuffle_op_test.cc
stand_alone_samplers_test.cc stand_alone_samplers_test.cc
status_test.cc status_test.cc
@ -71,15 +71,12 @@ SET(DE_UT_SRCS
subset_random_sampler_test.cc subset_random_sampler_test.cc
weighted_random_sampler_test.cc weighted_random_sampler_test.cc
mnist_op_test.cc mnist_op_test.cc
manifest_op_test.cc
voc_op_test.cc
cifar_op_test.cc cifar_op_test.cc
celeba_op_test.cc celeba_op_test.cc
take_op_test.cc take_op_test.cc
clue_op_test.cc clue_op_test.cc
csv_op_test.cc csv_op_test.cc
text_file_op_test.cc text_file_op_test.cc
filter_op_test.cc
concat_op_test.cc concat_op_test.cc
jieba_tokenizer_op_test.cc jieba_tokenizer_op_test.cc
tokenizer_op_test.cc tokenizer_op_test.cc
@ -95,10 +92,19 @@ SET(DE_UT_SRCS
tensor_op_fusion_pass_test.cc tensor_op_fusion_pass_test.cc
sliding_window_op_test.cc sliding_window_op_test.cc
epoch_ctrl_op_test.cc epoch_ctrl_op_test.cc
sentence_piece_vocab_op_test.cc swap_red_blue_test.cc
swap_red_blue_test.cc
) )
if (ENABLE_PYTHON)
set(DE_UT_SRCS
${DE_UT_SRCS}
filter_op_test.cc
manifest_op_test.cc
voc_op_test.cc
sentence_piece_vocab_op_test.cc
)
endif ()
add_executable(de_ut_tests ${DE_UT_SRCS}) add_executable(de_ut_tests ${DE_UT_SRCS})
set_target_properties(de_ut_tests PROPERTIES INSTALL_RPATH "$ORIGIN/../lib:$ORIGIN/../lib64") set_target_properties(de_ut_tests PROPERTIES INSTALL_RPATH "$ORIGIN/../lib:$ORIGIN/../lib64")