diff --git a/build.sh b/build.sh index 3ed4cbc461..95c3e6be43 100755 --- a/build.sh +++ b/build.sh @@ -26,7 +26,7 @@ usage() echo "bash build.sh [-d] [-r] [-v] [-c on|off] [-t on|off] [-g on|off] [-h] [-b ge] [-m infer|train] \\" echo " [-a on|off] [-Q on|off] [-p on|off] [-i] [-L] [-R] [-D on|off] [-j[n]] [-e gpu|d|cpu] \\" echo " [-P on|off] [-z [on|off]] [-M on|off] [-V 9.2|10.1] [-I arm64|arm32|x86_64] [-K] \\" - echo " [-B on|off] [-w on|off] [-E] [-l on|off] [-n]" + echo " [-B on|off] [-w on|off] [-E] [-l on|off] [-n full|lite|off]" echo "" echo "Options:" echo " -d Debug mode" @@ -50,10 +50,10 @@ usage() echo " -Q Enable dump memory, default off" echo " -D Enable dumping of function graph ir, default on" echo " -z Compile dataset & mindrecord, default on" - echo " -n Compile minddata lite" + echo " -n Compile minddata with mindspore-lite, available: off, lite, full, default is lite" echo " -M Enable MPI and NCCL for GPU training, gpu default on" echo " -V Specify the minimum required cuda version, default CUDA 10.1" - echo " -I Compile lite" + echo " -I Compile mindspore-lite" echo " -K Compile with AKG, default on" echo " -s Enable serving module, default off" echo " -w Enable acl module, default off" @@ -93,7 +93,7 @@ checkopts() ENABLE_DUMPE2E="off" ENABLE_DUMP_IR="on" COMPILE_MINDDATA="on" - COMPILE_MINDDATA_LITE="off" + COMPILE_MINDDATA_LITE="lite" ENABLE_MPI="off" CUDA_VERSION="10.1" COMPILE_LITE="off" @@ -109,7 +109,7 @@ checkopts() ENABLE_GPU="off" # Process the options - while getopts 'drvj:c:t:hsb:a:g:p:ie:m:l:I:LRP:Q:D:zM:V:K:swB:EnT:' opt + while getopts 'drvj:c:t:hsb:a:g:p:ie:m:l:I:LRP:Q:D:zM:V:K:swB:En:T:' opt do OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]') case "${opt}" in @@ -117,7 +117,13 @@ checkopts() DEBUG_MODE="on" ;; n) - COMPILE_MINDDATA_LITE="on" + if [[ "X$OPTARG" == "Xoff" || "X$OPTARG" == "Xlite" || "X$OPTARG" == "Xfull" ]]; then + COMPILE_MINDDATA_LITE="$OPTARG" + else + echo "Invalid value ${OPTARG} for option -n" + usage + exit 1 + fi ;; r) DEBUG_MODE="off" @@ -548,16 +554,16 @@ build_minddata_lite_deps() { echo "start build minddata lite project" if [[ "${LITE_PLATFORM}" == "arm64" ]]; then - CMAKE_MINDDATA_ARGS="-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_NATIVE_API_LEVEL=19 \ - -DANDROID_NDK=${ANDROID_NDK} -DANDROID_ABI=arm64-v8a -DANDROID_TOOLCHAIN_NAME=aarch64-linux-android-clang \ - -DANDROID_STL=c++_shared -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" - elif [[ "${LITE_PLATFORM}" == "arm32" ]]; then - CMAKE_MINDDATA_ARGS="-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_NATIVE_API_LEVEL=19 \ - -DANDROID_NDK=${ANDROID_NDK} -DANDROID_ABI=armeabi-v7a -DANDROID_TOOLCHAIN_NAME=clang \ - -DANDROID_STL=c++_shared -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" - else - CMAKE_MINDDATA_ARGS="-DCMAKE_BUILD_TYPE=${BUILD_TYPE}" - fi + CMAKE_MINDDATA_ARGS="-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_NATIVE_API_LEVEL=19 \ + -DANDROID_NDK=${ANDROID_NDK} -DANDROID_ABI=arm64-v8a -DANDROID_TOOLCHAIN_NAME=aarch64-linux-android-clang \ + -DANDROID_STL=c++_shared -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" + elif [[ "${LITE_PLATFORM}" == "arm32" ]]; then + CMAKE_MINDDATA_ARGS="-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_NATIVE_API_LEVEL=19 \ + -DANDROID_NDK=${ANDROID_NDK} -DANDROID_ABI=armeabi-v7a -DANDROID_TOOLCHAIN_NAME=clang \ + -DANDROID_STL=c++_shared -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" + else + CMAKE_MINDDATA_ARGS="-DCMAKE_BUILD_TYPE=${BUILD_TYPE}" + fi build_opencv build_eigen build_jpeg_turbo @@ -577,7 +583,7 @@ build_lite() build_flatbuffer build_gtest - if [ "${COMPILE_MINDDATA_LITE}" == "on" ]; then + if [ "${COMPILE_MINDDATA_LITE}" == "lite" ] || [ "${COMPILE_MINDDATA_LITE}" == "full" ]; then build_minddata_lite_deps fi diff --git a/cmake/package_lite.cmake b/cmake/package_lite.cmake index 4fb60c021c..6f220fa8a9 100644 --- a/cmake/package_lite.cmake +++ b/cmake/package_lite.cmake @@ -13,24 +13,30 @@ set(TURBO_DIR_RUN_X86 ${MAIN_DIR}-${RUN_X86_COMPONENT_NAME}/third_party/libjpeg- set(OPENCV_DIR_RUN_X86 ${MAIN_DIR}-${RUN_X86_COMPONENT_NAME}/third_party/opencv) set(PROTOBF_DIR_RUN_X86 ${MAIN_DIR}-${RUN_X86_COMPONENT_NAME}/third_party/protobuf) set(FLATBF_DIR_RUN_X86 ${MAIN_DIR}-${RUN_X86_COMPONENT_NAME}/third_party/flatbuffers) -if (BUILD_MINDDATA) - install(DIRECTORY ${TOP_DIR}/mindspore/ccsrc/minddata/dataset/include/ DESTINATION ${INC_DIR} COMPONENT ${COMPONENT_NAME} FILES_MATCHING PATTERN "*.h") - install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${LIB_DIR} COMPONENT ${COMPONENT_NAME}) - install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME}) - install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME}) +if (BUILD_MINDDATA STREQUAL "lite") + install(DIRECTORY ${TOP_DIR}/mindspore/ccsrc/minddata/dataset/include/ DESTINATION ${INC_DIR} COMPONENT ${COMPONENT_NAME} FILES_MATCHING PATTERN "*.h") if (PLATFORM_ARM64) - install(FILES ${TOP_DIR}/third_party/opencv/build/lib/arm64-v8a/libopencv_core.so DESTINATION ${OPENCV_DIR}/lib/arm64-v8a COMPONENT ${COMPONENT_NAME}) - install(FILES ${TOP_DIR}/third_party/opencv/build/lib/arm64-v8a/libopencv_imgcodecs.so DESTINATION ${OPENCV_DIR}/lib/arm64-v8a COMPONENT ${COMPONENT_NAME}) - install(FILES ${TOP_DIR}/third_party/opencv/build/lib/arm64-v8a/libopencv_imgproc.so DESTINATION ${OPENCV_DIR}/lib/arm64-v8a COMPONENT ${COMPONENT_NAME}) + install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${LIB_DIR} COMPONENT ${COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/opencv/build/lib/arm64-v8a/libopencv_core.so DESTINATION ${OPENCV_DIR}/lib COMPONENT ${COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/opencv/build/lib/arm64-v8a/libopencv_imgcodecs.so DESTINATION ${OPENCV_DIR}/lib COMPONENT ${COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/opencv/build/lib/arm64-v8a/libopencv_imgproc.so DESTINATION ${OPENCV_DIR}/lib COMPONENT ${COMPONENT_NAME}) elseif (PLATFORM_ARM32) - install(FILES ${TOP_DIR}/third_party/opencv/build/lib/armeabi-v7a/libopencv_core.so DESTINATION ${OPENCV_DIR}/lib/armeabi-v7a COMPONENT ${COMPONENT_NAME}) - install(FILES ${TOP_DIR}/third_party/opencv/build/lib/armeabi-v7a/libopencv_imgcodecs.so DESTINATION ${OPENCV_DIR}/lib/armeabi-v7a COMPONENT ${COMPONENT_NAME}) - install(FILES ${TOP_DIR}/third_party/opencv/build/lib/armeabi-v7a/libopencv_imgproc.so DESTINATION ${OPENCV_DIR}/lib/armeabi-v7a COMPONENT ${COMPONENT_NAME}) + install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${LIB_DIR} COMPONENT ${COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/opencv/build/lib/armeabi-v7a/libopencv_core.so DESTINATION ${OPENCV_DIR}/lib COMPONENT ${COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/opencv/build/lib/armeabi-v7a/libopencv_imgcodecs.so DESTINATION ${OPENCV_DIR}/lib COMPONENT ${COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/opencv/build/lib/armeabi-v7a/libopencv_imgproc.so DESTINATION ${OPENCV_DIR}/lib COMPONENT ${COMPONENT_NAME}) else () - install(FILES ${TOP_DIR}/third_party/opencv/build/lib/libopencv_core.so DESTINATION ${OPENCV_DIR}/lib COMPONENT ${COMPONENT_NAME}) - install(FILES ${TOP_DIR}/third_party/opencv/build/lib/libopencv_imgcodecs.so DESTINATION ${OPENCV_DIR}/lib COMPONENT ${COMPONENT_NAME}) - install(FILES ${TOP_DIR}/third_party/opencv/build/lib/libopencv_imgproc.so DESTINATION ${OPENCV_DIR}/lib COMPONENT ${COMPONENT_NAME}) + install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${LIB_DIR_RUN_X86} COMPONENT ${RUN_X86_COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so.62.3.0 DESTINATION ${TURBO_DIR_RUN_X86}/lib RENAME libjpeg.so.62 COMPONENT ${RUN_X86_COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so.0.2.0 DESTINATION ${TURBO_DIR_RUN_X86}/lib RENAME libturbojpeg.so.0 COMPONENT ${RUN_X86_COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/opencv/build/lib/libopencv_core.so.4.2.0 DESTINATION ${OPENCV_DIR_RUN_X86}/lib RENAME libopencv_core.so.4.2 COMPONENT ${RUN_X86_COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/opencv/build/lib/libopencv_imgcodecs.so.4.2.0 DESTINATION ${OPENCV_DIR_RUN_X86}/lib RENAME libopencv_imgcodecs.so.4.2 COMPONENT ${RUN_X86_COMPONENT_NAME}) + install(FILES ${TOP_DIR}/third_party/opencv/build/lib/libopencv_imgproc.so.4.2.0 DESTINATION ${OPENCV_DIR_RUN_X86}/lib RENAME libopencv_imgproc.so.4.2 COMPONENT ${RUN_X86_COMPONENT_NAME}) endif () endif () diff --git a/mindspore/ccsrc/minddata/dataset/api/datasets.cc b/mindspore/ccsrc/minddata/dataset/api/datasets.cc index ee14dc3cd6..d43375a692 100644 --- a/mindspore/ccsrc/minddata/dataset/api/datasets.cc +++ b/mindspore/ccsrc/minddata/dataset/api/datasets.cc @@ -32,7 +32,9 @@ #include "minddata/dataset/engine/datasetops/source/mnist_op.h" #include "minddata/dataset/engine/datasetops/source/random_data_op.h" #include "minddata/dataset/engine/datasetops/source/text_file_op.h" +#ifndef ENABLE_ANDROID #include "minddata/dataset/engine/datasetops/source/voc_op.h" +#endif // Dataset operator headers (in alphabetical order) #include "minddata/dataset/engine/datasetops/batch_op.h" #include "minddata/dataset/engine/datasetops/build_vocab_op.h" @@ -200,6 +202,7 @@ std::shared_ptr ImageFolder(const std::string &dataset_dir, return ds->ValidateParams() ? ds : nullptr; } +#ifndef ENABLE_ANDROID // Function to create a ManifestDataset. std::shared_ptr Manifest(const std::string &dataset_file, const std::string &usage, const std::shared_ptr &sampler, @@ -209,6 +212,7 @@ std::shared_ptr Manifest(const std::string &dataset_file, const // Call derived class validation method. return ds->ValidateParams() ? ds : nullptr; } +#endif // Function to create a MnistDataset. std::shared_ptr Mnist(const std::string &dataset_dir, const std::shared_ptr &sampler) { @@ -236,6 +240,7 @@ std::shared_ptr TextFile(const std::vector &datase return ds->ValidateParams() ? ds : nullptr; } +#ifndef ENABLE_ANDROID // Function to create a VOCDataset. std::shared_ptr VOC(const std::string &dataset_dir, const std::string &task, const std::string &mode, const std::map &class_indexing, bool decode, @@ -245,6 +250,7 @@ std::shared_ptr VOC(const std::string &dataset_dir, const std::strin // Call derived class validation method. return ds->ValidateParams() ? ds : nullptr; } +#endif // Function to create a ZipDataset. std::shared_ptr Zip(const std::vector> &datasets) { @@ -274,6 +280,7 @@ std::shared_ptr Dataset::Batch(int32_t batch_size, bool drop_remai return ds; } +#ifndef ENABLE_ANDROID // Function to create a Vocab from dataset std::shared_ptr Dataset::BuildVocab(const std::vector &columns, const std::pair &freq_range, int64_t top_k, @@ -304,6 +311,7 @@ std::shared_ptr Dataset::BuildVocab(const std::vector &colum return vocab; } +#endif // Function to create a Concat dataset std::shared_ptr Dataset::Concat(const std::vector> &datasets) { @@ -1266,6 +1274,7 @@ std::vector> ImageFolderDataset::Build() { return node_ops; } +#ifndef ENABLE_ANDROID ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage, const std::shared_ptr &sampler, const std::map &class_indexing, bool decode) @@ -1310,6 +1319,7 @@ std::vector> ManifestDataset::Build() { node_ops.push_back(manifest_op); return node_ops; } +#endif MnistDataset::MnistDataset(std::string dataset_dir, std::shared_ptr sampler) : dataset_dir_(dataset_dir), sampler_(sampler) {} @@ -1465,6 +1475,7 @@ std::vector> TextFileDataset::Build() { return node_ops; } +#ifndef ENABLE_ANDROID // Constructor for VOCDataset VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &mode, const std::map &class_indexing, bool decode, @@ -1542,6 +1553,7 @@ std::vector> VOCDataset::Build() { node_ops.push_back(voc_op); return node_ops; } +#endif // DERIVED DATASET CLASSES LEAF-NODE DATASETS // (In alphabetical order) @@ -1578,6 +1590,7 @@ bool BatchDataset::ValidateParams() { return true; } +#ifndef ENABLE_ANDROID BuildVocabDataset::BuildVocabDataset(std::shared_ptr vocab, const std::vector &columns, const std::pair &freq_range, int64_t top_k, const std::vector &special_tokens, bool special_first) @@ -1616,6 +1629,7 @@ bool BuildVocabDataset::ValidateParams() { } return true; } +#endif // Function to build ConcatOp ConcatDataset::ConcatDataset(const std::vector> &datasets) : datasets_(datasets) { diff --git a/mindspore/ccsrc/minddata/dataset/api/de_tensor.cc b/mindspore/ccsrc/minddata/dataset/api/de_tensor.cc index 0b03c221c3..b8c6a5c163 100644 --- a/mindspore/ccsrc/minddata/dataset/api/de_tensor.cc +++ b/mindspore/ccsrc/minddata/dataset/api/de_tensor.cc @@ -42,7 +42,7 @@ MSTensor *DETensor::CreateFromMemory(TypeId data_type, const std::vector &s std::transform(shape.begin(), shape.end(), std::back_inserter(t_shape), [](int s) -> dataset::dsize_t { return static_cast(s); }); - (void)dataset::Tensor::CreateFromMemory(dataset::TensorShape(t_shape), MSTypeToDEType(data_type), + (void)dataset::Tensor::CreateFromMemory(dataset::TensorShape(t_shape), dataset::MSTypeToDEType(data_type), static_cast(data), &t); return new DETensor(std::move(t)); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc index dfb131a43d..cd3fa6426c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc @@ -370,6 +370,7 @@ Status ManifestOp::CountDatasetInfo() { return Status::OK(); } +#ifdef ENABLE_PYTHON Status ManifestOp::CountTotalRows(const std::string &file, const py::dict &dict, const std::string &usage, int64_t *count, int64_t *numClasses) { // the logic of counting the number of samples is copied from ParseManifestFile() @@ -412,6 +413,7 @@ Status ManifestOp::GetClassIndexing(const std::string &file, const py::dict &dic return Status::OK(); } +#endif // Visitor accept method for NodePass Status ManifestOp::Accept(NodePass *p, bool *modified) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.h index 5a1a1d726b..dbe7a28e94 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.h @@ -165,12 +165,14 @@ class ManifestOp : public ParallelOp, public RandomAccessOp { // @param show_all void Print(std::ostream &out, bool show_all) const override; +#ifdef ENABLE_PYTHON static Status CountTotalRows(const std::string &file, const py::dict &dict, const std::string &usage, int64_t *count, int64_t *numClasses); // Get str-to-int mapping from label name to index static Status GetClassIndexing(const std::string &file, const py::dict &dict, const std::string &usage, std::map *output_class_indexing); +#endif /// \brief Base-class override for NodePass visitor acceptor /// \param[in] p Pointer to the NodePass to be accepted diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc index 49f25aa695..f89c824369 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc @@ -419,6 +419,7 @@ Status VOCOp::ReadAnnotationToTensor(const std::string &path, TensorRow *row) { return Status::OK(); } +#ifdef ENABLE_PYTHON Status VOCOp::CountTotalRows(const std::string &dir, const std::string &task_type, const std::string &task_mode, const py::dict &dict, int64_t *count) { if (task_type == "Detection") { @@ -467,6 +468,8 @@ Status VOCOp::GetClassIndexing(const std::string &dir, const std::string &task_t return Status::OK(); } +#endif + // Visitor accept method for NodePass Status VOCOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.h index ef5578f467..6ed42c7990 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.h @@ -188,6 +188,7 @@ class VOCOp : public ParallelOp, public RandomAccessOp { // @param show_all void Print(std::ostream &out, bool show_all) const override; +#ifdef ENABLE_PYTHON // @param const std::string &dir - VOC dir path // @param const std::string &task_type - task type of reading voc job // @param const std::string &task_mode - task mode of reading voc job @@ -204,6 +205,7 @@ class VOCOp : public ParallelOp, public RandomAccessOp { // @param std::map *output_class_indexing - output class index of VOCDataset static Status GetClassIndexing(const std::string &dir, const std::string &task_type, const std::string &task_mode, const py::dict &dict, std::map *output_class_indexing); +#endif /// \brief Base-class override for NodePass visitor acceptor /// \param[in] p Pointer to the NodePass to be accepted diff --git a/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc b/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc index 79c954595e..7012d85dd3 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc @@ -21,8 +21,10 @@ #include "minddata/dataset/util/task_manager.h" #include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/engine/opt/pre/removal_pass.h" +#ifndef ENABLE_ANDROID #include "minddata/dataset/engine/opt/pre/cache_transform_pass.h" #include "minddata/dataset/engine/opt/post/repeat_pass.h" +#endif #include "minddata/dataset/engine/opt/pre/epoch_injection_pass.h" #include "mindspore/ccsrc/minddata/dataset/engine/opt/optional/tensor_op_fusion_pass.h" #include "minddata/dataset/engine/perf/profiling.h" @@ -227,7 +229,9 @@ Status ExecutionTree::PrepareTreePreAction() { MS_LOG(INFO) << "Running pre pass loops."; pre_actions.push_back(std::make_unique()); pre_actions.push_back(std::make_unique()); +#ifndef ENABLE_ANDROID pre_actions.push_back(std::make_unique()); +#endif // Apply pre action passes for (auto &pass : pre_actions) { RETURN_IF_NOT_OK(pass->Run(this, &modified)); @@ -244,7 +248,9 @@ Status ExecutionTree::PrepareTreePostAction() { std::vector> post_actions; // Construct pre actions MS_LOG(INFO) << "Running post pass loops."; +#ifndef ENABLE_ANDROID post_actions.push_back(std::make_unique()); +#endif // Apply post action passes for (auto &pass : post_actions) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc b/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc index 43fff5925e..847be38d5d 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc @@ -17,10 +17,12 @@ #include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/engine/datasetops/batch_op.h" #include "minddata/dataset/engine/datasetops/build_vocab_op.h" +#ifndef ENABLE_ANDROID #include "minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h" #include "minddata/dataset/engine/datasetops/cache_op.h" #include "minddata/dataset/engine/datasetops/cache_merge_op.h" #include "minddata/dataset/engine/datasetops/cache_lookup_op.h" +#endif #include "minddata/dataset/engine/datasetops/dataset_op.h" #include "minddata/dataset/engine/datasetops/device_queue_op.h" #include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h" @@ -205,10 +207,12 @@ Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { return RunOnNode(std::static_pointer_cast(node), modified); } +#ifndef ENABLE_ANDROID Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { // Fallback to base class visitor by default return RunOnNode(std::static_pointer_cast(node), modified); } +#endif Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { // Fallback to base class visitor by default @@ -235,6 +239,7 @@ Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { return RunOnNode(std::static_pointer_cast(node), modified); } +#ifndef ENABLE_ANDROID Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { // Fallback to base class visitor by default return RunOnNode(std::static_pointer_cast(node), modified); @@ -244,6 +249,7 @@ Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) // Fallback to base class visitor by default return RunOnNode(std::static_pointer_cast(node), modified); } +#endif Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) { // Fallback to base class visitor by default @@ -255,6 +261,7 @@ Status NodePass::PreRunOnNode(std::shared_ptr node, bool *modified) { return PreRunOnNode(std::static_pointer_cast(node), modified); } +#ifndef ENABLE_ANDROID Status NodePass::PreRunOnNode(std::shared_ptr node, bool *modified) { // Fallback to base class visitor by default return PreRunOnNode(std::static_pointer_cast(node), modified); @@ -264,6 +271,7 @@ Status NodePass::PreRunOnNode(std::shared_ptr node, bool *modified // Fallback to base class visitor by default return PreRunOnNode(std::static_pointer_cast(node), modified); } +#endif Status NodePass::PreRunOnNode(std::shared_ptr node, bool *modified) { // Fallback to base class visitor by default @@ -275,9 +283,12 @@ Status NodePass::PreRunOnNode(std::shared_ptr node, bool *modified return PreRunOnNode(std::static_pointer_cast(node), modified); } +#ifndef ENABLE_ANDROID Status NodePass::PreRunOnNode(std::shared_ptr node, bool *modified) { // Fallback to base class visitor by default return PreRunOnNode(std::static_pointer_cast(node), modified); } +#endif + } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pass.h b/mindspore/ccsrc/minddata/dataset/engine/opt/pass.h index b4c676589e..626dd7687e 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pass.h +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pass.h @@ -63,7 +63,9 @@ class DeviceQueueOp; class ImageFolderOp; +#ifndef ENABLE_ANDROID class CacheOp; +#endif class MnistOp; @@ -77,15 +79,19 @@ class CocoOp; class CelebAOp; +#ifndef ENABLE_ANDROID class CacheMergeOp; class CacheLookupOp; +#endif class EpochCtrlOp; class BuildVocabOp; +#ifndef ENABLE_ANDROID class BuildSentencePieceVocabOp; +#endif // The base class Pass is the basic unit of tree transformation. // The actual implementation of the passes will be derived from here. @@ -188,7 +194,9 @@ class NodePass : public Pass { virtual Status RunOnNode(std::shared_ptr node, bool *modified); +#ifndef ENABLE_ANDROID virtual Status RunOnNode(std::shared_ptr node, bool *modified); +#endif virtual Status RunOnNode(std::shared_ptr node, bool *modified); @@ -202,23 +210,31 @@ class NodePass : public Pass { virtual Status RunOnNode(std::shared_ptr node, bool *modified); +#ifndef ENABLE_ANDROID virtual Status RunOnNode(std::shared_ptr node, bool *modified); virtual Status RunOnNode(std::shared_ptr node, bool *modified); +#endif virtual Status RunOnNode(std::shared_ptr node, bool *modified); +#ifndef ENABLE_ANDROID virtual Status PreRunOnNode(std::shared_ptr node, bool *modified); +#endif virtual Status PreRunOnNode(std::shared_ptr node, bool *modified); +#ifndef ENABLE_ANDROID virtual Status PreRunOnNode(std::shared_ptr node, bool *modified); +#endif virtual Status PreRunOnNode(std::shared_ptr node, bool *modified); virtual Status PreRunOnNode(std::shared_ptr node, bool *modified); +#ifndef ENABLE_ANDROID virtual Status PreRunOnNode(std::shared_ptr node, bool *modified); +#endif private: // Helper function to perform DFS visit diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/epoch_injection_pass.cc b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/epoch_injection_pass.cc index 2cd1f74089..d202b9a174 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/epoch_injection_pass.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/epoch_injection_pass.cc @@ -33,6 +33,7 @@ Status EpochInjectionPass::InjectionFinder::PreRunOnNode(std::shared_ptr node, bool *modified) { @@ -46,6 +47,7 @@ Status EpochInjectionPass::InjectionFinder::PreRunOnNode(std::shared_ptr node, bool *modified) { // Assumption: There is only one DeviceQueueOp in a pipeline. This assumption is not validated here. diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/epoch_injection_pass.h b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/epoch_injection_pass.h index aa035674b3..a80c0d6650 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/epoch_injection_pass.h +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/epoch_injection_pass.h @@ -48,6 +48,7 @@ class EpochInjectionPass : public TreePass { /// \return Status The error code return Status PreRunOnNode(std::shared_ptr node, bool *modified) override; +#ifndef ENABLE_ANDROID /// \brief Performs finder work for BuildSentencePieceVocabOp that has special rules about epoch control injection. /// \param[in] node The node being visited /// \param[inout] modified Indicator if the node was changed at all @@ -60,6 +61,7 @@ class EpochInjectionPass : public TreePass { /// \param[inout] modified Indicator if the node was changed at all /// \return Status The error code return Status PreRunOnNode(std::shared_ptr node, bool *modified) override; +#endif /// \brief Register the DeviceQueueOp for further action. /// \param[in] node The node being visited diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/removal_pass.cc b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/removal_pass.cc index 14c00b4a63..3e2150e22f 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/removal_pass.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/removal_pass.cc @@ -25,6 +25,7 @@ namespace dataset { RemovalPass::RemovalNodes::RemovalNodes() : is_caching_(false) {} +#ifndef ENABLE_ANDROID // Identifies the subtree below this node as a cached descendant tree. Status RemovalPass::RemovalNodes::PreRunOnNode(std::shared_ptr node, bool *modified) { *modified = false; @@ -40,6 +41,7 @@ Status RemovalPass::RemovalNodes::RunOnNode(std::shared_ptr node, bool is_caching_ = false; return Status::OK(); } +#endif // Perform ShuffleOp removal check. Status RemovalPass::RemovalNodes::RunOnNode(std::shared_ptr node, bool *modified) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/removal_pass.h b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/removal_pass.h index f1e8b79495..2698263b3a 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/removal_pass.h +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/removal_pass.h @@ -42,6 +42,7 @@ class RemovalPass : public TreePass { /// \brief Destructor ~RemovalNodes() = default; +#ifndef ENABLE_ANDROID /// \brief Identifies the subtree below this node as a cached descendant tree. /// \param[in] node The node being visited /// \param[inout] modified Indicator if the node was changed at all @@ -53,6 +54,7 @@ class RemovalPass : public TreePass { /// \param[inout] modified Indicator if the node was changed at all /// \return Status The error code return Status RunOnNode(std::shared_ptr node, bool *modified) override; +#endif /// \brief Perform ShuffleOp removal check /// \param[in] node The node being visited diff --git a/mindspore/ccsrc/minddata/dataset/include/datasets.h b/mindspore/ccsrc/minddata/dataset/include/datasets.h index fb9ea146d8..f8ba200064 100644 --- a/mindspore/ccsrc/minddata/dataset/include/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/include/datasets.h @@ -30,7 +30,9 @@ #include "minddata/dataset/include/iterator.h" #include "minddata/dataset/include/samplers.h" #include "minddata/dataset/include/type_id.h" +#ifndef ENABLE_ANDROID #include "minddata/dataset/text/vocab.h" +#endif namespace mindspore { namespace dataset { @@ -40,7 +42,9 @@ class DatasetOp; class DataSchema; class Tensor; class TensorShape; +#ifndef ENABLE_ANDROID class Vocab; +#endif namespace api { @@ -57,14 +61,20 @@ class CocoDataset; class CSVDataset; class CsvBase; class ImageFolderDataset; +#ifndef ENABLE_ANDROID class ManifestDataset; +#endif class MnistDataset; class RandomDataset; class TextFileDataset; +#ifndef ENABLE_ANDROID class VOCDataset; +#endif // Dataset Op classes (in alphabetical order) class BatchDataset; +#ifndef ENABLE_ANDROID class BuildVocabDataset; +#endif class ConcatDataset; class MapDataset; class ProjectDataset; @@ -210,6 +220,7 @@ std::shared_ptr ImageFolder(const std::string &dataset_dir, const std::set &extensions = {}, const std::map &class_indexing = {}); +#ifndef ENABLE_ANDROID /// \brief Function to create a ManifestDataset /// \notes The generated dataset has two columns ['image', 'label'] /// \param[in] dataset_file The dataset file to be read @@ -224,6 +235,7 @@ std::shared_ptr Manifest(const std::string &dataset_file, const const std::shared_ptr &sampler = RandomSampler(), const std::map &class_indexing = {}, bool decode = false); +#endif /// \brief Function to create a MnistDataset /// \notes The generated dataset has two columns ['image', 'label'] @@ -276,6 +288,7 @@ std::shared_ptr TextFile(const std::vector &datase ShuffleMode shuffle = ShuffleMode::kGlobal, int32_t num_shards = 1, int32_t shard_id = 0); +#ifndef ENABLE_ANDROID /// \brief Function to create a VOCDataset /// \notes The generated dataset has multi-columns : /// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['label', dtype=uint32], @@ -293,6 +306,7 @@ std::shared_ptr VOC(const std::string &dataset_dir, const std::strin const std::string &mode = "train", const std::map &class_indexing = {}, bool decode = false, const std::shared_ptr &sampler = RandomSampler()); +#endif /// \brief Function to create a ZipDataset /// \notes Applies zip to the dataset @@ -356,6 +370,7 @@ class Dataset : public std::enable_shared_from_this { /// \return Shared pointer to the current BatchDataset std::shared_ptr Batch(int32_t batch_size, bool drop_remainder = false); +#ifndef ENABLE_ANDROID /// \brief Function to create a Vocab from source dataset /// \notes Build a vocab from a dataset. This would collect all the unique words in a dataset and return a vocab /// which contains top_k most frequent words (if top_k is specified) @@ -373,6 +388,7 @@ class Dataset : public std::enable_shared_from_this { const std::pair &freq_range = {0, kDeMaxFreq}, int64_t top_k = kDeMaxTopk, const std::vector &special_tokens = {}, bool special_first = true); +#endif /// \brief Function to create a ConcatDataset /// \notes Concat the datasets in the input @@ -753,6 +769,7 @@ class ImageFolderDataset : public Dataset { std::set exts_; }; +#ifndef ENABLE_ANDROID class ManifestDataset : public Dataset { public: /// \brief Constructor @@ -777,6 +794,7 @@ class ManifestDataset : public Dataset { std::map class_index_; std::shared_ptr sampler_; }; +#endif class MnistDataset : public Dataset { public: @@ -873,6 +891,7 @@ class TextFileDataset : public Dataset { ShuffleMode shuffle_; }; +#ifndef ENABLE_ANDROID class VOCDataset : public Dataset { public: /// \brief Constructor @@ -904,6 +923,7 @@ class VOCDataset : public Dataset { bool decode_; std::shared_ptr sampler_; }; +#endif // DERIVED DATASET CLASSES FOR DATASET OPS // (In alphabetical order) @@ -933,6 +953,7 @@ class BatchDataset : public Dataset { std::map>> pad_map_; }; +#ifndef ENABLE_ANDROID class BuildVocabDataset : public Dataset { public: /// \brief Constructor @@ -959,6 +980,7 @@ class BuildVocabDataset : public Dataset { std::vector special_tokens_; bool special_first_; }; +#endif class ConcatDataset : public Dataset { public: diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc index beb0391901..2cee55a9e8 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc @@ -187,7 +187,7 @@ void JpegSetSource(j_decompress_ptr cinfo, const void *data, int64_t datasize) { (*cinfo->mem->alloc_small)(reinterpret_cast(cinfo), JPOOL_PERMANENT, sizeof(struct jpeg_source_mgr))); cinfo->src->init_source = JpegInitSource; cinfo->src->fill_input_buffer = JpegFillInputBuffer; -#if defined(_WIN32) || defined(_WIN64) +#if defined(_WIN32) || defined(_WIN64) || defined(ENABLE_ARM32) cinfo->src->skip_input_data = reinterpret_cast(JpegSkipInputData); #else cinfo->src->skip_input_data = JpegSkipInputData; diff --git a/mindspore/ccsrc/minddata/dataset/text/vocab.cc b/mindspore/ccsrc/minddata/dataset/text/vocab.cc index e975f5d918..8bd4449d6c 100644 --- a/mindspore/ccsrc/minddata/dataset/text/vocab.cc +++ b/mindspore/ccsrc/minddata/dataset/text/vocab.cc @@ -31,6 +31,7 @@ WordIdType Vocab::Lookup(const WordType &word) const { return itr == word2id_.end() ? kNoTokenExists : itr->second; } +#ifdef ENABLE_PYTHON Status Vocab::BuildFromPyList(const py::list &words, const py::list &special_tokens, bool prepend_special, std::shared_ptr *vocab) { // check of duplication on both words and special_tokens will be performed in python @@ -61,6 +62,7 @@ Status Vocab::BuildFromPyDict(const py::dict &words, std::shared_ptr *voc *vocab = std::make_shared(std::move(word2id)); return Status::OK(); } +#endif void Vocab::append_word(const std::string &word) { if (word2id_.find(word) == word2id_.end()) { diff --git a/mindspore/ccsrc/minddata/dataset/text/vocab.h b/mindspore/ccsrc/minddata/dataset/text/vocab.h index f3d46a4cb5..cd59c24132 100644 --- a/mindspore/ccsrc/minddata/dataset/text/vocab.h +++ b/mindspore/ccsrc/minddata/dataset/text/vocab.h @@ -23,18 +23,23 @@ #include #include "minddata/dataset/util/status.h" +#ifdef ENABLE_PYTHON #include "pybind11/pybind11.h" #include "pybind11/stl.h" +#endif namespace mindspore { namespace dataset { +#ifdef ENABLE_PYTHON namespace py = pybind11; +#endif using WordIdType = int32_t; using WordType = std::string; class Vocab { public: +#ifdef ENABLE_PYTHON // Build a vocab from a python dictionary key is each word ,id needs to start from 2, no duplicate and continuous // @param const py::dict &words - a dictionary containing word, word id pair. // @param std::shared_ptr *vocab - return value, vocab object @@ -56,6 +61,7 @@ class Vocab { // @return error code static Status BuildFromFile(const std::string &path, const std::string &delimiter, int32_t vocab_size, const py::list &special_tokens, bool prepend_special, std::shared_ptr *vocab); +#endif /// \brief Build a vocab from a c++ map. id needs to start from 2, no duplicate and continuous /// \param[in] words An unordered_map containing word, word id pair. diff --git a/mindspore/ccsrc/minddata/dataset/util/services.cc b/mindspore/ccsrc/minddata/dataset/util/services.cc index 882dc2628f..c75478cc58 100644 --- a/mindspore/ccsrc/minddata/dataset/util/services.cc +++ b/mindspore/ccsrc/minddata/dataset/util/services.cc @@ -16,7 +16,7 @@ #include "minddata/dataset/util/services.h" #include -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) #include #else #include @@ -31,7 +31,7 @@ namespace dataset { std::unique_ptr Services::instance_ = nullptr; std::once_flag Services::init_instance_flag_; -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) std::string Services::GetUserName() { char user[LOGIN_NAME_MAX]; (void)getlogin_r(user, sizeof(user)); diff --git a/mindspore/ccsrc/minddata/dataset/util/services.h b/mindspore/ccsrc/minddata/dataset/util/services.h index a077f670cb..884e6e9cc4 100644 --- a/mindspore/ccsrc/minddata/dataset/util/services.h +++ b/mindspore/ccsrc/minddata/dataset/util/services.h @@ -63,7 +63,7 @@ class Services { std::shared_ptr GetServiceMemPool() { return pool_; } -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) static std::string GetUserName(); static std::string GetHostName(); diff --git a/mindspore/ccsrc/minddata/dataset/util/sig_handler.cc b/mindspore/ccsrc/minddata/dataset/util/sig_handler.cc index eed3b4ee4d..8aa5951b14 100644 --- a/mindspore/ccsrc/minddata/dataset/util/sig_handler.cc +++ b/mindspore/ccsrc/minddata/dataset/util/sig_handler.cc @@ -16,7 +16,7 @@ #include "minddata/dataset/util/sig_handler.h" #include #include -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) #include #endif #include @@ -25,7 +25,7 @@ namespace mindspore { namespace dataset { // Register the custom signal handlers -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) void RegisterHandlers() { struct sigaction new_int_action; diff --git a/mindspore/ccsrc/minddata/dataset/util/sig_handler.h b/mindspore/ccsrc/minddata/dataset/util/sig_handler.h index 03f644a45c..3f6d86da92 100644 --- a/mindspore/ccsrc/minddata/dataset/util/sig_handler.h +++ b/mindspore/ccsrc/minddata/dataset/util/sig_handler.h @@ -22,7 +22,7 @@ namespace mindspore { namespace dataset { // Register the custom signal handlers -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) extern void RegisterHandlers(); // A signal handler for SIGINT. Drives interrupt to watchdog diff --git a/mindspore/ccsrc/minddata/dataset/util/task.cc b/mindspore/ccsrc/minddata/dataset/util/task.cc index fb71e93379..9cbaa7074b 100644 --- a/mindspore/ccsrc/minddata/dataset/util/task.cc +++ b/mindspore/ccsrc/minddata/dataset/util/task.cc @@ -17,6 +17,9 @@ #include "utils/ms_utils.h" #include "minddata/dataset/util/task_manager.h" #include "utils/log_adapter.h" +#if defined(__ANDROID__) || defined(ANDROID) +#include "minddata/dataset/util/services.h" +#endif namespace mindspore { namespace dataset { @@ -29,6 +32,10 @@ void Task::operator()() { id_ = this_thread::get_id(); std::stringstream ss; ss << id_; +#if defined(__ANDROID__) || defined(ANDROID) + // The thread id in Linux may be duplicate + ss << Services::GetUniqueID(); +#endif MS_LOG(DEBUG) << my_name_ << " Thread ID " << ss.str() << " Started."; try { // Previously there is a timing hole where the thread is spawn but hit error immediately before we can set diff --git a/mindspore/ccsrc/minddata/dataset/util/task_manager.cc b/mindspore/ccsrc/minddata/dataset/util/task_manager.cc index 38019155a2..5e426b0c7e 100644 --- a/mindspore/ccsrc/minddata/dataset/util/task_manager.cc +++ b/mindspore/ccsrc/minddata/dataset/util/task_manager.cc @@ -124,6 +124,7 @@ TaskManager::TaskManager() try : global_interrupt_(0), master_->is_master_ = true; #if !defined(_WIN32) && !defined(_WIN64) gMyTask = master_.get(); +#if !defined(__ANDROID__) && !defined(ANDROID) // Initialize the semaphore for the watchdog errno_t rc = sem_init(&sem_, 0, 0); if (rc == -1) { @@ -131,6 +132,7 @@ TaskManager::TaskManager() try : global_interrupt_(0), std::terminate(); } #endif +#endif } catch (const std::exception &e) { MS_LOG(ERROR) << "MindData initialization failed: " << e.what() << "."; std::terminate(); @@ -145,14 +147,14 @@ TaskManager::~TaskManager() { watchdog_grp_ = nullptr; watchdog_ = nullptr; } -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) (void)sem_destroy(&sem_); #endif } Status TaskManager::DoServiceStart() { MS_LOG(INFO) << "Starting Task Manager."; -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) // Create a watchdog for control-c std::shared_ptr mp = Services::GetInstance().GetServiceMemPool(); // A dummy group just for the watchdog. We aren't really using it. But most code assumes a thread must @@ -181,7 +183,7 @@ Status TaskManager::DoServiceStop() { Status TaskManager::WatchDog() { TaskManager::FindMe()->Post(); -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) errno_t err = sem_wait(&sem_); if (err == -1) { RETURN_STATUS_UNEXPECTED("Errno = " + std::to_string(errno)); diff --git a/mindspore/ccsrc/minddata/dataset/util/task_manager.h b/mindspore/ccsrc/minddata/dataset/util/task_manager.h index 2a3e3f07c8..0724d27cc0 100644 --- a/mindspore/ccsrc/minddata/dataset/util/task_manager.h +++ b/mindspore/ccsrc/minddata/dataset/util/task_manager.h @@ -16,7 +16,7 @@ #ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_TASK_MANAGER_H_ #define MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_TASK_MANAGER_H_ -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) #include #include // for sig_atomic_t #endif @@ -92,7 +92,7 @@ class TaskManager : public Service { static void InterruptMaster(const Status &rc = Status::OK()); static void WakeUpWatchDog() { -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) TaskManager &tm = TaskManager::GetInstance(); (void)sem_post(&tm.sem_); #endif @@ -113,7 +113,7 @@ class TaskManager : public Service { std::shared_ptr master_; List lru_; List free_lst_; -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) sem_t sem_; #endif TaskGroup *watchdog_grp_; diff --git a/mindspore/ccsrc/utils/system/base.h b/mindspore/ccsrc/utils/system/base.h index 4cfb5b312d..441d90200b 100644 --- a/mindspore/ccsrc/utils/system/base.h +++ b/mindspore/ccsrc/utils/system/base.h @@ -78,7 +78,7 @@ class Platform { #if defined(SYSTEM_ENV_POSIX) platform_ = kPlatformPosix; #elif defined(SYSTEM_ENV_POSIX_ANDROID) - platform_ = kPlatformPosixAndroid + platform_ = kPlatformPosixAndroid; #elif defined(SYSTEM_ENV_WINDOWS) platform_ = kPlatformWindows; #endif diff --git a/mindspore/lite/CMakeLists.txt b/mindspore/lite/CMakeLists.txt index 62da9d7cae..f81c4052bb 100644 --- a/mindspore/lite/CMakeLists.txt +++ b/mindspore/lite/CMakeLists.txt @@ -62,7 +62,8 @@ option(BUILD_CONVERTER "if build converter" on) option(ENABLE_FP16 "if build fp16 ops" off) option(SUPPORT_GPU "if support gpu" off) option(OFFLINE_COMPILE "if offline compile OpenCL kernel" off) -option(BUILD_MINDDATA "" off) +set(BUILD_MINDDATA "off" CACHE STRING "off, lite, or full") +option(BUILD_MINDDATA_EXAMPLE "" on) set(CMAKE_VERBOSE_MAKEFILE on) add_compile_definitions(USE_ANDROID_LOG) @@ -172,7 +173,9 @@ if (BUILD_DEVICE) endif () endif () -if (BUILD_MINDDATA) +if (BUILD_MINDDATA STREQUAL "lite" OR BUILD_MINDDATA STREQUAL "full") + # TODO: add sentencepiece dependency + #include(${TOP_DIR}/cmake/external_libs/sentencepiece.cmake) # opencv set(OpenCV_DIR ${TOP_DIR}/third_party/opencv/build) find_package(OpenCV REQUIRED) diff --git a/mindspore/lite/minddata/CMakeLists.txt b/mindspore/lite/minddata/CMakeLists.txt index fa76960560..46abb56b9f 100644 --- a/mindspore/lite/minddata/CMakeLists.txt +++ b/mindspore/lite/minddata/CMakeLists.txt @@ -1,27 +1,190 @@ set(MINDDATA_DIR ${CCSRC_DIR}/minddata/dataset) set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall -Wno-deprecated-declarations") -set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb") + +set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D_LIBCPP_DISABLE_EXTERN_TEMPLATE=1 -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp") + +set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -I/usr/local/include -std=c++17 -Wall -fPIC") + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPTION_CXX_FLAGS}") + +if (PLATFORM_ARM64 OR PLATFORM_ARM32) + set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Werror -Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare -Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move -Wno-inconsistent-missing-override -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") +else() + set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Wl,--allow-shlib-undefined -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") +endif() + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-non-virtual-dtor") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-private-field") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reorder") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-switch") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sequence-point") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable") +if (PLATFORM_ARM64 OR PLATFORM_ARM32) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-virtual-dtor") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-uninitialized") +else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-maybe-uninitialized") +endif() +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default") + set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS} -s") +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/api MINDDATA_API_SRC_FILES) + +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/callback MINDDATA_CALLBACK_SRC_FILES) + AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/core MINDDATA_CORE_SRC_FILES) -list(REMOVE_ITEM MINDDATA_CORE_SRC_FILES "${MINDDATA_DIR}/core/client.cc") + +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/engine MINDDATA_ENGINE_SRC_FILES) + +# TODO: add cache +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/engine/cache MINDDATA_ENGINE_CACHE_SRC_FILES) + +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/engine/datasetops MINDDATA_ENGINE_DATASETOPS_SRC_FILES) +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/engine/datasetops/map_op MINDDATA_ENGINE_DATASETOPS_MAPOP_SRC_FILES) +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/engine/datasetops/source MINDDATA_ENGINE_DATASETOPS_SOURCE_SRC_FILES) +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/engine/datasetops/source/sampler MINDDATA_ENGINE_DATASETOPS_SOURCE_SAMPLER_SRC_FILES) + +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/engine/opt MINDDATA_ENGINE_OPT_SRC_FILES) +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/engine/opt/optional MINDDATA_ENGINE_OPT_OPTIONAL_SRC_FILES) +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/engine/opt/post MINDDATA_ENGINE_OPT_POST_SRC_FILES) +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/engine/opt/pre MINDDATA_ENGINE_OPT_PRE_SRC_FILES) +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/engine/opt/util MINDDATA_ENGINE_OPT_UTIL_SRC_FILES) + +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/engine/perf MINDDATA_ENGINE_PERF_SRC_FILES) AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/kernels MINDDATA_KERNELS_SRC_FILES) -list(REMOVE_ITEM MINDDATA_KERNELS_SRC_FILES "${MINDDATA_DIR}/kernels/py_func_op.cc") AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/kernels/image MINDDATA_KERNELS_IMAGE_SRC_FILES) AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/kernels/data MINDDATA_KERNELS_DATA_SRC_FILES) -add_library(minddata-eager OBJECT +# TODO: add text src +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/text MINDDATA_TEXT_SRC_FILES) + +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/text/kernels MINDDATA_TEXT_KERNELS_SRC_FILES) + +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/util MINDDATA_UTIL_SRC_FILES) + +if (BUILD_MINDDATA STREQUAL "full") + list(REMOVE_ITEM MINDDATA_API_SRC_FILES + "${MINDDATA_DIR}/api/text.cc" + "${MINDDATA_DIR}/api/de_tensor.cc" + "${MINDDATA_DIR}/api/execute.cc" + ) + + list(REMOVE_ITEM MINDDATA_CALLBACK_SRC_FILES + "${MINDDATA_DIR}/callback/py_ds_callback.cc" + ) + + list(REMOVE_ITEM MINDDATA_KERNELS_SRC_FILES "${MINDDATA_DIR}/kernels/py_func_op.cc") + list(REMOVE_ITEM MINDDATA_ENGINE_DATASETOPS_SRC_FILES + "${MINDDATA_DIR}/engine/datasetops/build_sentence_piece_vocab_op.cc" + "${MINDDATA_DIR}/engine/datasetops/filter_op.cc" + "${MINDDATA_DIR}/engine/datasetops/barrier_op.cc" + "${MINDDATA_DIR}/engine/datasetops/bucket_batch_by_length_op.cc" + "${MINDDATA_DIR}/engine/datasetops/build_vocab_op.cc" + "${MINDDATA_DIR}/engine/datasetops/cache_merge_op.cc" + "${MINDDATA_DIR}/engine/datasetops/cache_base_op.cc" + "${MINDDATA_DIR}/engine/datasetops/cache_lookup_op.cc" + "${MINDDATA_DIR}/engine/datasetops/cache_op.cc" + ) + + list(REMOVE_ITEM MINDDATA_ENGINE_DATASETOPS_SOURCE_SRC_FILES + "${MINDDATA_DIR}/engine/datasetops/source/generator_op.cc" + "${MINDDATA_DIR}/engine/datasetops/source/voc_op.cc" + "${MINDDATA_DIR}/engine/datasetops/source/manifest_op.cc" + "${MINDDATA_DIR}/engine/datasetops/source/mindrecord_op.cc" + "${MINDDATA_DIR}/engine/datasetops/source/tf_reader_op.cc" + ) + + list(REMOVE_ITEM MINDDATA_ENGINE_DATASETOPS_SOURCE_SAMPLER_SRC_FILES + "${MINDDATA_DIR}/engine/datasetops/source/sampler/python_sampler.cc" + ) + + list(REMOVE_ITEM MINDDATA_ENGINE_OPT_POST_SRC_FILES + "${MINDDATA_DIR}/engine/opt/post/repeat_pass.cc" + ) + + list(REMOVE_ITEM MINDDATA_ENGINE_OPT_PRE_SRC_FILES + "${MINDDATA_DIR}/engine/opt/pre/cache_transform_pass.cc" + ) + + include_directories("${CMAKE_BINARY_DIR}/minddata/dataset/engine/cache") + + if (BUILD_MINDDATA_EXAMPLE AND (PLATFORM_ARM32 OR PLATFORM_ARM64)) + set(MINDDATA_EXAMPLE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/example/jni-example.cc) + endif() + + add_library(minddata-lite SHARED + ${MINDDATA_API_SRC_FILES} + ${MINDDATA_CALLBACK_SRC_FILES} + ${MINDDATA_CORE_SRC_FILES} + ${MINDDATA_ENGINE_SRC_FILES} + #${MINDDATA_ENGINE_CACHE_SRC_FILES} + ${MINDDATA_ENGINE_DATASETOPS_SRC_FILES} + ${MINDDATA_ENGINE_DATASETOPS_MAPOP_SRC_FILES} + ${MINDDATA_ENGINE_DATASETOPS_SOURCE_SRC_FILES} + ${MINDDATA_ENGINE_DATASETOPS_SOURCE_SAMPLER_SRC_FILES} + ${MINDDATA_ENGINE_OPT_SRC_FILES} + ${MINDDATA_ENGINE_OPT_OPTIONAL_SRC_FILES} + ${MINDDATA_ENGINE_OPT_POST_SRC_FILES} + ${MINDDATA_ENGINE_OPT_PRE_SRC_FILES} + ${MINDDATA_ENGINE_OPT_UTIL_SRC_FILES} + ${MINDDATA_ENGINE_PERF_SRC_FILES} + ${MINDDATA_KERNELS_SRC_FILES} + ${MINDDATA_KERNELS_IMAGE_SRC_FILES} + ${MINDDATA_KERNELS_DATA_SRC_FILES} + ${MINDDATA_UTIL_SRC_FILES} + ${MINDDATA_EXAMPLE_SRC} + ${CMAKE_CURRENT_SOURCE_DIR}/../src/common/log_adapter.cc + ${CORE_DIR}/utils/ms_utils.cc + ${CORE_DIR}/gvar/logging_level.cc + ${CCSRC_DIR}/utils/system/crc32c.cc + ) + + target_link_libraries(minddata-lite + securec + jpeg-turbo + jpeg + opencv_core + opencv_imgcodecs + opencv_imgproc + mindspore::json + ) + + # ref: https://github.com/android/ndk/issues/1202 + if (PLATFORM_ARM32) + file(GLOB_RECURSE LIBCLANG_RT_LIB $ENV{ANDROID_NDK}/libclang_rt.builtins-arm-android.a) + if (LIBCLANG_RT_LIB STREQUAL "") + MESSAGE(FATAL_ERROR "Cannot find libclang_rt.builtins-arm-androi2d.a in $ENV{ANDROID_NDK}") + endif() + target_link_libraries(minddata-lite ${LIBCLANG_RT_LIB}) + endif() + + if (PLATFORM_ARM32 OR PLATFORM_ARM64) + target_link_libraries(minddata-lite log) + elseif (BUILD_MINDDATA_EXAMPLE) + add_executable(mdlite-example ${CMAKE_CURRENT_SOURCE_DIR}/example/x86-example.cc) + target_link_libraries(mdlite-example minddata-lite) + add_custom_command(TARGET mdlite-example POST_BUILD + COMMAND cp -rf ${CMAKE_CURRENT_SOURCE_DIR}/example/testCifar10Data ${CMAKE_BINARY_DIR}/minddata + ) + endif() + +elseif (BUILD_MINDDATA STREQUAL "lite") + list(REMOVE_ITEM MINDDATA_CORE_SRC_FILES "${MINDDATA_DIR}/core/client.cc") + list(REMOVE_ITEM MINDDATA_KERNELS_SRC_FILES "${MINDDATA_DIR}/kernels/py_func_op.cc") + add_library(minddata-eager OBJECT ${MINDDATA_DIR}/api/de_tensor.cc ${MINDDATA_DIR}/api/execute.cc ) -add_library(minddata-lite SHARED + add_library(minddata-lite SHARED ${MINDDATA_CORE_SRC_FILES} ${MINDDATA_KERNELS_SRC_FILES} ${MINDDATA_KERNELS_IMAGE_SRC_FILES} @@ -34,7 +197,7 @@ add_library(minddata-lite SHARED ${CORE_DIR}/gvar/logging_level.cc ) -target_link_libraries(minddata-lite + target_link_libraries(minddata-lite securec jpeg-turbo jpeg @@ -42,4 +205,14 @@ target_link_libraries(minddata-lite opencv_imgcodecs opencv_imgproc mindspore::json - ) \ No newline at end of file + ) + + # ref: https://github.com/android/ndk/issues/1202 + if (PLATFORM_ARM32) + file(GLOB_RECURSE LIBCLANG_RT_LIB $ENV{ANDROID_NDK}/libclang_rt.builtins-arm-android.a) + if (LIBCLANG_RT_LIB STREQUAL "") + MESSAGE(FATAL_ERROR "Cannot find libclang_rt.builtins-arm-androi2d.a in $ENV{ANDROID_NDK}") + endif() + target_link_libraries(minddata-lite ${LIBCLANG_RT_LIB}) + endif() +endif() diff --git a/mindspore/lite/minddata/example/jni-example.cc b/mindspore/lite/minddata/example/jni-example.cc new file mode 100644 index 0000000000..2524d97552 --- /dev/null +++ b/mindspore/lite/minddata/example/jni-example.cc @@ -0,0 +1,95 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include "minddata/dataset/include/datasets.h" +#include "minddata/dataset/util/path.h" +#if defined(__ANDROID__) || defined(ANDROID) +#include +#include +#endif + +extern "C" JNIEXPORT jstring JNICALL +Java_com_example_mindsporepredict_MainActivity_stringFromJNI( + JNIEnv* env, + jobject /* this */) { + std::string hello = "Hello World!"; + MS_LOG(DEBUG) << hello; + return env->NewStringUTF(hello.c_str()); +} + +using Dataset = mindspore::dataset::api::Dataset; +using Iterator = mindspore::dataset::api::Iterator; +using mindspore::dataset::Tensor; +using mindspore::dataset::Path; +using mindspore::dataset::api::Cifar10; +using mindspore::dataset::api::RandomSampler; + +extern "C" JNIEXPORT void JNICALL +Java_com_example_mindsporepredict_MainActivity_pathTest( + JNIEnv* env, + jobject /* this */, + jstring path) { + MS_LOG(WARNING) << env->GetStringUTFChars(path, 0); + Path f(env->GetStringUTFChars(path, 0)); + MS_LOG(WARNING) << f.Exists() << f.IsDirectory() << f.ParentPath(); + // Print out the first few items in the directory + auto dir_it = Path::DirIterator::OpenDirectory(&f); + MS_LOG(WARNING) << dir_it.get(); + int i = 0; + while (dir_it->hasNext()) { + Path v = dir_it->next(); + MS_LOG(WARNING) << v.toString(); + i++; + if (i > 5) + break; + } +} + +extern "C" JNIEXPORT void JNICALL +Java_com_example_mindsporepredict_MainActivity_TestCifar10Dataset( + JNIEnv* env, + jobject /* this */, + jstring path) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10Dataset."; + + // Create a Cifar10 Dataset + std::string folder_path = env->GetStringUTFChars(path, 0); + std::shared_ptr ds = Cifar10(folder_path, RandomSampler(false, 10)); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + iter->GetNextRow(&row); + } + + // Manually terminate the pipeline + iter->Stop(); +} diff --git a/mindspore/lite/minddata/example/testCifar10Data/data_batch_1.bin b/mindspore/lite/minddata/example/testCifar10Data/data_batch_1.bin new file mode 100644 index 0000000000..b3ec462f79 Binary files /dev/null and b/mindspore/lite/minddata/example/testCifar10Data/data_batch_1.bin differ diff --git a/mindspore/lite/minddata/example/x86-example.cc b/mindspore/lite/minddata/example/x86-example.cc new file mode 100644 index 0000000000..4aff13ed1d --- /dev/null +++ b/mindspore/lite/minddata/example/x86-example.cc @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include "minddata/dataset/include/datasets.h" +#include "minddata/dataset/util/path.h" + +using Dataset = mindspore::dataset::api::Dataset; +using Iterator = mindspore::dataset::api::Iterator; +using mindspore::dataset::Tensor; +using mindspore::dataset::api::Cifar10; +using mindspore::dataset::api::RandomSampler; + +int main() { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10Dataset."; + + // Create a Cifar10 Dataset + std::string folder_path = "./testCifar10Data/"; + std::shared_ptr ds = Cifar10(folder_path, RandomSampler(false, 10)); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + iter->GetNextRow(&row); + } + + // Manually terminate the pipeline + iter->Stop(); +} diff --git a/mindspore/lite/src/CMakeLists.txt b/mindspore/lite/src/CMakeLists.txt index 0dbf379c4f..eda8d8c089 100644 --- a/mindspore/lite/src/CMakeLists.txt +++ b/mindspore/lite/src/CMakeLists.txt @@ -59,7 +59,7 @@ add_subdirectory(runtime/kernel/arm) if (PLATFORM_ARM32 OR PLATFORM_ARM64) target_link_libraries(mindspore-lite log) endif () -if (BUILD_MINDDATA) +if (BUILD_MINDDATA STREQUAL "lite") target_link_libraries(mindspore-lite minddata-eager minddata-lite) endif () diff --git a/mindspore/lite/test/CMakeLists.txt b/mindspore/lite/test/CMakeLists.txt index 74028c8b3d..d922ba61f0 100644 --- a/mindspore/lite/test/CMakeLists.txt +++ b/mindspore/lite/test/CMakeLists.txt @@ -163,13 +163,14 @@ if (SUPPORT_GPU) ) endif() ### minddata lite -if (BUILD_MINDDATA) +if (BUILD_MINDDATA STREQUAL "lite") include_directories(${CCSRC_DIR}/minddata) set(DATASET_TEST_DIR ${TEST_DIR}/ut/src/dataset) set(TEST_MINDDATA_SRC ${DATASET_TEST_DIR}/de_tensor_test.cc ${DATASET_TEST_DIR}/eager_test.cc ) +elseif (BUILD_MINDDATA STREQUAL "full") endif() ### runtime framework file(GLOB_RECURSE OPS_SRC ${LITE_DIR}/src/ops/*.cc) @@ -355,7 +356,7 @@ endif () add_executable(lite-test ${TEST_SRC}) target_link_libraries(lite-test dl ${GTEST_LIBRARY}) -if (BUILD_MINDDATA) +if (BUILD_MINDDATA STREQUAL "lite") target_link_libraries(lite-test minddata-lite minddata-eager @@ -363,6 +364,10 @@ if (BUILD_MINDDATA) if (PLATFORM_ARM32 OR PLATFORM_ARM64) target_link_libraries(lite-test log) endif() +elseif(BUILD_MINDDATA STREQUAL "full") + if (PLATFORM_ARM32 OR PLATFORM_ARM64) + target_link_libraries(lite-test log) + endif() endif() if (BUILD_CONVERTER) target_link_libraries(lite-test diff --git a/mindspore/lite/test/run_benchmark_nets.sh b/mindspore/lite/test/run_benchmark_nets.sh index ae5f15d651..d30f556266 100644 --- a/mindspore/lite/test/run_benchmark_nets.sh +++ b/mindspore/lite/test/run_benchmark_nets.sh @@ -11,8 +11,8 @@ function Run_x86() { echo ${model_name} >> "${run_benchmark_log_file}" echo 'cd '${convertor_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86} >> "{run_benchmark_log_file}" cd ${convertor_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86} || return 1 - echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/'${model_name}'.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}" - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 >> "${run_benchmark_log_file}" + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/'${model_name}'.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 >> "${run_benchmark_log_file}" if [ $? = 0 ]; then run_result='x86: '${model_name}' pass' echo ${run_result} >> ${run_benchmark_result_file} @@ -32,8 +32,8 @@ function Run_x86() { echo ${model_name} >> "${run_benchmark_log_file}" echo 'cd '${convertor_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86} >> "${run_benchmark_log_file}" cd ${convertor_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86} || return 1 - echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/'${model_name}'.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}" - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 >> "${run_benchmark_log_file}" + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/'${model_name}'.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 >> "${run_benchmark_log_file}" if [ $? = 0 ]; then run_result='x86: '${model_name}' pass' echo ${run_result} >> ${run_benchmark_result_file} @@ -53,8 +53,8 @@ function Run_x86() { echo ${model_name} >> "${run_benchmark_log_file}" echo 'cd '${convertor_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86} >> "${run_benchmark_log_file}" cd ${convertor_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86} || return 1 - echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/'${model_name}'.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}" - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 >> "${run_benchmark_log_file}" + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/'${model_name}'.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 >> "${run_benchmark_log_file}" if [ $? = 0 ]; then run_result='x86: '${model_name}' pass' echo ${run_result} >> ${run_benchmark_result_file} @@ -74,8 +74,8 @@ function Run_x86() { echo ${model_name} >> "${run_benchmark_log_file}" echo 'cd '${convertor_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86} >> "${run_benchmark_log_file}" cd ${convertor_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86} || return 1 - echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/quantTraining/mnist_calibration_data/00099.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'_posttraining.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}" - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}_posttraining.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/quantTraining/mnist_calibration_data/00099.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}_posttraining.ms.out --warmUpLoopCount=1 --loopCount=1 >> "${run_benchmark_log_file}" + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/quantTraining/mnist_calibration_data/00099.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'_posttraining.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}_posttraining.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/quantTraining/mnist_calibration_data/00099.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}_posttraining.ms.out --warmUpLoopCount=1 --loopCount=1 >> "${run_benchmark_log_file}" if [ $? = 0 ]; then run_result='x86: '${model_name}'_posttraining pass' echo ${run_result} >> ${run_benchmark_result_file} @@ -95,8 +95,8 @@ function Run_x86() { echo ${model_name} >> "${run_benchmark_log_file}" echo 'cd '${convertor_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86} >> "${run_benchmark_log_file}" cd ${convertor_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86} || return 1 - echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1 --numThreads=1' >> "${run_benchmark_log_file}" - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 --numThreads=1 >> "${run_benchmark_log_file}" + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1 --numThreads=1' >> "${run_benchmark_log_file}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 --numThreads=1 >> "${run_benchmark_log_file}" if [ $? = 0 ]; then run_result='x86: '${model_name}'_awaretraining pass' echo ${run_result} >> ${run_benchmark_result_file} @@ -116,8 +116,8 @@ function Run_x86() { echo ${model_name} >> "${run_benchmark_log_file}" echo 'cd '${convertor_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86} >> "${run_benchmark_log_file}" cd ${convertor_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86} || return 1 - echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/'${model_name}'.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}" - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 --accuracyThreshold=1.5 >> "${run_benchmark_log_file}" + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/'${model_name}'.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 --accuracyThreshold=1.5 >> "${run_benchmark_log_file}" if [ $? = 0 ]; then run_result='x86: '${model_name}' pass' echo ${run_result} >> ${run_benchmark_result_file} @@ -498,6 +498,17 @@ rm -rf ${basepath}/benchmark_test mkdir -p ${basepath}/benchmark_test benchmark_test_path=${basepath}/benchmark_test cd ${benchmark_test_path} || exit 1 + +# If build with minddata, copy the minddata related libs +if [ -f ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/lib/libminddata-lite.so ]; then + cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/third_party/libjpeg-turbo/lib/libjpeg.so ${benchmark_test_path}/libjpeg.so || exit 1 + cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/third_party/libjpeg-turbo/lib/libturbojpeg.so ${benchmark_test_path}/libturbojpeg.so || exit 1 + cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/third_party/opencv/lib/libopencv_core.so ${benchmark_test_path}/libopencv_core.so || exit 1 + cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/third_party/opencv/lib/libopencv_imgcodecs.so ${benchmark_test_path}/libopencv_imgcodecs.so || exit 1 + cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/third_party/opencv/lib/libopencv_imgproc.so ${benchmark_test_path}/libopencv_imgproc.so || exit 1 + cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/lib/libminddata-lite.so ${benchmark_test_path}/libminddata-lite.so || exit 1 +fi + cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/lib/libmindspore-lite.so ${benchmark_test_path}/libmindspore-lite.so || exit 1 cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/lib/liboptimize.so ${benchmark_test_path}/liboptimize.so || exit 1 cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/benchmark/benchmark ${benchmark_test_path}/benchmark || exit 1