p85971406
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

unify include file of minddata

This commit is contained in:
YangLuo 2021-04-17 15:44:57 +08:00
parent 0fd6b0a7ce
commit bf48400c10
172 changed files with 366 additions and 1960 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
build.sh
View File

@ -668,11 +668,11 @@ build_lite_java_arm64() {
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/lib/libmindspore-lite.so ${JAVA_PATH}/java/app/libs/arm64-v8a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/lib/libmindspore-lite.so ${JAVA_PATH}/native/libs/arm64-v8a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/minddata/lib/libminddata-lite.so ${JAVA_PATH}/java/app/libs/arm64-v8a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/minddata/lib/libminddata-lite.so ${JAVA_PATH}/native/libs/arm64-v8a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/lib/libminddata-lite.so ${JAVA_PATH}/java/app/libs/arm64-v8a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/lib/libminddata-lite.so ${JAVA_PATH}/native/libs/arm64-v8a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/minddata/third_party/libjpeg-turbo/lib/*.so ${JAVA_PATH}/java/app/libs/arm64-v8a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/minddata/third_party/libjpeg-turbo/lib/*.so ${JAVA_PATH}/native/libs/arm64-v8a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/third_party/libjpeg-turbo/lib/*.so ${JAVA_PATH}/java/app/libs/arm64-v8a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/third_party/libjpeg-turbo/lib/*.so ${JAVA_PATH}/native/libs/arm64-v8a/
else
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/inference/lib/libmindspore-lite.so ${JAVA_PATH}/java/app/libs/arm64-v8a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/inference/lib/libmindspore-lite.so ${JAVA_PATH}/native/libs/arm64-v8a/
@ -700,11 +700,11 @@ build_lite_java_arm32() {
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/lib/libmindspore-lite.so ${JAVA_PATH}/java/app/libs/armeabi-v7a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/lib/libmindspore-lite.so ${JAVA_PATH}/native/libs/armeabi-v7a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/minddata/lib/libminddata-lite.so ${JAVA_PATH}/java/app/libs/armeabi-v7a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/minddata/lib/libminddata-lite.so ${JAVA_PATH}/native/libs/armeabi-v7a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/lib/libminddata-lite.so ${JAVA_PATH}/java/app/libs/armeabi-v7a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/lib/libminddata-lite.so ${JAVA_PATH}/native/libs/armeabi-v7a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/minddata/third_party/libjpeg-turbo/lib/*.so ${JAVA_PATH}/java/app/libs/armeabi-v7a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/minddata/third_party/libjpeg-turbo/lib/*.so ${JAVA_PATH}/native/libs/armeabi-v7a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/third_party/libjpeg-turbo/lib/*.so ${JAVA_PATH}/java/app/libs/armeabi-v7a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/third_party/libjpeg-turbo/lib/*.so ${JAVA_PATH}/native/libs/armeabi-v7a/
else
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/inference/lib/libmindspore-lite.so ${JAVA_PATH}/java/app/libs/armeabi-v7a/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/inference/lib/libmindspore-lite.so ${JAVA_PATH}/native/libs/armeabi-v7a/
@ -733,11 +733,11 @@ build_lite_java_x86() {
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/lib/libmindspore-lite.so ${JAVA_PATH}/java/linux_x86/libs/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/lib/libmindspore-lite.so ${JAVA_PATH}/native/libs/linux_x86/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/minddata/lib/libminddata-lite.so ${JAVA_PATH}/java/linux_x86/libs/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/minddata/lib/libminddata-lite.so ${JAVA_PATH}/native/libs/linux_x86/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/lib/libminddata-lite.so ${JAVA_PATH}/java/linux_x86/libs/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/lib/libminddata-lite.so ${JAVA_PATH}/native/libs/linux_x86/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/minddata/third_party/libjpeg-turbo/lib/*.so* ${JAVA_PATH}/java/linux_x86/libs/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/minddata/third_party/libjpeg-turbo/lib/*.so* ${JAVA_PATH}/native/libs/linux_x86/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/third_party/libjpeg-turbo/lib/*.so* ${JAVA_PATH}/java/linux_x86/libs/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/train/third_party/libjpeg-turbo/lib/*.so* ${JAVA_PATH}/native/libs/linux_x86/
else
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/inference/lib/libmindspore-lite.so ${JAVA_PATH}/java/linux_x86/libs/
cp ${BASEPATH}/mindspore/lite/build/java/${JTARBALL}/inference/lib/libmindspore-lite.so ${JAVA_PATH}/native/libs/linux_x86/

14
cmake/package.cmake
View File

@ -367,13 +367,13 @@ install(
## Public header files for minddata
install(
FILES ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/constants.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/transforms.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/vision.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/vision_lite.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/vision_ascend.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/execute.h
DESTINATION ${INSTALL_BASE_DIR}/include/minddata/dataset/include
FILES ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/constants.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/transforms.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision_lite.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision_ascend.h
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/execute.h
DESTINATION ${INSTALL_BASE_DIR}/include/dataset
COMPONENT mindspore
)

46
cmake/package_lite.cmake
View File

@ -10,9 +10,8 @@ if(SUPPORT_TRAIN)
set(RUNTIME_DIR ${RUNTIME_PKG_NAME}/train)
set(RUNTIME_INC_DIR ${RUNTIME_PKG_NAME}/train/include)
set(RUNTIME_LIB_DIR ${RUNTIME_PKG_NAME}/train/lib)
set(MIND_DATA_INC_DIR ${RUNTIME_PKG_NAME}/train/minddata/include)
set(MIND_DATA_LIB_DIR ${RUNTIME_PKG_NAME}/train/minddata/lib)
set(TURBO_DIR ${RUNTIME_PKG_NAME}/train/minddata/third_party/libjpeg-turbo)
set(MIND_DATA_INC_DIR ${RUNTIME_PKG_NAME}/train/include/dataset)
set(TURBO_DIR ${RUNTIME_PKG_NAME}/train/third_party/libjpeg-turbo)
set(MINDSPORE_LITE_LIB_NAME libmindspore-lite-train)
set(BENCHMARK_NAME benchmark_train)
set(BENCHMARK_ROOT_DIR ${RUNTIME_PKG_NAME}/tools/benchmark_train)
@ -20,31 +19,38 @@ else()
set(RUNTIME_DIR ${RUNTIME_PKG_NAME}/inference)
set(RUNTIME_INC_DIR ${RUNTIME_PKG_NAME}/inference/include)
set(RUNTIME_LIB_DIR ${RUNTIME_PKG_NAME}/inference/lib)
set(MIND_DATA_INC_DIR ${RUNTIME_PKG_NAME}/inference/minddata/include)
set(MIND_DATA_LIB_DIR ${RUNTIME_PKG_NAME}/inference/minddata/lib)
set(TURBO_DIR ${RUNTIME_PKG_NAME}/inference/minddata/third_party/libjpeg-turbo)
set(MIND_DATA_INC_DIR ${RUNTIME_PKG_NAME}/inference/include/dataset)
set(TURBO_DIR ${RUNTIME_PKG_NAME}/inference/third_party/libjpeg-turbo)
set(MINDSPORE_LITE_LIB_NAME libmindspore-lite)
set(BENCHMARK_NAME benchmark)
set(BENCHMARK_ROOT_DIR ${RUNTIME_PKG_NAME}/tools/benchmark)
endif()
if(BUILD_MINDDATA STREQUAL "full")
install(DIRECTORY ${TOP_DIR}/mindspore/ccsrc/minddata/dataset/liteapi/include/ DESTINATION
${MIND_DATA_INC_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME})
install(FILES
${TOP_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/constants.h
${TOP_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/data_helper.h
${TOP_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/execute.h
${TOP_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/iterator.h
${TOP_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/samplers.h
${TOP_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/transforms.h
${TOP_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision_lite.h
${TOP_DIR}/mindspore/ccsrc/minddata/dataset/liteapi/include/datasets.h
DESTINATION ${MIND_DATA_INC_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME})
if(PLATFORM_ARM64)
file(GLOB JPEGTURBO_LIB_LIST ${jpeg_turbo_LIBPATH}/*.so)
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so
DESTINATION ${MIND_DATA_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME})
DESTINATION ${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME})
install(FILES ${JPEGTURBO_LIB_LIST} DESTINATION ${TURBO_DIR}/lib COMPONENT ${RUNTIME_COMPONENT_NAME})
elseif(PLATFORM_ARM32)
file(GLOB JPEGTURBO_LIB_LIST ${jpeg_turbo_LIBPATH}/*.so)
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION
${MIND_DATA_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME})
${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME})
install(FILES ${JPEGTURBO_LIB_LIST} DESTINATION ${TURBO_DIR}/lib COMPONENT ${RUNTIME_COMPONENT_NAME})
else()
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION
${MIND_DATA_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME})
${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME})
install(FILES ${jpeg_turbo_LIBPATH}/libjpeg.so.62.3.0 DESTINATION ${TURBO_DIR}/lib
RENAME libjpeg.so.62 COMPONENT ${RUNTIME_COMPONENT_NAME})
install(FILES ${jpeg_turbo_LIBPATH}/libturbojpeg.so.0.2.0 DESTINATION ${TURBO_DIR}/lib
@ -57,16 +63,16 @@ if(BUILD_MINDDATA STREQUAL "wrapper")
COMPONENT ${RUNTIME_COMPONENT_NAME} FILES_MATCHING PATTERN "*.h" PATTERN "vision.h" EXCLUDE)
if(PLATFORM_ARM64)
file(GLOB JPEGTURBO_LIB_LIST ${jpeg_turbo_LIBPATH}/*.so)
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${MIND_DATA_LIB_DIR}
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${RUNTIME_LIB_DIR}
COMPONENT ${RUNTIME_COMPONENT_NAME})
install(FILES ${JPEGTURBO_LIB_LIST} DESTINATION ${TURBO_DIR}/lib COMPONENT ${RUNTIME_COMPONENT_NAME})
elseif(PLATFORM_ARM32)
file(GLOB JPEGTURBO_LIB_LIST ${jpeg_turbo_LIBPATH}/*.so)
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${MIND_DATA_LIB_DIR}
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${RUNTIME_LIB_DIR}
COMPONENT ${RUNTIME_COMPONENT_NAME})
install(FILES ${JPEGTURBO_LIB_LIST} DESTINATION ${TURBO_DIR}/lib COMPONENT ${RUNTIME_COMPONENT_NAME})
else()
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${MIND_DATA_LIB_DIR}
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${RUNTIME_LIB_DIR}
COMPONENT ${RUNTIME_COMPONENT_NAME})
install(FILES ${jpeg_turbo_LIBPATH}/libjpeg.so.62.3.0 DESTINATION ${TURBO_DIR}/lib RENAME libjpeg.so.62
COMPONENT ${RUNTIME_COMPONENT_NAME})
@ -79,21 +85,21 @@ if(BUILD_MINDDATA STREQUAL "lite")
install(DIRECTORY ${TOP_DIR}/mindspore/ccsrc/minddata/dataset/include/ DESTINATION ${MIND_DATA_INC_DIR}
COMPONENT ${RUNTIME_COMPONENT_NAME} FILES_MATCHING PATTERN "*.h")
if(PLATFORM_ARM64)
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${MIND_DATA_LIB_DIR}
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${RUNTIME_LIB_DIR}
COMPONENT ${RUNTIME_COMPONENT_NAME})
install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so DESTINATION ${TURBO_DIR}/lib
COMPONENT ${RUNTIME_COMPONENT_NAME})
install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so DESTINATION ${TURBO_DIR}/lib
COMPONENT ${RUNTIME_COMPONENT_NAME})
elseif(PLATFORM_ARM32)
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${MIND_DATA_LIB_DIR}
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${RUNTIME_LIB_DIR}
COMPONENT ${RUNTIME_COMPONENT_NAME})
install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so DESTINATION ${TURBO_DIR}/lib
COMPONENT ${RUNTIME_COMPONENT_NAME})
install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so DESTINATION ${TURBO_DIR}/lib
COMPONENT ${RUNTIME_COMPONENT_NAME})
else()
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${MIND_DATA_LIB_DIR}
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${RUNTIME_LIB_DIR}
COMPONENT ${RUNTIME_COMPONENT_NAME})
install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so.62.3.0
DESTINATION ${TURBO_DIR}/lib RENAME libjpeg.so.62 COMPONENT ${RUNTIME_COMPONENT_NAME})
@ -107,16 +113,16 @@ if(BUILD_MINDDATA STREQUAL "lite_cv")
install(DIRECTORY ${TOP_DIR}/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv
DESTINATION ${MIND_DATA_INC_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME} FILES_MATCHING PATTERN "*.h")
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so
DESTINATION ${MIND_DATA_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME})
DESTINATION ${RUNTIME_LIB_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME})
elseif(PLATFORM_ARM32)
install(DIRECTORY ${TOP_DIR}/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv
DESTINATION ${MIND_DATA_INC_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME} FILES_MATCHING PATTERN "*.h")
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${MIND_DATA_LIB_DIR}
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${RUNTIME_LIB_DIR}
COMPONENT ${RUNTIME_COMPONENT_NAME})
else()
install(DIRECTORY ${TOP_DIR}/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv
DESTINATION ${MIND_DATA_INC_DIR} COMPONENT ${RUNTIME_COMPONENT_NAME} FILES_MATCHING PATTERN "*.h")
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${MIND_DATA_LIB_DIR}
install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${RUNTIME_LIB_DIR}
COMPONENT ${RUNTIME_COMPONENT_NAME})
endif()
endif()

2
mindspore/ccsrc/minddata/dataset/CMakeLists.txt
View File

@ -52,7 +52,7 @@ include_directories(${CMAKE_BINARY_DIR}) # for protobuf generated .h
include_directories(${CMAKE_SOURCE_DIR}/mindspore)
include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/mindrecord/include)
include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include)
include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset)
include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/kernels/image)
######################################################################

2
mindspore/ccsrc/minddata/dataset/api/config.cc
View File

@ -16,7 +16,7 @@
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/include/config.h"
#include "minddata/dataset/include/dataset/config.h"
#include "minddata/dataset/util/log_adapter.h"
#include "minddata/dataset/util/status.h"

2
mindspore/ccsrc/minddata/dataset/api/data_helper.cc
View File

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "minddata/dataset/include/data_helper.h"
#include "minddata/dataset/include/dataset/data_helper.h"
#include <algorithm>
#include <iostream>

8
mindspore/ccsrc/minddata/dataset/api/datasets.cc
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/dataset/datasets.h"
#include <algorithm>
#include <fstream>
#include <unordered_set>
@ -23,9 +23,9 @@
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/engine/runtime_context.h"
#include "minddata/dataset/include/iterator.h"
#include "minddata/dataset/include/samplers.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/dataset/iterator.h"
#include "minddata/dataset/include/dataset/samplers.h"
#include "minddata/dataset/include/dataset/transforms.h"
#include "minddata/dataset/util/path.h"
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/core/client.h"

2
mindspore/ccsrc/minddata/dataset/api/execute.cc
View File

@ -16,7 +16,7 @@
#include <algorithm>
#include <fstream>
#include "minddata/dataset/include/execute.h"
#include "minddata/dataset/include/dataset/execute.h"
#include "minddata/dataset/core/de_tensor.h"
#include "minddata/dataset/core/tensor_row.h"
#include "minddata/dataset/core/tensor.h"

4
mindspore/ccsrc/minddata/dataset/api/iterator.cc
View File

@ -13,11 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "minddata/dataset/include/iterator.h"
#include "minddata/dataset/include/dataset/iterator.h"
#include "minddata/dataset/engine/consumers/pull_based_tree_consumer.h"
#include "minddata/dataset/engine/consumers/tree_consumer.h"
#include "minddata/dataset/engine/runtime_context.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/dataset/datasets.h"
namespace mindspore {
namespace dataset {

2
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/core/bindings.cc
View File

@ -21,7 +21,7 @@
#include "minddata/dataset/core/client.h" // DE client
#include "minddata/dataset/util/status.h"
#include "pybind11/numpy.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
namespace mindspore {
namespace dataset {

2
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/bindings.cc
View File

@ -18,7 +18,7 @@
#include "minddata/dataset/api/python/pybind_conversion.h"
#include "minddata/dataset/api/python/pybind_register.h"
#include "minddata/dataset/callback/py_ds_callback.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/engine/serdes.h"
#include "minddata/dataset/text/sentence_piece_vocab.h"

4
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc
View File

@ -17,8 +17,8 @@
#include "minddata/dataset/api/python/pybind_conversion.h"
#include "minddata/dataset/api/python/pybind_register.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/include/dataset/datasets.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/data_type.h"

2
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/samplers/bindings.cc
View File

@ -19,7 +19,7 @@
#include "minddata/dataset/engine/datasetops/source/sampler/python_sampler.h"
#include "minddata/dataset/api/python/pybind_conversion.h"
#include "minddata/dataset/api/python/pybind_register.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/engine/ir/datasetops/source/samplers/samplers_ir.h"

2
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/execute/bindings.cc
View File

@ -18,7 +18,7 @@
#include "minddata/dataset/api/python/pybind_conversion.h"
#include "minddata/dataset/api/python/pybind_register.h"
#include "minddata/dataset/core/type_id.h"
#include "minddata/dataset/include/execute.h"
#include "minddata/dataset/include/dataset/execute.h"
namespace mindspore {
namespace dataset {

4
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/schema/bindings.cc
View File

@ -19,8 +19,8 @@
#include "minddata/dataset/api/python/pybind_register.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/core/type_id.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/include/dataset/datasets.h"
namespace mindspore {
namespace dataset {

2
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/ir/bindings.cc
View File

@ -15,7 +15,7 @@
*/
#include "pybind11/pybind11.h"
#include "mindspore/ccsrc/minddata/dataset/include/transforms.h"
#include "mindspore/ccsrc/minddata/dataset/include/dataset/transforms.h"
#include "minddata/dataset/api/python/pybind_register.h"
#include "minddata/dataset/core/global_context.h"

2
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/ir/image/bindings.cc
View File

@ -17,7 +17,7 @@
#include "minddata/dataset/api/python/pybind_conversion.h"
#include "minddata/dataset/api/python/pybind_register.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/dataset/transforms.h"
#include "minddata/dataset/kernels/ir/vision/vision_ir.h"
namespace mindspore {

2
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/bindings.cc
View File

@ -20,7 +20,7 @@
#include "minddata/dataset/api/python/pybind_register.h"
#include "minddata/dataset/text/vocab.h"
#include "minddata/dataset/text/sentence_piece_vocab.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
namespace mindspore {
namespace dataset {

4
mindspore/ccsrc/minddata/dataset/api/python/pybind_conversion.h
View File

@ -32,8 +32,8 @@
#include "minddata/dataset/core/type_id.h"
#include "minddata/dataset/engine/ir/cache/pre_built_dataset_cache.h"
#include "minddata/dataset/engine/ir/datasetops/source/csv_node.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/samplers.h"
#include "minddata/dataset/include/dataset/datasets.h"
#include "minddata/dataset/include/dataset/samplers.h"
#include "minddata/dataset/kernels/ir/data/transforms_ir.h"
#include "minddata/dataset/kernels/py_func_op.h"
namespace py = pybind11;

2
mindspore/ccsrc/minddata/dataset/api/samplers.cc
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
#include "minddata/dataset/include/samplers.h"
#include "minddata/dataset/include/dataset/samplers.h"
#include "minddata/dataset/engine/ir/datasetops/source/samplers/samplers_ir.h"
namespace mindspore {

4
mindspore/ccsrc/minddata/dataset/api/text.cc
View File

@ -18,10 +18,10 @@
#include <fstream>
#include <regex>
#include "minddata/dataset/include/text.h"
#include "mindspore/core/ir/dtype/type_id.h"
#include "minddata/dataset/include/dataset/text.h"
#include "minddata/dataset/core/type_id.h"
#include "minddata/dataset/text/ir/kernels/text_ir.h"
#include "mindspore/core/ir/dtype/type_id.h"
namespace mindspore {
namespace dataset {

2
mindspore/ccsrc/minddata/dataset/api/transforms.cc
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/dataset/transforms.h"
#include <algorithm>

6
mindspore/ccsrc/minddata/dataset/api/vision.cc
View File

@ -14,13 +14,13 @@
* limitations under the License.
*/
#include "minddata/dataset/include/vision.h"
#include "minddata/dataset/include/dataset/vision.h"
#ifdef ENABLE_ACL
#include "minddata/dataset/include/vision_ascend.h"
#include "minddata/dataset/include/dataset/vision_ascend.h"
#include "minddata/dataset/kernels/ir/vision/ascend_vision_ir.h"
#endif
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/dataset/transforms.h"
#include "minddata/dataset/kernels/ir/vision/vision_ir.h"
#ifndef ENABLE_ANDROID

2
mindspore/ccsrc/minddata/dataset/core/client.h
View File

@ -19,7 +19,7 @@
// client.h
// Include file for DE client functions
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/data_type.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/core/tensor_shape.h"

2
mindspore/ccsrc/minddata/dataset/core/config_manager.h
View File

@ -23,7 +23,7 @@
#include <nlohmann/json.hpp>
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/util/path.h"
#include "minddata/dataset/util/status.h"

2
mindspore/ccsrc/minddata/dataset/core/cv_tensor.cc
View File

@ -18,7 +18,7 @@
#include <memory>
#include <vector>
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/tensor.h"
namespace mindspore {

2
mindspore/ccsrc/minddata/dataset/core/cv_tensor.h
View File

@ -24,7 +24,7 @@
#include "./securec.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/data_type.h"
#include "minddata/dataset/core/tensor.h"

2
mindspore/ccsrc/minddata/dataset/core/data_type.h
View File

@ -29,7 +29,7 @@ namespace py = pybind11;
#else
#include "base/float16.h"
#endif
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
namespace mindspore {
namespace dataset {

2
mindspore/ccsrc/minddata/dataset/core/de_tensor.cc
View File

@ -16,7 +16,7 @@
#include "minddata/dataset/core/de_tensor.h"
#include "minddata/dataset/core/device_tensor.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/type_id.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"

2
mindspore/ccsrc/minddata/dataset/core/device_tensor.h
View File

@ -20,7 +20,7 @@
#include <utility>
#include <vector>
#include "include/api/status.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/data_type.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/util/status.h"

2
mindspore/ccsrc/minddata/dataset/core/global_context.h
View File

@ -21,7 +21,7 @@
#include "include/api/status.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/util/allocator.h"
namespace mindspore {

3
mindspore/ccsrc/minddata/dataset/core/tensor.cc
View File

@ -19,6 +19,7 @@
#include <iomanip>
#include <iostream>
#include <fstream>
#include <functional>
#include <memory>
#include <vector>
#include <utility>
@ -26,7 +27,7 @@
#include "minddata/dataset/core/type_id.h"
#include "utils/ms_utils.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#ifndef ENABLE_ANDROID
#include "minddata/dataset/core/cv_tensor.h"

2
mindspore/ccsrc/minddata/dataset/core/tensor.h
View File

@ -37,7 +37,7 @@
#include "pybind11/stl.h"
#endif
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/data_type.h"
#include "minddata/dataset/core/tensor_helpers.h"
#include "minddata/dataset/core/tensor_shape.h"

4
mindspore/ccsrc/minddata/dataset/core/tensor_helpers.h
View File

@ -19,8 +19,8 @@
#include <memory>
#include <vector>
#include "mindspore/ccsrc/minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/constants.h"
#include "mindspore/ccsrc/minddata/dataset/include/dataset/transforms.h"
#include "minddata/dataset/include/dataset/constants.h"
namespace mindspore {
namespace dataset {

2
mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc
View File

@ -25,7 +25,7 @@
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
namespace mindspore {
namespace dataset {

2
mindspore/ccsrc/minddata/dataset/core/tensor_shape.h
View File

@ -31,7 +31,7 @@
namespace py = pybind11;
#endif
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/util/allocator.h"

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc vendored
View File

@ -28,7 +28,7 @@
#include "minddata/dataset/engine/cache/cache_server.h"
#include "minddata/dataset/engine/cache/cache_ipc.h"
#include "minddata/dataset/util/path.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
namespace mindspore {
namespace dataset {

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc vendored
View File

@ -22,7 +22,7 @@
#include <cstdlib>
#include <cstring>
#include <thread>
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/engine/cache/cache_client.h"
#include "minddata/dataset/engine/cache/cache_fbb.h"
namespace mindspore {

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc vendored
View File

@ -18,7 +18,7 @@
#include <functional>
#include <limits>
#include <vector>
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/engine/cache/cache_ipc.h"
#include "minddata/dataset/engine/cache/cache_service.h"
#include "minddata/dataset/engine/cache/cache_request.h"

2
mindspore/ccsrc/minddata/dataset/engine/data_schema.h
View File

@ -23,7 +23,7 @@
#include <unordered_map>
#include <vector>
#include <nlohmann/json.hpp>
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/data_type.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/util/status.h"

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc
View File

@ -16,7 +16,7 @@
#include "minddata/dataset/engine/datasetops/barrier_op.h"
#include <iomanip>
#include <utility>
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/core/config_manager.h"

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc
View File

@ -15,7 +15,7 @@
*/
#include "minddata/dataset/engine/datasetops/cache_lookup_op.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/engine/execution_tree.h"
#ifndef ENABLE_ANDROID

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc
View File

@ -20,7 +20,7 @@
#include <iomanip>
#include <utility>
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/util/system_pool.h"

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc
View File

@ -18,7 +18,7 @@
#include <memory>
#include <utility>
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/engine/datasetops/repeat_op.h"
#include "minddata/dataset/engine/dataset_iterator.h"

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h
View File

@ -24,7 +24,7 @@
#include <utility>
#include "minddata/dataset/callback/callback_manager.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/util/status.h"

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc
View File

@ -21,7 +21,7 @@
#include "minddata/dataset/callback/callback_param.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/engine/datasetops/map_op/cpu_map_job.h"

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.h
View File

@ -19,7 +19,7 @@
#include <memory>
#include <string>
#include <vector>
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/engine/datasetops/dataset_op.h"
#include "minddata/dataset/engine/datasetops/source/io_block.h"
#include "minddata/dataset/util/status.h"

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc
View File

@ -22,7 +22,7 @@
#include "utils/ms_utils.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc
View File

@ -17,7 +17,7 @@
#include <algorithm>
#include <utility>
#include <iomanip>
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/core/config_manager.h"

2
mindspore/ccsrc/minddata/dataset/engine/db_connector.h
View File

@ -21,7 +21,7 @@
#include "minddata/dataset/core/tensor_row.h"
#include "minddata/dataset/engine/connector.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
namespace mindspore {
namespace dataset {

2
mindspore/ccsrc/minddata/dataset/engine/gpu_item_connector.h
View File

@ -23,7 +23,7 @@
#include <vector>
#include "minddata/dataset/engine/connector.h"
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "runtime/device/gpu/blocking_queue.h"
using mindspore::device::DataItemGpu;

2
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/build_sentence_piece_vocab_node.h
View File

@ -24,7 +24,7 @@
#include <vector>
#include "minddata/dataset/engine/ir/datasetops/dataset_node.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/dataset/datasets.h"
namespace mindspore {
namespace dataset {

2
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.h
View File

@ -37,7 +37,7 @@
#include "minddata/dataset/engine/datasetops/take_op.h"
#include "minddata/dataset/engine/ir/cache/dataset_cache.h"
#include "minddata/dataset/engine/ir/datasetops/source/samplers/samplers_ir.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/dataset/datasets.h"
#include "minddata/dataset/util/path.h"
#include "minddata/dataset/util/status.h"

2
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/random_node.h
View File

@ -24,7 +24,7 @@
#include "minddata/dataset/engine/datasetops/source/sampler/sampler.h"
#include "minddata/dataset/engine/ir/datasetops/dataset_node.h"
#include "minddata/dataset/include/samplers.h"
#include "minddata/dataset/include/dataset/samplers.h"
namespace mindspore {
namespace dataset {

2
mindspore/ccsrc/minddata/dataset/engine/jagged_connector.h
View File

@ -23,7 +23,7 @@
#include "minddata/dataset/engine/connector.h"
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
namespace mindspore {
namespace dataset {

2
mindspore/ccsrc/minddata/dataset/engine/opt/pre/deep_copy_pass.cc
View File

@ -15,7 +15,7 @@
*/
#include <string>
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/dataset/datasets.h"
#include "minddata/dataset/engine/opt/pre/deep_copy_pass.h"
#include "minddata/dataset/engine/ir/datasetops/root_node.h"

2
mindspore/ccsrc/minddata/dataset/engine/opt/pre/input_validation_pass.cc
View File

@ -15,7 +15,7 @@
*/
#include <string>
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/dataset/datasets.h"
#include "minddata/dataset/engine/opt/pre/input_validation_pass.h"
namespace mindspore {

2
mindspore/ccsrc/minddata/dataset/engine/perf/cyclic_array.h
View File

@ -21,7 +21,7 @@
#include <algorithm>
#include <cstring>
#include <type_traits>
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
namespace mindspore {
namespace dataset {

2
mindspore/ccsrc/minddata/dataset/engine/perf/perf_data.h
View File

@ -18,7 +18,7 @@
#define MINDSPORE_CCSRC_MINDDATA_DATASET_PERF_DATA_H
#include <vector>
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
namespace mindspore {
namespace dataset {

6
mindspore/ccsrc/minddata/dataset/include/config.h → mindspore/ccsrc/minddata/dataset/include/dataset/config.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONFIG_H
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONFIG_H
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_CONFIG_H
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_CONFIG_H
#include <cstdint>
#include <string>
@ -83,4 +83,4 @@ inline bool load(std::string file) { return load(StringToChar(file)); }
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONFIG_H
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_CONFIG_H

6
mindspore/ccsrc/minddata/dataset/include/constants.h → mindspore/ccsrc/minddata/dataset/include/dataset/constants.h
View File

@ -13,8 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONSTANTS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONSTANTS_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_CONSTANTS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_CONSTANTS_H_
#include <cstdint>
#include <limits>
@ -119,4 +119,4 @@ using row_id_type = int64_t;
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONSTANTS_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_CONSTANTS_H_

6
mindspore/ccsrc/minddata/dataset/include/data_helper.h → mindspore/ccsrc/minddata/dataset/include/dataset/data_helper.h
View File

@ -13,8 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATA_HELPER_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATA_HELPER_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATA_HELPER_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATA_HELPER_H_
#include <sys/stat.h>
#include <fstream>
@ -445,4 +445,4 @@ class DataHelper {
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATA_HELPER_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATA_HELPER_H_

14
mindspore/ccsrc/minddata/dataset/include/datasets.h → mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASETS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASETS_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATASETS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATASETS_H_
#include <sys/stat.h>
#include <unistd.h>
@ -33,10 +33,10 @@
#include "include/api/dual_abi_helper.h"
#include "include/api/types.h"
#include "minddata/dataset/include/iterator.h"
#include "minddata/dataset/include/json_fwd.hpp"
#include "minddata/dataset/include/samplers.h"
#include "minddata/dataset/include/text.h"
#include "include/dataset/iterator.h"
#include "include/dataset/json_fwd.hpp"
#include "include/dataset/samplers.h"
#include "include/dataset/text.h"
namespace mindspore {
namespace dataset {
@ -1675,4 +1675,4 @@ inline std::shared_ptr<ZipDataset> Zip(const std::vector<std::shared_ptr<Dataset
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASETS_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATASETS_H_

10
mindspore/ccsrc/minddata/dataset/include/execute.h → mindspore/ccsrc/minddata/dataset/include/dataset/execute.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_EXECUTE_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_EXECUTE_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_EXECUTE_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_EXECUTE_H_
#include <string>
#include <vector>
@ -23,8 +23,8 @@
#include <memory>
#include "include/api/context.h"
#include "include/api/types.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/transforms.h"
#include "include/dataset/constants.h"
#include "include/dataset/transforms.h"
namespace mindspore {
namespace dataset {
@ -89,4 +89,4 @@ class Execute {
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_EXECUTE_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_EXECUTE_H_

6
mindspore/ccsrc/minddata/dataset/include/iterator.h → mindspore/ccsrc/minddata/dataset/include/dataset/iterator.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_ITERATOR_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_ITERATOR_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_ITERATOR_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_ITERATOR_H_
#include <map>
#include <memory>
@ -150,4 +150,4 @@ class PullIterator : public Iterator {
};
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_ITERATOR_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_ITERATOR_H_

0
mindspore/ccsrc/minddata/dataset/include/json_fwd.hpp → mindspore/ccsrc/minddata/dataset/include/dataset/json_fwd.hpp
View File

6
mindspore/ccsrc/minddata/dataset/include/samplers.h → mindspore/ccsrc/minddata/dataset/include/dataset/samplers.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_SAMPLERS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_SAMPLERS_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_SAMPLERS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_SAMPLERS_H_
#include <memory>
#include <vector>
@ -248,4 +248,4 @@ class WeightedRandomSampler final : public Sampler {
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_SAMPLERS_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_SAMPLERS_H_

10
mindspore/ccsrc/minddata/dataset/include/text.h → mindspore/ccsrc/minddata/dataset/include/dataset/text.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_TEXT_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_TEXT_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_TEXT_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_TEXT_H_
#include <memory>
#include <optional>
@ -25,8 +25,8 @@
#include "include/api/dual_abi_helper.h"
#include "include/api/status.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/transforms.h"
#include "include/dataset/constants.h"
#include "include/dataset/transforms.h"
namespace mindspore {
namespace dataset {
@ -539,4 +539,4 @@ class WhitespaceTokenizer final : public TensorTransform {
} // namespace text
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_TEXT_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_TEXT_H_

8
mindspore/ccsrc/minddata/dataset/include/transforms.h → mindspore/ccsrc/minddata/dataset/include/dataset/transforms.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_TRANSFORMS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_TRANSFORMS_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_TRANSFORMS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_TRANSFORMS_H_
#include <map>
#include <memory>
@ -25,7 +25,7 @@
#include "include/api/dual_abi_helper.h"
#include "include/api/status.h"
#include "include/api/types.h"
#include "minddata/dataset/include/constants.h"
#include "include/dataset/constants.h"
namespace mindspore {
namespace dataset {
@ -410,4 +410,4 @@ class Unique final : public TensorTransform {
} // namespace transforms
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_TRANSFORMS_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_TRANSFORMS_H_

12
mindspore/ccsrc/minddata/dataset/include/vision.h → mindspore/ccsrc/minddata/dataset/include/dataset/vision.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_VISION_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_VISION_H_
#include <map>
#include <memory>
@ -25,9 +25,9 @@
#include "include/api/dual_abi_helper.h"
#include "include/api/status.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/vision_lite.h"
#include "include/dataset/constants.h"
#include "include/dataset/transforms.h"
#include "include/dataset/vision_lite.h"
namespace mindspore {
namespace dataset {
@ -952,4 +952,4 @@ class UniformAugment final : public TensorTransform {
} // namespace vision
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_VISION_H_

10
mindspore/ccsrc/minddata/dataset/include/vision_ascend.h → mindspore/ccsrc/minddata/dataset/include/dataset/vision_ascend.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_ASCEND_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_ASCEND_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_VISION_ASCEND_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_VISION_ASCEND_H_
#include <map>
#include <memory>
@ -23,8 +23,8 @@
#include <utility>
#include <vector>
#include "include/api/status.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/transforms.h"
#include "include/dataset/constants.h"
#include "include/dataset/transforms.h"
namespace mindspore {
namespace dataset {
@ -97,4 +97,4 @@ class DvppDecodePng final : public TensorTransform {
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_ASCEND_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_VISION_ASCEND_H_

10
mindspore/ccsrc/minddata/dataset/include/vision_lite.h → mindspore/ccsrc/minddata/dataset/include/dataset/vision_lite.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_LITE_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_LITE_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_VISION_LITE_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_VISION_LITE_H_
#include <map>
#include <memory>
@ -23,8 +23,8 @@
#include <utility>
#include <vector>
#include "include/api/status.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/transforms.h"
#include "include/dataset/constants.h"
#include "include/dataset/transforms.h"
namespace mindspore {
namespace dataset {
@ -289,4 +289,4 @@ class Rotate final : public TensorTransform {
} // namespace vision
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_LITE_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_VISION_LITE_H_

2
mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.cc
View File

@ -22,7 +22,7 @@
#include <vector>
#include <utility>
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/data_type.h"
#ifdef ENABLE_PYTHON
#include "minddata/dataset/core/pybind_support.h"

2
mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.h
View File

@ -20,7 +20,7 @@
#include <string>
#include <vector>
#include <unordered_map>
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#ifndef ENABLE_ANDROID
#include "minddata/dataset/core/cv_tensor.h"
#endif

2
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/utils/MDAclProcess.cc
View File

@ -13,7 +13,7 @@
* limitations under the License.
*/
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/kernels/image/image_utils.h"
#include "MDAclProcess.h"

2
mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc
View File

@ -24,7 +24,7 @@
#include "minddata/dataset/core/cv_tensor.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/kernels/image/math_utils.h"
#include "minddata/dataset/kernels/image/resize_cubic_op.h"

2
mindspore/ccsrc/minddata/dataset/kernels/image/lite_image_utils.cc
View File

@ -19,7 +19,7 @@
#include <stdexcept>
#include "minddata/dataset/kernels/image/lite_cv/lite_mat.h"
#include "minddata/dataset/kernels/image/lite_cv/image_process.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/util/random.h"

2
mindspore/ccsrc/minddata/dataset/kernels/image/pad_op.cc
View File

@ -16,7 +16,7 @@
#include "minddata/dataset/kernels/image/pad_op.h"
#include "minddata/dataset/kernels/image/image_utils.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/util/status.h"
namespace mindspore {

2
mindspore/ccsrc/minddata/dataset/kernels/image/pad_op.h
View File

@ -22,7 +22,7 @@
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/kernels/tensor_op.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/util/status.h"
namespace mindspore {

4
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/ascend_vision_ir.h
View File

@ -24,8 +24,8 @@
#include <vector>
#include "include/api/status.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/include/dataset/transforms.h"
#include "minddata/dataset/kernels/ir/tensor_operation.h"
namespace mindspore {

4
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/vision_ir.h
View File

@ -24,8 +24,8 @@
#include <vector>
#include "include/api/status.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/include/dataset/transforms.h"
#include "minddata/dataset/kernels/ir/tensor_operation.h"
namespace mindspore {

112
mindspore/ccsrc/minddata/dataset/liteapi/include/constants.h
View File

@ -1,112 +0,0 @@
/**
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONSTANTS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONSTANTS_H_
#include <cstdint>
#include <limits>
#include <random>
namespace mindspore {
namespace dataset {
// Various type defines for convenience
using uchar = unsigned char;
using dsize_t = int64_t;
// Target devices to perform map operation
enum class MapTargetDevice { kCpu, kGpu, kAscend310 };
// Possible dataset types for holding the data and client type
enum class DatasetType { kUnknown, kArrow, kTf };
// Possible flavours of Tensor implementations
enum class TensorImpl { kNone, kFlexible, kCv, kNP };
// Possible values for shuffle
enum class ShuffleMode { kFalse = 0, kFiles = 1, kGlobal = 2 };
// Possible values for Border types
enum class BorderType { kConstant = 0, kEdge = 1, kReflect = 2, kSymmetric = 3 };
// Possible values for Image format types in a batch
enum class ImageBatchFormat { kNHWC = 0, kNCHW = 1 };
// Possible values for Image format types
enum class ImageFormat { HWC = 0, CHW = 1, HW = 2 };
// Possible interpolation modes
enum class InterpolationMode { kLinear = 0, kNearestNeighbour = 1, kCubic = 2, kArea = 3 };
// Possible JiebaMode modes
enum class JiebaMode { kMix = 0, kMp = 1, kHmm = 2 };
// Possible values for SPieceTokenizerOutType
enum class SPieceTokenizerOutType { kString = 0, kInt = 1 };
// Possible values for SPieceTokenizerLoadType
enum class SPieceTokenizerLoadType { kFile = 0, kModel = 1 };
// Possible values for SentencePieceModel
enum class SentencePieceModel { kUnigram = 0, kBpe = 1, kChar = 2, kWord = 3 };
// Possible values for NormalizeForm
enum class NormalizeForm {
kNone = 0,
kNfc,
kNfkc,
kNfd,
kNfkd,
};
// Possible values for SamplingStrategy
enum class SamplingStrategy { kRandom = 0, kEdgeWeight = 1 };
// convenience functions for 32bit int bitmask
inline bool BitTest(uint32_t bits, uint32_t bitMask) { return (bits & bitMask) == bitMask; }
inline void BitSet(uint32_t *bits, uint32_t bitMask) { *bits |= bitMask; }
inline void BitClear(uint32_t *bits, uint32_t bitMask) { *bits &= (~bitMask); }
constexpr int32_t kDeMaxDim = std::numeric_limits<int32_t>::max(); // 2147483647 or 2^32 -1
constexpr int32_t kDeMaxRank = std::numeric_limits<int32_t>::max();
constexpr int64_t kDeMaxFreq = std::numeric_limits<int64_t>::max(); // 9223372036854775807 or 2^(64-1)
constexpr int64_t kDeMaxTopk = std::numeric_limits<int64_t>::max();
constexpr uint32_t kCfgRowsPerBuffer = 1;
constexpr uint32_t kCfgParallelWorkers = 8;
constexpr uint32_t kCfgWorkerConnectorSize = 16;
constexpr uint32_t kCfgOpConnectorSize = 16;
constexpr int32_t kCfgDefaultRankId = -1;
constexpr uint32_t kCfgDefaultSeed = std::mt19937::default_seed;
constexpr uint32_t kCfgMonitorSamplingInterval = 10;
constexpr uint32_t kCfgCallbackTimeout = 60; // timeout value for callback in seconds
constexpr int32_t kCfgDefaultCachePort = 50052;
constexpr char kCfgDefaultCacheHost[] = "127.0.0.1";
constexpr int32_t kDftPrefetchSize = 20;
constexpr int32_t kDftNumConnections = 12;
constexpr int32_t kDftAutoNumWorkers = false;
// Invalid OpenCV type should not be from 0 to 7 (opencv4/opencv2/core/hal/interface.h)
constexpr uint8_t kCVInvalidType = 255;
using connection_id_type = uint64_t;
using session_id_type = uint32_t;
using row_id_type = int64_t;
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONSTANTS_H_

448
mindspore/ccsrc/minddata/dataset/liteapi/include/data_helper.h
View File

@ -1,448 +0,0 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATA_HELPER_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATA_HELPER_H_
#include <sys/stat.h>
#include <fstream>
#include <iostream>
#include <map>
#include <memory>
#include <sstream>
#include <string>
#include <unordered_map>
#include <vector>
#include "include/api/dual_abi_helper.h"
#include "include/api/status.h"
namespace mindspore {
namespace dataset {
/// \brief Simple class to do data manipulation, contains helper function to update json files in dataset
class DataHelper {
public:
/// \brief constructor
DataHelper() {}
/// \brief Destructor
~DataHelper() = default;
/// \brief Create an Album dataset while taking in a path to a image folder
/// Creates the output directory if doesn't exist
/// \param[in] in_dir Image folder directory that takes in images
/// \param[in] out_dir Directory containing output json files
Status CreateAlbum(const std::string &in_dir, const std::string &out_dir) {
return CreateAlbumIF(StringToChar(in_dir), StringToChar(out_dir));
}
/// \brief Update a json file field with a vector of string values
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value array to write to file
/// \param out_file Optional input for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<std::string> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), VectorStringToChar(value), StringToChar(out_file));
}
/// \brief Update a json file field with a vector of bool values
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value array to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<bool> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with a vector of int8 values
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value array to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int8_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with a vector of uint8 values
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value array to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint8_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with a vector of int16 values
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value array to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int16_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with a vector of uint16 values
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value array to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint16_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with a vector of int32 values
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value array to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int32_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with a vector of uint32 values
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value array to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint32_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with a vector of int64 values
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value array to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int64_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with a vector of uint64 values
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value array to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint64_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with a vector of float values
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value array to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<float> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with a vector of double values
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value array to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<double> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with a string value
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateValue(const std::string &in_file, const std::string &key, const std::string &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), StringToChar(value), StringToChar(out_file));
}
/// \brief Update a json file field with a bool value
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateValue(const std::string &in_file, const std::string &key, const bool &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with an int8 value
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateValue(const std::string &in_file, const std::string &key, const int8_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with an uint8 value
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateValue(const std::string &in_file, const std::string &key, const uint8_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with an int16 value
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateValue(const std::string &in_file, const std::string &key, const int16_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with an uint16 value
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateValue(const std::string &in_file, const std::string &key, const uint16_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with an int32 value
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateValue(const std::string &in_file, const std::string &key, const int32_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with an uint32 value
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateValue(const std::string &in_file, const std::string &key, const uint32_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with an int64 value
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateValue(const std::string &in_file, const std::string &key, const int64_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with an uint64 value
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateValue(const std::string &in_file, const std::string &key, const uint64_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with a float value
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateValue(const std::string &in_file, const std::string &key, const float &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Update a json file field with a double value
/// \param in_file The input file name to read in
/// \param key Key of field to write to
/// \param value Value to write to file
/// \param out_file Optional parameter for output file path, will write to input file if not specified
/// \return Status The status code returned
Status UpdateValue(const std::string &in_file, const std::string &key, const double &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
/// \brief Template function to write tensor to file
/// \param[in] in_file File to write to
/// \param[in] data Array of type T values
/// \return Status The status code returned
template <typename T>
Status WriteBinFile(const std::string &in_file, const std::vector<T> &data) {
try {
std::ofstream o(in_file, std::ios::binary | std::ios::out);
if (!o.is_open()) {
return Status(kMDUnexpectedError, "Error opening Bin file to write");
}
size_t length = data.size();
o.write(reinterpret_cast<const char *>(&data[0]), std::streamsize(length * sizeof(T)));
o.close();
}
// Catch any exception and convert to Status return code
catch (const std::exception &err) {
return Status(kMDUnexpectedError, "Write bin file failed ");
}
return Status::OK();
}
/// \brief Write pointer to bin, use pointer to avoid memcpy
/// \param[in] in_file File name to write to
/// \param[in] data Pointer to data
/// \param[in] length Length of values to write from pointer
/// \return Status The status code returned
template <typename T>
Status WriteBinFile(const std::string &in_file, T *data, size_t length) {
try {
std::ofstream o(in_file, std::ios::binary | std::ios::out);
if (!o.is_open()) {
return Status(kMDUnexpectedError, "Error opening Bin file to write");
}
o.write(reinterpret_cast<const char *>(data), std::streamsize(length * sizeof(T)));
o.close();
}
// Catch any exception and convert to Status return code
catch (const std::exception &err) {
return Status(kMDUnexpectedError, "Write bin file failed ");
}
return Status::OK();
}
/// \brief Helper function to copy content of a tensor to buffer
/// \note This function iterates over the tensor in bytes, since
/// \param[in] tensor_addr The memory held by a tensor
/// \param[in] tensor_size The amount of data in bytes in tensor_addr, e.g. tensor->SizeInBytes()
/// \param[out] addr The address to copy tensor data to
/// \param[in] buffer_size The buffer size of addr
/// \return The size of the tensor (bytes copied
size_t DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr, const size_t &buffer_size);
/// \brief Helper function to delete key in json file
/// note This function will return okay even if key not found
/// \param[in] in_file Json file to remove key from
/// \param[in] key The key to remove
/// \return Status The status code returned
Status RemoveKey(const std::string &in_file, const std::string &key, const std::string &out_file = "") {
return RemoveKeyIF(StringToChar(in_file), StringToChar(key), StringToChar(out_file));
}
/// \brief A print method typically used for debugging
/// \param out - The output stream to write output to
void Print(std::ostream &out) const;
/// \brief << Stream output operator overload
/// \notes This allows you to write the debug print info using stream operators
/// \param out Reference to the output stream being overloaded
/// \param ds Reference to the DataSchema to display
/// \return The output stream must be returned
friend std::ostream &operator<<(std::ostream &out, const DataHelper &dh) {
dh.Print(out);
return out;
}
private:
// Helper function for dual ABI support
Status CreateAlbumIF(const std::vector<char> &in_dir, const std::vector<char> &out_dir);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<std::vector<char>> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<bool> &value,
const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<int8_t> &value,
const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<uint8_t> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<int16_t> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<uint16_t> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<int32_t> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<uint32_t> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<int64_t> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<uint64_t> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<float> &value,
const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<double> &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<char> &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const bool &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int8_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint8_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int16_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint16_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int32_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint32_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int64_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint64_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const float &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const double &value,
const std::vector<char> &out_file);
Status RemoveKeyIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<char> &out_file);
};
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATA_HELPER_H_

12
mindspore/ccsrc/minddata/dataset/liteapi/include/datasets.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASETS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASETS_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATASETS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATASETS_H_
#include <sys/stat.h>
#include <unistd.h>
@ -32,9 +32,9 @@
#include "include/api/dual_abi_helper.h"
#include "include/api/types.h"
#include "include/iterator.h"
#include "include/samplers.h"
#include "include/transforms.h"
#include "include/dataset/iterator.h"
#include "include/dataset/samplers.h"
#include "include/dataset/transforms.h"
namespace mindspore {
namespace dataset {
@ -570,4 +570,4 @@ inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASETS_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATASETS_H_

88
mindspore/ccsrc/minddata/dataset/liteapi/include/execute.h
View File

@ -1,88 +0,0 @@
/**
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_EXECUTE_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_EXECUTE_H_
#include <string>
#include <vector>
#include <map>
#include <memory>
#include "include/api/context.h"
#include "include/api/types.h"
#include "include/constants.h"
#include "include/transforms.h"
namespace mindspore {
namespace dataset {
class DeviceResource;
// class to run tensor operations in eager mode
class Execute {
public:
/// \brief Constructor
// FIXME - Temporarily overload Execute to support both TensorOperation and TensorTransform
explicit Execute(std::shared_ptr<TensorOperation> op, MapTargetDevice deviceType = MapTargetDevice::kCpu);
explicit Execute(std::shared_ptr<TensorTransform> op, MapTargetDevice deviceType = MapTargetDevice::kCpu);
explicit Execute(std::reference_wrapper<TensorTransform> op, MapTargetDevice deviceType = MapTargetDevice::kCpu);
explicit Execute(TensorTransform *op, MapTargetDevice deviceType = MapTargetDevice::kCpu);
explicit Execute(std::vector<std::shared_ptr<TensorOperation>> ops,
MapTargetDevice deviceType = MapTargetDevice::kCpu);
explicit Execute(std::vector<std::shared_ptr<TensorTransform>> ops,
MapTargetDevice deviceType = MapTargetDevice::kCpu);
explicit Execute(const std::vector<std::reference_wrapper<TensorTransform>> ops,
MapTargetDevice deviceType = MapTargetDevice::kCpu);
explicit Execute(std::vector<TensorTransform *> ops, MapTargetDevice deviceType = MapTargetDevice::kCpu);
/// \brief Destructor
~Execute();
/// \brief callable function to execute the TensorOperation in eager mode
/// \param[in] input Tensor to be transformed
/// \param[out] output Transformed tensor
/// \return Status code
Status operator()(const mindspore::MSTensor &input, mindspore::MSTensor *output);
/// \brief callable function to execute the TensorOperation in eager mode
/// \param[in] input_tensor_list List of Tensor to be transformed
/// \param[out] out Result tensor after transform
/// \return - Status
Status operator()(const std::vector<mindspore::MSTensor> &input_tensor_list, std::vector<mindspore::MSTensor> *out);
Status DeviceMemoryRelease();
std::string AippCfgGenerator();
private:
Status ParseTransforms_();
Status validate_device_();
std::vector<std::shared_ptr<TensorTransform>> transforms_;
std::vector<std::shared_ptr<TensorOperation>> ops_;
MapTargetDevice device_type_;
std::shared_ptr<DeviceResource> device_resource_;
struct ExtraInfo;
std::shared_ptr<ExtraInfo> info_;
};
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_EXECUTE_H_

166
mindspore/ccsrc/minddata/dataset/liteapi/include/iterator.h
View File

@ -1,166 +0,0 @@
/**
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_ITERATOR_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_ITERATOR_H_
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "include/api/dual_abi_helper.h"
#include "include/api/status.h"
#include "include/api/types.h"
namespace mindspore {
namespace dataset {
// Forward declare
class ExecutionTree;
class DatasetIterator;
class DatasetOp;
class Tensor;
class NativeRuntimeContext;
class IteratorConsumer;
class PullBasedIteratorConsumer;
class Dataset;
using MSTensorMap = std::unordered_map<std::string, mindspore::MSTensor>;
using MSTensorMapChar = std::map<std::vector<char>, mindspore::MSTensor>;
using MSTensorVec = std::vector<mindspore::MSTensor>;
// Abstract class for iterating over the dataset.
class Iterator {
public:
/// \brief Constructor
Iterator();
/// \brief Destructor
~Iterator();
/// \brief Method for building and launching the pipeline.
/// \param[in] ops - a vector of DatasetOp in the data pipeline.
/// \param[in] num_epochs Number of epochs passed down to EpochCtrlNode, default -1, infinite epochs
/// \return - a Status error code, returns OK if no error encountered.
Status BuildAndLaunchTree(std::shared_ptr<Dataset> ds, int32_t num_epochs);
/// \brief Function to get the next row from the data pipeline.
/// \note Type of return data is a map(with column name).
/// \param[out] row - the output tensor row.
/// \return - a Status error code, returns OK if no error encountered.
Status GetNextRow(MSTensorMap *row) {
MSTensorMapChar row_;
row_.clear();
row->clear();
Status s = GetNextRowCharIF(&row_);
TensorMapCharToString(&row_, row);
return s;
}
// Char interface(CharIF) of GetNextRow
// This api exists because std::string will constrained by ABI compile macro but char don't.
Status GetNextRowCharIF(MSTensorMapChar *row);
/// \brief Function to get the next row from the data pipeline.
/// \note Type of return data is a vector(without column name).
/// \param[out] row - the output tensor row.
/// \return - a Status error code, returns OK if no error encountered.
virtual Status GetNextRow(MSTensorVec *row);
/// \brief Function to shut down the data pipeline.
void Stop();
class _Iterator {
public:
explicit _Iterator(Iterator *lt) : lt_{lt}, cur_row_{nullptr} {
if (lt_) {
cur_row_ = new MSTensorMap();
lt_->GetNextRow(cur_row_);
}
}
// Destructor
~_Iterator() {
if (cur_row_) {
delete cur_row_;
}
}
_Iterator &operator++() {
if (lt_) {
++ind_;
lt_->GetNextRow(cur_row_);
}
if (cur_row_ && cur_row_->size() == 0) {
delete cur_row_;
cur_row_ = nullptr;
}
return *this;
} // prefix ++ overload
MSTensorMap &operator*() { return *cur_row_; } // dereference operator
MSTensorMap *operator->() { return cur_row_; }
bool operator!=(const _Iterator &rhs) { return cur_row_ != rhs.cur_row_; }
private:
int ind_; // the cur node our Iterator points to
Iterator *lt_;
MSTensorMap *cur_row_;
};
_Iterator begin() { return _Iterator(this); }
_Iterator end() { return _Iterator(nullptr); }
private:
std::unique_ptr<NativeRuntimeContext> runtime_context_;
IteratorConsumer *consumer_;
};
class PullIterator : public Iterator {
public:
/// \brief Constructor
PullIterator();
/// \brief Function to get next row from the data pipeline.
/// \note Type of return data is a vector(without column name).
/// \param[out] row - the output tensor row.
/// \return Returns true if no error encountered else false.
Status GetNextRow(MSTensorVec *row) override;
/// \brief Function to get specified rows from the data pipeline.
/// \note Type of return data is a vector(without column name).
/// \note This behavior is subject to change
/// \param[in] num_rows - the number of rows to fetch.
/// \param[out] row - the output tensor row.
/// \return Returns true if no error encountered else false.
Status GetRows(int32_t num_rows, std::vector<MSTensorVec> *row);
/// \brief Method for building and launching the pipeline.
/// \note Consider making this function protected.
/// \param[in] ds - The root node that calls the function
/// \return - a Status error code, returns OK if no error encountered.
Status BuildAndLaunchTree(std::shared_ptr<Dataset> ds);
private:
std::unique_ptr<PullBasedIteratorConsumer> pull_consumer_;
};
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_ITERATOR_H_

251
mindspore/ccsrc/minddata/dataset/liteapi/include/samplers.h
View File

@ -1,251 +0,0 @@
/**
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_SAMPLERS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_SAMPLERS_H_
#include <memory>
#include <vector>
namespace mindspore {
namespace dataset {
// Forward declare
class SamplerObj;
// Abstract class to represent a sampler in the data pipeline.
/// \class Sampler samplers.h
/// \brief An abstract base class to represent a sampler in the data pipeline.
class Sampler : std::enable_shared_from_this<Sampler> {
friend class AlbumDataset;
friend class CelebADataset;
friend class Cifar10Dataset;
friend class Cifar100Dataset;
friend class CLUEDataset;
friend class CocoDataset;
friend class CSVDataset;
friend class ImageFolderDataset;
friend class ManifestDataset;
friend class MindDataDataset;
friend class MnistDataset;
friend class RandomDataDataset;
friend class TextFileDataset;
friend class TFRecordDataset;
friend class VOCDataset;
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
public:
/// \brief Constructor
Sampler() {}
/// \brief Destructor
~Sampler() = default;
/// \brief A virtual function to add a child sampler.
/// \param[in] child The child sampler to be added as a children of this sampler.
virtual void AddChild(std::shared_ptr<Sampler> child) { children_.push_back(child); }
protected:
/// \brief Pure virtual function to convert a Sampler class into an IR Sampler object.
/// \return shared pointer to the newly created TensorOperation.
virtual std::shared_ptr<SamplerObj> Parse() const = 0;
std::vector<std::shared_ptr<Sampler>> children_;
};
/// \brief A class to represent a Distributed Sampler in the data pipeline.
/// \notes A Sampler that accesses a shard of the dataset.
class DistributedSampler final : public Sampler {
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
public:
/// \brief Constructor
/// \param[in] num_shards - Number of shards to divide the dataset into.
/// \param[in] shard_id - Shard ID of the current shard within num_shards.
/// \param[in] shuffle - If true, the indices are shuffled.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
/// \param[in] seed - The seed in use when shuffle is true.
/// \param[in] offset - The starting position where access to elements in the dataset begins.
/// \param[in] even_dist - If true, each shard would return the same number of rows (default to true).
/// If false the total rows returned by all the shards would not have overlap.
explicit DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle = true, int64_t num_samples = 0,
uint32_t seed = 1, int64_t offset = -1, bool even_dist = true);
/// \brief Destructor.
~DistributedSampler() = default;
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t num_shards_;
int64_t shard_id_;
bool shuffle_;
int64_t num_samples_;
uint32_t seed_;
int64_t offset_;
bool even_dist_;
};
/// \brief A class to represent a PK Sampler in the data pipeline.
/// \notes Samples K elements for each P class in the dataset.
/// This will sample all classes.
class PKSampler final : public Sampler {
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
public:
/// \brief Constructor
/// \param[in] num_val - Number of elements to sample for each class.
/// \param[in] shuffle - If true, the class IDs are shuffled.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
explicit PKSampler(int64_t num_val, bool shuffle = false, int64_t num_samples = 0);
/// \brief Destructor.
~PKSampler() = default;
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t num_val_;
bool shuffle_;
int64_t num_samples_;
};
/// \brief A class to represent a Random Sampler in the data pipeline.
/// \notes Samples the elements randomly.
class RandomSampler final : public Sampler {
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
public:
/// \brief Constructor
/// \param[in] replacement - If true, put the sample ID back for the next draw.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
explicit RandomSampler(bool replacement = false, int64_t num_samples = 0);
/// \brief Destructor.
~RandomSampler() = default;
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() const override;
private:
bool replacement_;
int64_t num_samples_;
};
/// \brief A class to represent a Sequential Sampler in the data pipeline.
/// \notes Samples the dataset elements sequentially, same as not having a sampler.
class SequentialSampler final : public Sampler {
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
public:
/// \brief Constructor
/// \param[in] start_index - Index to start sampling at (default to start at first id).
/// \param[in] num_samples - The number of samples to draw (default to all elements).
explicit SequentialSampler(int64_t start_index = 0, int64_t num_samples = 0);
/// \brief Destructor.
~SequentialSampler() = default;
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t start_index_;
int64_t num_samples_;
};
/// \brief A class to represent a Subset Sampler in the data pipeline.
/// \notes Samples the elements from a sequence of indices.
class SubsetSampler : public Sampler {
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
public:
/// \brief Constructor
/// \param[in] indices - A vector sequence of indices.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
explicit SubsetSampler(std::vector<int64_t> indices, int64_t num_samples = 0);
/// \brief Destructor.
~SubsetSampler() = default;
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() const override;
std::vector<int64_t> indices_;
int64_t num_samples_;
};
/// \brief A class to represent a Subset Random Sampler in the data pipeline.
/// \notes Samples the elements randomly from a sequence of indices.
class SubsetRandomSampler final : public SubsetSampler {
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
public:
/// \brief Constructor
/// \param[in] indices - A vector sequence of indices.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
explicit SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples = 0);
/// \brief Destructor.
~SubsetRandomSampler() = default;
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() const override;
};
/// \brief A class to represent a Weighted Random Sampler in the data pipeline.
/// \notes Samples the elements from [0, len(weights) - 1] randomly with the given
/// weights (probabilities).
class WeightedRandomSampler final : public Sampler {
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
public:
/// \brief Constructor
/// \param[in] weights - A vector sequence of weights, not necessarily summing up to 1.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
/// \param[in] replacement - If true, put the sample ID back for the next draw.
explicit WeightedRandomSampler(std::vector<double> weights, int64_t num_samples = 0, bool replacement = true);
/// \brief Destructor.
~WeightedRandomSampler() = default;
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() const override;
private:
std::vector<double> weights_;
int64_t num_samples_;
bool replacement_;
};
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_SAMPLERS_H_

243
mindspore/ccsrc/minddata/dataset/liteapi/include/transforms.h
View File

@ -1,243 +0,0 @@
/**
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_TRANSFORMS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_TRANSFORMS_H_
#include <map>
#include <memory>
#include <string>
#include <vector>
#include "include/api/dual_abi_helper.h"
#include "include/api/status.h"
#include "include/constants.h"
namespace mindspore {
namespace dataset {
class TensorOperation;
// We need the following two groups of forward declaration to friend the class in class TensorTransform.
namespace transforms {
class Compose;
class RandomApply;
class RandomChoice;
} // namespace transforms
namespace vision {
class BoundingBoxAugment;
class RandomSelectSubpolicy;
class UniformAugment;
} // namespace vision
// Abstract class to represent a tensor transform operation in the data pipeline.
/// \class TensorTransform transforms.h
/// \brief A base class to represent a tensor transform operation in the data pipeline.
class TensorTransform : public std::enable_shared_from_this<TensorTransform> {
friend class Dataset;
friend class Execute;
friend class transforms::Compose;
friend class transforms::RandomApply;
friend class transforms::RandomChoice;
friend class vision::BoundingBoxAugment;
friend class vision::RandomSelectSubpolicy;
friend class vision::UniformAugment;
public:
/// \brief Constructor
TensorTransform() {}
/// \brief Destructor
~TensorTransform() = default;
protected:
/// \brief Pure virtual function to convert a TensorTransform class into a IR TensorOperation object.
/// \return shared pointer to the newly created TensorOperation.
virtual std::shared_ptr<TensorOperation> Parse() = 0;
/// \brief Virtual function to convert a TensorTransform class into a IR TensorOperation object.
/// \param[in] env A string to determine the running environment
/// \return shared pointer to the newly created TensorOperation.
virtual std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) { return nullptr; }
};
// Transform operations for performing data transformation.
namespace transforms {
/// \brief Compose Op.
/// \notes Compose a list of transforms into a single transform.
class Compose final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] transforms A vector of raw pointers to TensorTransform objects to be applied.
explicit Compose(const std::vector<TensorTransform *> &transforms);
/// \brief Constructor.
/// \param[in] transforms A vector of shared pointers to TensorTransform objects to be applied.
explicit Compose(const std::vector<std::shared_ptr<TensorTransform>> &transforms);
/// \brief Constructor.
/// \param[in] transforms A vector of TensorTransform objects to be applied.
explicit Compose(const std::vector<std::reference_wrapper<TensorTransform>> &transforms);
/// \brief Destructor
~Compose() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief Duplicate Op.
/// \notes Duplicate the input tensor to a new output tensor.
/// The input tensor is carried over to the output list.
class Duplicate final : public TensorTransform {
public:
/// \brief Constructor.
Duplicate();
/// \brief Destructor
~Duplicate() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief OneHot Op.
/// \notes Convert the labels into OneHot format.
class OneHot final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] num_classes number of classes.
explicit OneHot(int32_t num_classes);
/// \brief Destructor
~OneHot() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief RandomApply Op.
/// \notes Randomly perform a series of transforms with a given probability.
class RandomApply final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] transforms A vector of raw pointers to TensorTransform objects to be applied.
/// \param[in] prob The probability to apply the transformation list (default=0.5)
explicit RandomApply(const std::vector<TensorTransform *> &transforms, double prob = 0.5);
/// \brief Constructor.
/// \param[in] transforms A vector of shared pointers to TensorTransform objects to be applied.
/// \param[in] prob The probability to apply the transformation list (default=0.5)
explicit RandomApply(const std::vector<std::shared_ptr<TensorTransform>> &transforms, double prob = 0.5);
/// \brief Constructor.
/// \param[in] transforms A vector of TensorTransform objects to be applied.
/// \param[in] prob The probability to apply the transformation list (default=0.5)
explicit RandomApply(const std::vector<std::reference_wrapper<TensorTransform>> &transforms, double prob = 0.5);
/// \brief Destructor
~RandomApply() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief RandomChoice Op.
/// \notes Randomly selects one transform from a list of transforms to perform operation.
class RandomChoice final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] transforms A vector of raw pointers to TensorTransform objects to be applied.
explicit RandomChoice(const std::vector<TensorTransform *> &transforms);
/// \brief Constructor.
/// \param[in] transforms A vector of shared pointers to TensorTransform objects to be applied.
explicit RandomChoice(const std::vector<std::shared_ptr<TensorTransform>> &transforms);
/// \brief Constructor.
/// \param[in] transforms A vector of TensorTransform objects to be applied.
explicit RandomChoice(const std::vector<std::reference_wrapper<TensorTransform>> &transforms);
/// \brief Destructor
~RandomChoice() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief TypeCast Op.
/// \notes Tensor operation to cast to a given MindSpore data type.
class TypeCast final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] data_type mindspore::DataType to be cast to.
explicit TypeCast(mindspore::DataType data_type);
/// \brief Destructor
~TypeCast() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief Unique Op.
/// \notes Return an output tensor containing all the unique elements of the input tensor in
/// the same order that they occur in the input tensor.
class Unique final : public TensorTransform {
public:
/// \brief Constructor.
Unique();
/// \brief Destructor
~Unique() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
} // namespace transforms
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_TRANSFORMS_H_

292
mindspore/ccsrc/minddata/dataset/liteapi/include/vision_lite.h
View File

@ -1,292 +0,0 @@
/**
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_LITE_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_LITE_H_
#include <map>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "include/api/status.h"
#include "include/constants.h"
#include "include/transforms.h"
namespace mindspore {
namespace dataset {
// Transform operations for performing computer vision.
namespace vision {
// Forward Declarations
class RotateOperation;
/// \brief Affine TensorTransform.
/// \notes Apply affine transform on input image.
class Affine final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] degrees The degrees to rotate the image by
/// \param[in] translation The value representing vertical and horizontal translation (default = {0.0, 0.0})
/// The first value represent the x axis translation while the second represents y axis translation.
/// \param[in] scale The scaling factor for the image (default = 0.0)
/// \param[in] shear A float vector of size 2, representing the shear degrees (default = {0.0, 0.0})
/// \param[in] interpolation An enum for the mode of interpolation
/// \param[in] fill_value A vector representing the value to fill the area outside the transform
/// in the output image. If 1 value is provided, it is used for all RGB channels.
/// If 3 values are provided, it is used to fill R, G, B channels respectively.
explicit Affine(float_t degrees, const std::vector<float> &translation = {0.0, 0.0}, float scale = 0.0,
const std::vector<float> &shear = {0.0, 0.0},
InterpolationMode interpolation = InterpolationMode::kNearestNeighbour,
const std::vector<uint8_t> &fill_value = {0, 0, 0});
/// \brief Destructor.
~Affine() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief CenterCrop TensorTransform.
/// \notes Crops the input image at the center to the given size.
class CenterCrop final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] size A vector representing the output size of the cropped image.
/// If size is a single value, a square crop of size (size, size) is returned.
/// If size has 2 values, it should be (height, width).
explicit CenterCrop(std::vector<int32_t> size);
/// \brief Destructor.
~CenterCrop() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief ResizePreserveAR TensorTransform.
/// \notes Keep the original picture ratio and fill the rest.
class ResizePreserveAR final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] height The height of image output value after resizing.
/// \param[in] width The width of image output value after resizing.
/// \param[in] img_orientation Angle method of image rotation.
ResizePreserveAR(int32_t height, int32_t width, int32_t img_orientation = 0);
/// \brief Destructor.
~ResizePreserveAR() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief RGB2GRAY TensorTransform.
/// \notes Convert RGB image or color image to grayscale image
class RGB2GRAY : public TensorTransform {
public:
/// \brief Constructor.
RGB2GRAY() = default;
/// \brief Destructor.
~RGB2GRAY() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief Crop TensorTransform.
/// \notes Crop an image based on location and crop size
class Crop final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] coordinates Starting location of crop. Must be a vector of two values, in the form of {x_coor, y_coor}
/// \param[in] size Size of the cropped area.
/// If size is a single value, a square crop of size (size, size) is returned.
/// If size has 2 values, it should be (height, width).
Crop(std::vector<int32_t> coordinates, std::vector<int32_t> size);
/// \brief Destructor.
~Crop() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief Decode TensorTransform.
/// \notes Decode the input image in RGB mode.
class Decode final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] rgb A boolean of whether to decode in RGB mode or not.
explicit Decode(bool rgb = true);
/// \brief Destructor.
~Decode() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief Normalize TensorTransform.
/// \notes Normalize the input image with respect to mean and standard deviation.
class Normalize final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] mean A vector of mean values for each channel, w.r.t channel order.
/// The mean values must be in range [0.0, 255.0].
/// \param[in] std A vector of standard deviations for each channel, w.r.t. channel order.
/// The standard deviation values must be in range (0.0, 255.0]
Normalize(std::vector<float> mean, std::vector<float> std);
/// \brief Destructor.
~Normalize() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief RandomAffine TensorTransform.
/// \notes Applies a Random Affine transformation on input image in RGB or Greyscale mode.
class RandomAffine final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] degrees A float vector of size 2, representing the starting and ending degree
/// \param[in] translate_range A float vector of size 2 or 4, representing percentages of translation on x and y axes.
/// if size is 2, (min_dx, max_dx, 0, 0)
/// if size is 4, (min_dx, max_dx, min_dy, max_dy)
/// all values are in range [-1, 1]
/// \param[in] scale_range A float vector of size 2, representing the starting and ending scales in the range.
/// \param[in] shear_ranges A float vector of size 2 or 4, representing the starting and ending shear degrees
/// vertically and horizontally.
/// if size is 2, (min_shear_x, max_shear_x, 0, 0)
/// if size is 4, (min_shear_x, max_shear_x, min_shear_y, max_shear_y)
/// \param[in] interpolation An enum for the mode of interpolation
/// \param[in] fill_value A vector representing the value to fill the area outside the transform
/// in the output image. If 1 value is provided, it is used for all RGB channels.
/// If 3 values are provided, it is used to fill R, G, B channels respectively.
explicit RandomAffine(const std::vector<float_t> &degrees,
const std::vector<float_t> &translate_range = {0.0, 0.0, 0.0, 0.0},
const std::vector<float_t> &scale_range = {1.0, 1.0},
const std::vector<float_t> &shear_ranges = {0.0, 0.0, 0.0, 0.0},
InterpolationMode interpolation = InterpolationMode::kNearestNeighbour,
const std::vector<uint8_t> &fill_value = {0, 0, 0});
/// \brief Destructor.
~RandomAffine() = default;
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief Resize TensorTransform.
/// \notes Resize the input image to the given size.
class Resize final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] size A vector representing the output size of the resized image.
/// If size is a single value, the image will be resized to this value with
/// the same image aspect ratio. If size has 2 values, it should be (height, width).
/// \param[in] interpolation An enum for the mode of interpolation
explicit Resize(std::vector<int32_t> size, InterpolationMode interpolation = InterpolationMode::kLinear);
/// \brief Destructor.
~Resize() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
std::shared_ptr<TensorOperation> Parse(const MapTargetDevice &env) override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief Rotate TensorTransform.
/// \notes Rotate the input image using a specified angle id.
class Rotate final : public TensorTransform {
public:
/// \brief Constructor.
Rotate();
/// \brief Destructor.
~Rotate() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
std::shared_ptr<RotateOperation> op_;
};
} // namespace vision
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_LITE_H_

2
mindspore/ccsrc/minddata/dataset/text/kernels/data_utils.h
View File

@ -20,7 +20,7 @@
#include <string>
#include <vector>
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/core/data_type.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/core/cv_tensor.h"

2
mindspore/ccsrc/minddata/dataset/text/kernels/jieba_tokenizer_op.h
View File

@ -19,7 +19,7 @@
#include <string>
#include <memory>
#include "cppjieba/Jieba.hpp"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/kernels/tensor_op.h"
#include "minddata/dataset/util/status.h"

2
mindspore/ccsrc/minddata/dataset/text/kernels/sentence_piece_tokenizer_op.h
View File

@ -23,7 +23,7 @@
#include <iostream>
#include <memory>
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/kernels/tensor_op.h"
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/text/sentence_piece_vocab.h"

2
mindspore/ccsrc/minddata/dataset/text/sentence_piece_vocab.h
View File

@ -22,7 +22,7 @@
#include <vector>
#include <unordered_map>
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/dataset/constants.h"
namespace mindspore {
namespace dataset {

2
mindspore/lite/CMakeLists.txt
View File

@ -118,7 +118,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src/runtime/kernel/arm)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src/)
include_directories(${TOP_DIR}/third_party)
include_directories(${CMAKE_BINARY_DIR})
include_directories(${CCSRC_DIR}/minddata/dataset/liteapi)
include_directories(${CCSRC_DIR}/minddata/dataset)
include(${TOP_DIR}/cmake/utils.cmake)
include(${TOP_DIR}/cmake/dependency_utils.cmake)

3
mindspore/lite/examples/train_lenet/prepare_and_run.sh
View File

@ -81,8 +81,7 @@ cp scripts/*.sh ${PACKAGE}/
# Copy the shared MindSpore ToD library
tar -xzf ${TARBALL}
mv mindspore-*/train/lib ${PACKAGE}/
mv mindspore-*/train/minddata/lib/* ${PACKAGE}/lib/
mv mindspore-*/train/minddata/third_party/libjpeg-turbo/lib/* ${PACKAGE}/lib/
mv mindspore-*/train/third_party/libjpeg-turbo/lib/* ${PACKAGE}/lib/
if [ "${TARGET}" == "arm64" ]; then
tar -xzf ${TARBALL} --wildcards --no-anchored hiai_ddk
mv mindspore-*/train/third_party/hiai_ddk/lib/* ${PACKAGE}/lib/

6
mindspore/lite/examples/train_lenet/src/net_runner.cc
View File

@ -29,9 +29,9 @@
#include "include/train/accuracy_metrics.h"
#include "include/train/classification_train_accuracy_monitor.h"
#include "src/utils.h"
#include "include/datasets.h"
#include "include/vision_lite.h"
#include "include/transforms.h"
#include "include/dataset/datasets.h"
#include "include/dataset/vision_lite.h"
#include "include/dataset/transforms.h"
using mindspore::dataset::Dataset;
using mindspore::dataset::Mnist;

2
mindspore/lite/examples/train_lenet/src/net_runner.h
View File

@ -26,7 +26,7 @@
#include "include/train/train_loop.h"
#include "include/train/accuracy_metrics.h"
#include "include/ms_tensor.h"
#include "include/datasets.h"
#include "include/dataset/datasets.h"
#include "include/model.h"
using mindspore::dataset::Dataset;

3
mindspore/lite/examples/transfer_learning/prepare_and_run.sh
View File

@ -80,8 +80,7 @@ cp scripts/*.sh ${PACKAGE}/
# Copy the shared MindSpore ToD library
tar -xzf ${TARBALL}
mv mindspore-*/train/lib ${PACKAGE}/
mv mindspore-*/train/minddata/lib/* ${PACKAGE}/lib/
mv mindspore-*/train/minddata/third_party/libjpeg-turbo/lib/* ${PACKAGE}/lib/
mv mindspore-*/train/third_party/libjpeg-turbo/lib/* ${PACKAGE}/lib/
if [ "${TARGET}" == "arm64" ]; then
tar -xzf ${TARBALL} --wildcards --no-anchored hiai_ddk
mv mindspore-*/train/third_party/hiai_ddk/lib/* ${PACKAGE}/lib/

1
mindspore/lite/minddata/CMakeLists.txt
View File

@ -102,6 +102,7 @@ AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/kernels/image/lite_cv MINDDATA_KERNELS_IMA
AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/kernels/ir MINDDATA_KERNELS_IR_SRC_FILES)
if(BUILD_MINDDATA STREQUAL "full")
include_directories("${MINDDATA_DIR}")
include_directories("${MINDDATA_DIR}/kernels/image")
include_directories("${MINDDATA_DIR}/liteapi")
include_directories("${TOP_DIR}")

16
mindspore/lite/minddata/example/CMakeLists.txt
View File

@ -4,10 +4,8 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall -fPIC -std=c++17")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare")
set(MD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mindspore-lite-1.2.0-inference-linux-x64/minddata")
set(MS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mindspore-lite-1.2.0-inference-linux-x64/")
set(MS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mindspore-lite-1.2.0-inference-linux-x64/inference")
include_directories(${MD_DIR})
include_directories(${MS_DIR})
@ -16,9 +14,9 @@ add_executable(testlenet
)
target_link_libraries(testlenet
${MD_DIR}/lib/libminddata-lite.so
${MD_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so.62
${MD_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so.0
${MS_DIR}/lib/libminddata-lite.so
${MS_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so.62
${MS_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so.0
${MS_DIR}/lib/libmindspore-lite.so
pthread)
@ -27,8 +25,8 @@ add_executable(testresize
)
target_link_libraries(testresize
${MD_DIR}/lib/libminddata-lite.so
${MD_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so.62
${MD_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so.0
${MS_DIR}/lib/libminddata-lite.so
${MS_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so.62
${MS_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so.0
${MS_DIR}/lib/libmindspore-lite.so
pthread)

Some files were not shown because too many files have changed in this diff Show More