From 82f7c6f2c7c8757facc404ca862ad2015c6d0eaf Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 5 Mar 2021 14:06:25 -0500 Subject: [PATCH] Moving data_helper exposing json WIP fix for removing templates Removing templates part 1 replacing core/constants.h removed template part 2 Changed fake data to filler data Fixing CI Fix CI part 2 CI fix part 2 Fix CI 3 Fixed lizard for data_util Fixed clang Fixing CI part 4 --- .../ccsrc/minddata/dataset/api/CMakeLists.txt | 1 + .../ccsrc/minddata/dataset/api/data_helper.cc | 193 ++++++++ .../python/bindings/dataset/core/bindings.cc | 2 +- .../dataset/engine/ir/datasetops/bindings.cc | 2 +- .../engine/ir/datasetops/source/bindings.cc | 2 +- .../ir/datasetops/source/samplers/bindings.cc | 2 +- .../dataset/engine/ir/schema/bindings.cc | 2 +- .../ccsrc/minddata/dataset/core/client.h | 2 +- .../minddata/dataset/core/config_manager.h | 4 +- .../ccsrc/minddata/dataset/core/constants.h | 112 ----- .../ccsrc/minddata/dataset/core/cv_tensor.cc | 2 +- .../ccsrc/minddata/dataset/core/cv_tensor.h | 2 +- .../ccsrc/minddata/dataset/core/data_type.h | 2 +- .../ccsrc/minddata/dataset/core/de_tensor.cc | 2 +- .../minddata/dataset/core/device_tensor.h | 2 +- .../minddata/dataset/core/global_context.h | 2 +- .../ccsrc/minddata/dataset/core/tensor.cc | 2 +- .../ccsrc/minddata/dataset/core/tensor.h | 2 +- .../minddata/dataset/core/tensor_helpers.h | 2 +- .../minddata/dataset/core/tensor_shape.cc | 2 +- .../minddata/dataset/core/tensor_shape.h | 2 +- .../dataset/engine/cache/cache_admin_arg.cc | 2 +- .../dataset/engine/cache/cache_request.cc | 2 +- .../dataset/engine/cache/cache_server.cc | 2 +- .../minddata/dataset/engine/data_buffer.h | 6 +- .../minddata/dataset/engine/data_schema.h | 8 +- .../dataset/engine/datasetops/barrier_op.cc | 2 +- .../engine/datasetops/cache_lookup_op.cc | 2 +- .../engine/datasetops/cache_merge_op.cc | 2 +- .../dataset/engine/datasetops/cache_op.cc | 2 +- .../dataset/engine/datasetops/dataset_op.h | 2 +- .../engine/datasetops/map_op/map_op.cc | 2 +- .../dataset/engine/datasetops/parallel_op.h | 2 +- .../dataset/engine/datasetops/rename_op.cc | 2 +- .../engine/datasetops/source/mindrecord_op.cc | 2 +- .../dataset/engine/datasetops/zip_op.cc | 2 +- .../minddata/dataset/engine/db_connector.h | 2 +- .../dataset/engine/gpu_item_connector.h | 2 +- .../dataset/engine/jagged_connector.h | 2 +- .../dataset/engine/perf/cyclic_array.h | 2 +- .../minddata/dataset/engine/perf/perf_data.h | 2 +- .../minddata/dataset/include/constants.h | 12 +- .../minddata/dataset/include/data_helper.h | 448 ++++++++++++++++++ .../ccsrc/minddata/dataset/include/datasets.h | 3 +- .../minddata/dataset/include/json_fwd.hpp | 74 +++ .../dataset/kernels/data/data_utils.cc | 193 ++++---- .../dataset/kernels/data/data_utils.h | 2 +- .../kernels/image/dvpp/utils/MDAclProcess.cc | 2 +- .../dataset/kernels/image/image_utils.cc | 2 +- .../dataset/kernels/image/lite_image_utils.cc | 2 +- .../minddata/dataset/kernels/image/pad_op.cc | 2 +- .../minddata/dataset/kernels/image/pad_op.h | 2 +- .../dataset/text/kernels/data_utils.h | 2 +- .../dataset/text/kernels/jieba_tokenizer_op.h | 2 +- .../kernels/sentence_piece_tokenizer_op.h | 2 +- .../util/{data_helper.cc => json_helper.cc} | 14 +- .../util/{data_helper.h => json_helper.h} | 16 +- mindspore/dataset/engine/datasets.py | 2 +- mindspore/lite/minddata/CMakeLists.txt | 6 +- mindspore/lite/minddata/wrapper/MDToDApi.cc | 2 +- tests/ut/cpp/dataset/bit_functions_test.cc | 2 +- tests/ut/cpp/dataset/common/bboxop_common.cc | 4 +- tests/ut/cpp/dataset/common/cvop_common.cc | 2 +- tests/ut/cpp/dataset/data_helper_test.cc | 2 +- tests/ut/cpp/dataset/datatype_test.cc | 2 +- .../cpp/dataset/distributed_sampler_test.cc | 2 +- tests/ut/cpp/dataset/rename_op_test.cc | 2 +- .../cpp/dataset/subset_random_sampler_test.cc | 2 +- tests/ut/cpp/dataset/subset_sampler_test.cc | 2 +- .../dataset/weighted_random_sampler_test.cc | 2 +- tests/ut/cpp/dataset/zip_op_test.cc | 2 +- 71 files changed, 910 insertions(+), 296 deletions(-) create mode 100644 mindspore/ccsrc/minddata/dataset/api/data_helper.cc delete mode 100644 mindspore/ccsrc/minddata/dataset/core/constants.h create mode 100644 mindspore/ccsrc/minddata/dataset/include/data_helper.h create mode 100644 mindspore/ccsrc/minddata/dataset/include/json_fwd.hpp rename mindspore/ccsrc/minddata/dataset/util/{data_helper.cc => json_helper.cc} (90%) rename mindspore/ccsrc/minddata/dataset/util/{data_helper.h => json_helper.h} (95%) diff --git a/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt index d884ebd0862..d2310b598d9 100644 --- a/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt @@ -29,6 +29,7 @@ endif() add_library(cpp-API OBJECT config.cc + data_helper.cc datasets.cc execute.cc iterator.cc diff --git a/mindspore/ccsrc/minddata/dataset/api/data_helper.cc b/mindspore/ccsrc/minddata/dataset/api/data_helper.cc new file mode 100644 index 00000000000..b8f55218f76 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/data_helper.cc @@ -0,0 +1,193 @@ +/** + * Copyright 2020-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "minddata/dataset/include/data_helper.h" + +#include +#include +#include +#include +#include +#include + +#include "minddata/dataset/util/json_helper.h" +#include "minddata/dataset/util/log_adapter.h" +#include "minddata/dataset/util/path.h" +#include "include/api/status.h" + +namespace mindspore { +namespace dataset { +// Create a numbered json file from image folder +Status DataHelper::CreateAlbumIF(const std::vector &in_dir, const std::vector &out_dir) { + auto jh = JsonHelper(); + return jh.CreateAlbum(CharToString(in_dir), CharToString(out_dir)); +} + +// A print method typically used for debugging +void DataHelper::Print(std::ostream &out) const { + out << " Data Helper" + << "\n"; +} + +Status DataHelper::UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector> &value, const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateArray(CharToString(in_file), CharToString(key), VectorCharToString(value), CharToString(out_file)); +} + +Status DataHelper::UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} + +Status DataHelper::UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} + +Status DataHelper::UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} + +Status DataHelper::UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} + +Status DataHelper::UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} + +Status DataHelper::UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} + +Status DataHelper::UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} + +Status DataHelper::UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} + +Status DataHelper::UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} + +Status DataHelper::UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} + +Status DataHelper::UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} + +Status DataHelper::UpdateValueIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateValue(CharToString(in_file), CharToString(key), CharToString(value), CharToString(out_file)); +} + +Status DataHelper::UpdateValueIF(const std::vector &in_file, const std::vector &key, const bool &value, + const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} + +Status DataHelper::UpdateValueIF(const std::vector &in_file, const std::vector &key, const int8_t &value, + const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} +Status DataHelper::UpdateValueIF(const std::vector &in_file, const std::vector &key, const uint8_t &value, + const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} +Status DataHelper::UpdateValueIF(const std::vector &in_file, const std::vector &key, const int16_t &value, + const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} +Status DataHelper::UpdateValueIF(const std::vector &in_file, const std::vector &key, const uint16_t &value, + const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} +Status DataHelper::UpdateValueIF(const std::vector &in_file, const std::vector &key, const int32_t &value, + const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} +Status DataHelper::UpdateValueIF(const std::vector &in_file, const std::vector &key, const uint32_t &value, + const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} +Status DataHelper::UpdateValueIF(const std::vector &in_file, const std::vector &key, const int64_t &value, + const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} +Status DataHelper::UpdateValueIF(const std::vector &in_file, const std::vector &key, const uint64_t &value, + const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} +Status DataHelper::UpdateValueIF(const std::vector &in_file, const std::vector &key, const float &value, + const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} + +Status DataHelper::UpdateValueIF(const std::vector &in_file, const std::vector &key, const double &value, + const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file)); +} + +Status DataHelper::RemoveKeyIF(const std::vector &in_file, const std::vector &key, + const std::vector &out_file) { + auto jh = JsonHelper(); + return jh.RemoveKey(CharToString(in_file), CharToString(key), CharToString(out_file)); +} + +size_t DataHelper::DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr, + const size_t &buffer_size) { + auto jh = JsonHelper(); + return jh.DumpData(tensor_addr, tensor_size, addr, buffer_size); +} + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/core/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/core/bindings.cc index 26b1a754cd0..cf47634bfbd 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/core/bindings.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/core/bindings.cc @@ -22,7 +22,7 @@ #include "minddata/dataset/core/client.h" // DE client #include "minddata/dataset/util/status.h" #include "pybind11/numpy.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/bindings.cc index 1ec749f1bb0..1854331eba0 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/bindings.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/bindings.cc @@ -20,7 +20,7 @@ #include "minddata/dataset/api/python/pybind_conversion.h" #include "minddata/dataset/api/python/pybind_register.h" #include "minddata/dataset/callback/py_ds_callback.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/serdes.h" #include "minddata/dataset/include/datasets.h" diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc index b0eab3bb2f9..4dab0a0f690 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc @@ -20,7 +20,7 @@ #include "minddata/dataset/api/python/pybind_conversion.h" #include "minddata/dataset/api/python/pybind_register.h" #include "minddata/dataset/callback/py_ds_callback.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/serdes.h" #include "minddata/dataset/include/datasets.h" diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/samplers/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/samplers/bindings.cc index 196ffe23d5c..d567815a4a3 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/samplers/bindings.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/samplers/bindings.cc @@ -21,7 +21,7 @@ #include "minddata/dataset/api/python/pybind_conversion.h" #include "minddata/dataset/api/python/pybind_register.h" #include "minddata/dataset/callback/py_ds_callback.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/ir/datasetops/source/samplers/samplers_ir.h" diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/schema/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/schema/bindings.cc index f56a7f38b0d..18691b9d3b8 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/schema/bindings.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/schema/bindings.cc @@ -19,7 +19,7 @@ #include "minddata/dataset/api/python/pybind_register.h" #include "minddata/dataset/core/global_context.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/api/python/pybind_conversion.h" #include "minddata/dataset/include/datasets.h" diff --git a/mindspore/ccsrc/minddata/dataset/core/client.h b/mindspore/ccsrc/minddata/dataset/core/client.h index 3de90cfeb26..7b88eb1b66d 100644 --- a/mindspore/ccsrc/minddata/dataset/core/client.h +++ b/mindspore/ccsrc/minddata/dataset/core/client.h @@ -19,7 +19,7 @@ // client.h // Include file for DE client functions -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/data_type.h" #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/core/tensor_shape.h" diff --git a/mindspore/ccsrc/minddata/dataset/core/config_manager.h b/mindspore/ccsrc/minddata/dataset/core/config_manager.h index 2c3231619d0..22d0d588757 100644 --- a/mindspore/ccsrc/minddata/dataset/core/config_manager.h +++ b/mindspore/ccsrc/minddata/dataset/core/config_manager.h @@ -22,7 +22,7 @@ #include -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/util/path.h" #include "minddata/dataset/util/status.h" @@ -148,7 +148,7 @@ class ConfigManager { void set_numa_enable(bool numa_enable); /// getter function - /// Now we want to seperate the numa link to _c_dataengine in the CMakeLists, + /// Now we want to separate the numa link to _c_dataengine in the CMakeLists, /// so we want user to choose whether to open numa switch. /// @return Get the current numa switch state. bool numa_enable() const { return numa_enable_; } diff --git a/mindspore/ccsrc/minddata/dataset/core/constants.h b/mindspore/ccsrc/minddata/dataset/core/constants.h deleted file mode 100644 index fc621e00d79..00000000000 --- a/mindspore/ccsrc/minddata/dataset/core/constants.h +++ /dev/null @@ -1,112 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_CONSTANTS_H_ -#define MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_CONSTANTS_H_ - -#include -#include -#include - -namespace mindspore { -namespace dataset { -// Various type defines for convenience -using uchar = unsigned char; -using dsize_t = int64_t; - -// Target devices to perform map operation -enum class MapTargetDevice { kCpu, kGpu, kDvpp }; - -// Possible dataset types for holding the data and client type -enum class DatasetType { kUnknown, kArrow, kTf }; - -// Possible flavours of Tensor implementations -enum class TensorImpl { kNone, kFlexible, kCv, kNP }; - -// Possible values for shuffle -enum class ShuffleMode { kFalse = 0, kFiles = 1, kGlobal = 2 }; - -// Possible values for Border types -enum class BorderType { kConstant = 0, kEdge = 1, kReflect = 2, kSymmetric = 3 }; - -// Possible values for Image format types in a batch -enum class ImageBatchFormat { kNHWC = 0, kNCHW = 1 }; - -// Possible values for Image format types -enum class ImageFormat { HWC = 0, CHW = 1, HW = 2 }; - -// Possible interpolation modes -enum class InterpolationMode { kLinear = 0, kNearestNeighbour = 1, kCubic = 2, kArea = 3 }; - -// Possible JiebaMode modes -enum class JiebaMode { kMix = 0, kMp = 1, kHmm = 2 }; - -// Possible values for SPieceTokenizerOutType -enum class SPieceTokenizerOutType { kString = 0, kInt = 1 }; - -// Possible values for SPieceTokenizerLoadType -enum class SPieceTokenizerLoadType { kFile = 0, kModel = 1 }; - -// Possible values for SentencePieceModel -enum class SentencePieceModel { kUnigram = 0, kBpe = 1, kChar = 2, kWord = 3 }; - -// Possible values for NormalizeForm -enum class NormalizeForm { - kNone = 0, - kNfc, - kNfkc, - kNfd, - kNfkd, -}; - -// Possible values for SamplingStrategy -enum class SamplingStrategy { kRandom = 0, kEdgeWeight = 1 }; - -// convenience functions for 32bit int bitmask -inline bool BitTest(uint32_t bits, uint32_t bitMask) { return (bits & bitMask) == bitMask; } - -inline void BitSet(uint32_t *bits, uint32_t bitMask) { *bits |= bitMask; } - -inline void BitClear(uint32_t *bits, uint32_t bitMask) { *bits &= (~bitMask); } - -constexpr int32_t kDeMaxDim = std::numeric_limits::max(); // 2147483647 or 2^32 -1 -constexpr int32_t kDeMaxRank = std::numeric_limits::max(); -constexpr int64_t kDeMaxFreq = std::numeric_limits::max(); // 9223372036854775807 or 2^(64-1) -constexpr int64_t kDeMaxTopk = std::numeric_limits::max(); - -constexpr uint32_t kCfgRowsPerBuffer = 1; -constexpr uint32_t kCfgParallelWorkers = 8; -constexpr uint32_t kCfgWorkerConnectorSize = 16; -constexpr uint32_t kCfgOpConnectorSize = 16; -constexpr int32_t kCfgDefaultRankId = -1; -constexpr uint32_t kCfgDefaultSeed = std::mt19937::default_seed; -constexpr uint32_t kCfgMonitorSamplingInterval = 10; -constexpr uint32_t kCfgCallbackTimeout = 60; // timeout value for callback in seconds -constexpr int32_t kCfgDefaultCachePort = 50052; -constexpr char kCfgDefaultCacheHost[] = "127.0.0.1"; -constexpr int32_t kDftPrefetchSize = 20; -constexpr int32_t kDftNumConnections = 12; -constexpr int32_t kDftAutoNumWorkers = false; - -// Invalid OpenCV type should not be from 0 to 7 (opencv4/opencv2/core/hal/interface.h) -constexpr uint8_t kCVInvalidType = 255; - -using connection_id_type = uint64_t; -using session_id_type = uint32_t; -using row_id_type = int64_t; -} // namespace dataset -} // namespace mindspore - -#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_CONSTANTS_H_ diff --git a/mindspore/ccsrc/minddata/dataset/core/cv_tensor.cc b/mindspore/ccsrc/minddata/dataset/core/cv_tensor.cc index 79c27d45cb2..bd9ff742fc3 100644 --- a/mindspore/ccsrc/minddata/dataset/core/cv_tensor.cc +++ b/mindspore/ccsrc/minddata/dataset/core/cv_tensor.cc @@ -18,7 +18,7 @@ #include #include -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/tensor.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/core/cv_tensor.h b/mindspore/ccsrc/minddata/dataset/core/cv_tensor.h index b67dd876838..988a20a9ae1 100644 --- a/mindspore/ccsrc/minddata/dataset/core/cv_tensor.h +++ b/mindspore/ccsrc/minddata/dataset/core/cv_tensor.h @@ -24,7 +24,7 @@ #include "./securec.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/data_type.h" #include "minddata/dataset/core/tensor.h" diff --git a/mindspore/ccsrc/minddata/dataset/core/data_type.h b/mindspore/ccsrc/minddata/dataset/core/data_type.h index 33aa1ab4523..8877bdbcddd 100644 --- a/mindspore/ccsrc/minddata/dataset/core/data_type.h +++ b/mindspore/ccsrc/minddata/dataset/core/data_type.h @@ -29,7 +29,7 @@ namespace py = pybind11; #else #include "base/float16.h" #endif -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/core/de_tensor.cc b/mindspore/ccsrc/minddata/dataset/core/de_tensor.cc index ad016a0656a..372c55d5676 100644 --- a/mindspore/ccsrc/minddata/dataset/core/de_tensor.cc +++ b/mindspore/ccsrc/minddata/dataset/core/de_tensor.cc @@ -16,7 +16,7 @@ #include "minddata/dataset/core/de_tensor.h" #include "minddata/dataset/core/device_tensor.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/data_type.h" #include "minddata/dataset/include/type_id.h" #include "mindspore/core/ir/dtype/type_id.h" diff --git a/mindspore/ccsrc/minddata/dataset/core/device_tensor.h b/mindspore/ccsrc/minddata/dataset/core/device_tensor.h index 10a1ba3973a..1880e532859 100644 --- a/mindspore/ccsrc/minddata/dataset/core/device_tensor.h +++ b/mindspore/ccsrc/minddata/dataset/core/device_tensor.h @@ -20,7 +20,7 @@ #include #include #include "include/api/status.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/data_type.h" #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/util/status.h" diff --git a/mindspore/ccsrc/minddata/dataset/core/global_context.h b/mindspore/ccsrc/minddata/dataset/core/global_context.h index 5cd8b71e6ef..793a5577d47 100644 --- a/mindspore/ccsrc/minddata/dataset/core/global_context.h +++ b/mindspore/ccsrc/minddata/dataset/core/global_context.h @@ -21,7 +21,7 @@ #include "include/api/status.h" #include "minddata/dataset/core/config_manager.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/util/allocator.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor.cc b/mindspore/ccsrc/minddata/dataset/core/tensor.cc index d35e87bfda9..24369e2c99e 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor.cc +++ b/mindspore/ccsrc/minddata/dataset/core/tensor.cc @@ -27,7 +27,7 @@ #include "minddata/dataset/include/type_id.h" #include "utils/ms_utils.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #ifndef ENABLE_ANDROID #include "minddata/dataset/core/cv_tensor.h" diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor.h b/mindspore/ccsrc/minddata/dataset/core/tensor.h index 220db2ddba3..a55f252bcdf 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor.h +++ b/mindspore/ccsrc/minddata/dataset/core/tensor.h @@ -37,7 +37,7 @@ #include "pybind11/stl.h" #endif -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/data_type.h" #include "minddata/dataset/core/tensor_helpers.h" #include "minddata/dataset/core/tensor_shape.h" diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor_helpers.h b/mindspore/ccsrc/minddata/dataset/core/tensor_helpers.h index e8c1d4fd0eb..a0383b7c65f 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor_helpers.h +++ b/mindspore/ccsrc/minddata/dataset/core/tensor_helpers.h @@ -19,7 +19,7 @@ #include #include -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc b/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc index d573486ee4b..16a24ce38e1 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc +++ b/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc @@ -25,7 +25,7 @@ #else #include "mindspore/lite/src/common/log_adapter.h" #endif -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor_shape.h b/mindspore/ccsrc/minddata/dataset/core/tensor_shape.h index 067641bed42..fda138e53a6 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor_shape.h +++ b/mindspore/ccsrc/minddata/dataset/core/tensor_shape.h @@ -31,7 +31,7 @@ namespace py = pybind11; #endif -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/util/status.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/util/allocator.h" diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc index 407716cd06b..f62b9761024 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc @@ -31,7 +31,7 @@ #include "minddata/dataset/engine/cache/cache_server.h" #include "minddata/dataset/engine/cache/cache_ipc.h" #include "minddata/dataset/util/path.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc index d0a664a4778..590609844e6 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc @@ -22,7 +22,7 @@ #include #include #include -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/engine/cache/cache_client.h" #include "minddata/dataset/engine/cache/cache_fbb.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc index 54159ca14f2..c4ed4bf83b2 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc @@ -18,7 +18,7 @@ #include #include #include -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/engine/cache/cache_ipc.h" #include "minddata/dataset/engine/cache/cache_service.h" #include "minddata/dataset/engine/cache/cache_request.h" diff --git a/mindspore/ccsrc/minddata/dataset/engine/data_buffer.h b/mindspore/ccsrc/minddata/dataset/engine/data_buffer.h index 5236446c12a..81dcbadd209 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/data_buffer.h +++ b/mindspore/ccsrc/minddata/dataset/engine/data_buffer.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ #include #include "minddata/dataset/util/allocator.h" #include "minddata/dataset/util/status.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/core/tensor_row.h" @@ -51,7 +51,7 @@ class DataBuffer { ~DataBuffer() = default; /// \brief A method for debug printing of the buffer - /// \param[inout] out The stream to write to + /// \param[in/out] out The stream to write to /// \param[in] show_all A boolean to toggle between details and summary printing void Print(std::ostream &out, bool show_all) const; diff --git a/mindspore/ccsrc/minddata/dataset/engine/data_schema.h b/mindspore/ccsrc/minddata/dataset/engine/data_schema.h index f34627b4a56..8167bfc3506 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/data_schema.h +++ b/mindspore/ccsrc/minddata/dataset/engine/data_schema.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ #include #include #include -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/data_type.h" #include "minddata/dataset/core/tensor_shape.h" #include "minddata/dataset/util/status.h" @@ -65,7 +65,7 @@ class ColDescriptor { /// If there is no starting TensorShape in this column, or if there is a shape but it contains /// an unknown dimension, then the output shape returned shall resolve dimensions as needed. /// \param[in] num_elements - The number of elements in the data for a Tensor - /// \param[inout] out_shape - The materialized output Tensor shape + /// \param[in/out] out_shape - The materialized output Tensor shape /// \return Status The status code returned Status MaterializeTensorShape(int32_t num_elements, TensorShape *out_shape) const; @@ -168,7 +168,7 @@ class DataSchema { static const char DEFAULT_DATA_SCHEMA_FILENAME[]; /// \brief Loops through all columns in the schema and returns a map with the column name to column index number. - /// \param[inout] out_column_name_map - The output map of columns names to column index + /// \param[in/out] out_column_name_map - The output map of columns names to column index /// \return Status The status code returned Status GetColumnNameMap(std::unordered_map *out_column_name_map); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc index 12f4f1595b5..7d92d569ce6 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc @@ -16,7 +16,7 @@ #include "minddata/dataset/engine/datasetops/barrier_op.h" #include #include -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/core/config_manager.h" diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc index cafe185af2b..53782e390bc 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc @@ -15,7 +15,7 @@ */ #include "minddata/dataset/engine/datasetops/cache_lookup_op.h" #include "minddata/dataset/core/config_manager.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/execution_tree.h" #ifndef ENABLE_ANDROID diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc index e40ac6dc9bf..a696e7aa5b0 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc @@ -21,7 +21,7 @@ #include #include #include "minddata/dataset/core/config_manager.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/util/system_pool.h" diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc index ebd00d153cd..77db6adc1b7 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc @@ -18,7 +18,7 @@ #include #include #include "minddata/dataset/core/config_manager.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/datasetops/repeat_op.h" #include "minddata/dataset/engine/data_buffer.h" diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h index 135e2da056c..2f3f44e6285 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h @@ -24,7 +24,7 @@ #include #include "minddata/dataset/callback/callback_manager.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/util/status.h" diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc index 18f69ff2370..c977016cf22 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc @@ -21,7 +21,7 @@ #include "minddata/dataset/callback/callback_param.h" #include "minddata/dataset/core/config_manager.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/datasetops/map_op/cpu_map_job.h" diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.h index b5dd73a174c..442b7e72066 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.h @@ -19,7 +19,7 @@ #include #include #include -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/engine/datasetops/dataset_op.h" #include "minddata/dataset/engine/datasetops/source/io_block.h" #include "minddata/dataset/util/status.h" diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc index 127015a4261..bde4bc9549f 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc @@ -20,7 +20,7 @@ #include #include "minddata/dataset/core/config_manager.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc index 42d9635a215..9c8f2e04ed6 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc @@ -23,7 +23,7 @@ #include "utils/ms_utils.h" #include "minddata/dataset/core/config_manager.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/datasetops/dataset_op.h" diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc index eb0da1d0638..498adcdcaeb 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc @@ -17,7 +17,7 @@ #include #include #include -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/core/config_manager.h" diff --git a/mindspore/ccsrc/minddata/dataset/engine/db_connector.h b/mindspore/ccsrc/minddata/dataset/engine/db_connector.h index c6647a798f7..d55d23d6e71 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/db_connector.h +++ b/mindspore/ccsrc/minddata/dataset/engine/db_connector.h @@ -20,7 +20,7 @@ #include #include "minddata/dataset/engine/connector.h" #include "minddata/dataset/engine/data_buffer.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/gpu_item_connector.h b/mindspore/ccsrc/minddata/dataset/engine/gpu_item_connector.h index 6571eff679a..97f740c3823 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/gpu_item_connector.h +++ b/mindspore/ccsrc/minddata/dataset/engine/gpu_item_connector.h @@ -23,7 +23,7 @@ #include #include "minddata/dataset/engine/connector.h" #include "minddata/dataset/util/status.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" #include "runtime/device/gpu/blocking_queue.h" using mindspore::device::DataItemGpu; diff --git a/mindspore/ccsrc/minddata/dataset/engine/jagged_connector.h b/mindspore/ccsrc/minddata/dataset/engine/jagged_connector.h index bfd543da56e..8871d0abf4d 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/jagged_connector.h +++ b/mindspore/ccsrc/minddata/dataset/engine/jagged_connector.h @@ -23,7 +23,7 @@ #include "minddata/dataset/engine/connector.h" #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/util/status.h" -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/perf/cyclic_array.h b/mindspore/ccsrc/minddata/dataset/engine/perf/cyclic_array.h index d0a30fd2b71..15ca8a51181 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/perf/cyclic_array.h +++ b/mindspore/ccsrc/minddata/dataset/engine/perf/cyclic_array.h @@ -21,7 +21,7 @@ #include #include #include -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/perf/perf_data.h b/mindspore/ccsrc/minddata/dataset/engine/perf/perf_data.h index b67ab5f4b08..4a82ca220fa 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/perf/perf_data.h +++ b/mindspore/ccsrc/minddata/dataset/engine/perf/perf_data.h @@ -18,7 +18,7 @@ #define MINDSPORE_CCSRC_MINDDATA_DATASET_PERF_DATA_H #include -#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/include/constants.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/include/constants.h b/mindspore/ccsrc/minddata/dataset/include/constants.h index 4f1607046d5..d2dfedac6e0 100644 --- a/mindspore/ccsrc/minddata/dataset/include/constants.h +++ b/mindspore/ccsrc/minddata/dataset/include/constants.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_CONSTANTS_H_ -#define MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_CONSTANTS_H_ +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONSTANTS_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONSTANTS_H_ #include #include @@ -87,9 +87,10 @@ constexpr int64_t kDeMaxFreq = std::numeric_limits::max(); // 92233720 constexpr int64_t kDeMaxTopk = std::numeric_limits::max(); constexpr uint32_t kCfgRowsPerBuffer = 1; -constexpr uint32_t kCfgParallelWorkers = 4; +constexpr uint32_t kCfgParallelWorkers = 8; constexpr uint32_t kCfgWorkerConnectorSize = 16; constexpr uint32_t kCfgOpConnectorSize = 16; +constexpr int32_t kCfgDefaultRankId = -1; constexpr uint32_t kCfgDefaultSeed = std::mt19937::default_seed; constexpr uint32_t kCfgMonitorSamplingInterval = 10; constexpr uint32_t kCfgCallbackTimeout = 60; // timeout value for callback in seconds @@ -97,6 +98,7 @@ constexpr int32_t kCfgDefaultCachePort = 50052; constexpr char kCfgDefaultCacheHost[] = "127.0.0.1"; constexpr int32_t kDftPrefetchSize = 20; constexpr int32_t kDftNumConnections = 12; +constexpr int32_t kDftAutoNumWorkers = false; // Invalid OpenCV type should not be from 0 to 7 (opencv4/opencv2/core/hal/interface.h) constexpr uint8_t kCVInvalidType = 255; @@ -107,4 +109,4 @@ using row_id_type = int64_t; } // namespace dataset } // namespace mindspore -#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_CONSTANTS_H_ +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONSTANTS_H_ diff --git a/mindspore/ccsrc/minddata/dataset/include/data_helper.h b/mindspore/ccsrc/minddata/dataset/include/data_helper.h new file mode 100644 index 00000000000..cecce91dc0c --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/include/data_helper.h @@ -0,0 +1,448 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATA_HELPER_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATA_HELPER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "include/api/dual_abi_helper.h" +#include "include/api/status.h" + +namespace mindspore { +namespace dataset { + +/// \brief Simple class to do data manipulation, contains helper function to update json files in dataset +class DataHelper { + public: + /// \brief constructor + DataHelper() {} + + /// \brief Destructor + ~DataHelper() = default; + + /// \brief Create an Album dataset while taking in a path to a image folder + /// Creates the output directory if doesn't exist + /// \param[in] in_dir Image folder directory that takes in images + /// \param[in] out_dir Directory containing output json files + Status CreateAlbum(const std::string &in_dir, const std::string &out_dir) { + return CreateAlbumIF(StringToChar(in_dir), StringToChar(out_dir)); + } + + /// \brief Update a json file field with a vector of string values + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value array to write to file + /// \param out_file Optional input for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector &value, + const std::string &out_file = "") { + return UpdateArrayIF(StringToChar(in_file), StringToChar(key), VectorStringToChar(value), StringToChar(out_file)); + } + + /// \brief Update a json file field with a vector of bool values + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value array to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector &value, + const std::string &out_file = "") { + return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with a vector of int8 values + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value array to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector &value, + const std::string &out_file = "") { + return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with a vector of uint8 values + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value array to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector &value, + const std::string &out_file = "") { + return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with a vector of int16 values + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value array to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector &value, + const std::string &out_file = "") { + return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with a vector of uint16 values + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value array to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector &value, + const std::string &out_file = "") { + return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with a vector of int32 values + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value array to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector &value, + const std::string &out_file = "") { + return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with a vector of uint32 values + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value array to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector &value, + const std::string &out_file = "") { + return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with a vector of int64 values + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value array to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector &value, + const std::string &out_file = "") { + return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with a vector of uint64 values + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value array to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector &value, + const std::string &out_file = "") { + return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with a vector of float values + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value array to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector &value, + const std::string &out_file = "") { + return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with a vector of double values + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value array to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector &value, + const std::string &out_file = "") { + return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with a string value + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateValue(const std::string &in_file, const std::string &key, const std::string &value, + const std::string &out_file = "") { + return UpdateValueIF(StringToChar(in_file), StringToChar(key), StringToChar(value), StringToChar(out_file)); + } + + /// \brief Update a json file field with a bool value + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateValue(const std::string &in_file, const std::string &key, const bool &value, + const std::string &out_file = "") { + return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with an int8 value + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateValue(const std::string &in_file, const std::string &key, const int8_t &value, + const std::string &out_file = "") { + return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with an uint8 value + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateValue(const std::string &in_file, const std::string &key, const uint8_t &value, + const std::string &out_file = "") { + return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with an int16 value + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateValue(const std::string &in_file, const std::string &key, const int16_t &value, + const std::string &out_file = "") { + return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with an uint16 value + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateValue(const std::string &in_file, const std::string &key, const uint16_t &value, + const std::string &out_file = "") { + return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with an int32 value + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateValue(const std::string &in_file, const std::string &key, const int32_t &value, + const std::string &out_file = "") { + return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with an uint32 value + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateValue(const std::string &in_file, const std::string &key, const uint32_t &value, + const std::string &out_file = "") { + return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with an int64 value + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateValue(const std::string &in_file, const std::string &key, const int64_t &value, + const std::string &out_file = "") { + return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with an uint64 value + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateValue(const std::string &in_file, const std::string &key, const uint64_t &value, + const std::string &out_file = "") { + return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with a float value + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateValue(const std::string &in_file, const std::string &key, const float &value, + const std::string &out_file = "") { + return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Update a json file field with a double value + /// \param in_file The input file name to read in + /// \param key Key of field to write to + /// \param value Value to write to file + /// \param out_file Optional parameter for output file path, will write to input file if not specified + /// \return Status The status code returned + Status UpdateValue(const std::string &in_file, const std::string &key, const double &value, + const std::string &out_file = "") { + return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file)); + } + + /// \brief Template function to write tensor to file + /// \param[in] in_file File to write to + /// \param[in] data Array of type T values + /// \return Status The status code returned + template + Status WriteBinFile(const std::string &in_file, const std::vector &data) { + try { + std::ofstream o(in_file, std::ios::binary | std::ios::out); + if (!o.is_open()) { + return Status(kMDUnexpectedError, "Error opening Bin file to write"); + } + size_t length = data.size(); + o.write(reinterpret_cast(&data[0]), std::streamsize(length * sizeof(T))); + o.close(); + } + // Catch any exception and convert to Status return code + catch (const std::exception &err) { + return Status(kMDUnexpectedError, "Write bin file failed "); + } + return Status::OK(); + } + + /// \brief Write pointer to bin, use pointer to avoid memcpy + /// \param[in] in_file File name to write to + /// \param[in] data Pointer to data + /// \param[in] length Length of values to write from pointer + /// \return Status The status code returned + template + Status WriteBinFile(const std::string &in_file, T *data, size_t length) { + try { + std::ofstream o(in_file, std::ios::binary | std::ios::out); + if (!o.is_open()) { + return Status(kMDUnexpectedError, "Error opening Bin file to write"); + } + o.write(reinterpret_cast(data), std::streamsize(length * sizeof(T))); + o.close(); + } + // Catch any exception and convert to Status return code + catch (const std::exception &err) { + return Status(kMDUnexpectedError, "Write bin file failed "); + } + return Status::OK(); + } + + /// \brief Helper function to copy content of a tensor to buffer + /// \note This function iterates over the tensor in bytes, since + /// \param[in] tensor_addr The memory held by a tensor + /// \param[in] tensor_size The amount of data in bytes in tensor_addr, e.g. tensor->SizeInBytes() + /// \param[out] addr The address to copy tensor data to + /// \param[in] buffer_size The buffer size of addr + /// \return The size of the tensor (bytes copied + size_t DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr, const size_t &buffer_size); + + /// \brief Helper function to delete key in json file + /// note This function will return okay even if key not found + /// \param[in] in_file Json file to remove key from + /// \param[in] key The key to remove + /// \return Status The status code returned + Status RemoveKey(const std::string &in_file, const std::string &key, const std::string &out_file = "") { + return RemoveKeyIF(StringToChar(in_file), StringToChar(key), StringToChar(out_file)); + } + + /// \brief A print method typically used for debugging + /// \param out - The output stream to write output to + void Print(std::ostream &out) const; + + /// \brief << Stream output operator overload + /// \notes This allows you to write the debug print info using stream operators + /// \param out Reference to the output stream being overloaded + /// \param ds Reference to the DataSchema to display + /// \return The output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const DataHelper &dh) { + dh.Print(out); + return out; + } + + private: + // Helper function for dual ABI support + Status CreateAlbumIF(const std::vector &in_dir, const std::vector &out_dir); + Status UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector> &value, const std::vector &out_file); + Status UpdateArrayIF(const std::vector &in_file, const std::vector &key, const std::vector &value, + const std::vector &out_file); + Status UpdateArrayIF(const std::vector &in_file, const std::vector &key, const std::vector &value, + const std::vector &out_file); + Status UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file); + Status UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file); + Status UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file); + Status UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file); + Status UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file); + Status UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file); + Status UpdateArrayIF(const std::vector &in_file, const std::vector &key, + const std::vector &value, const std::vector &out_file); + Status UpdateArrayIF(const std::vector &in_file, const std::vector &key, const std::vector &value, + const std::vector &out_file); + Status UpdateArrayIF(const std::vector &in_file, const std::vector &key, const std::vector &value, + const std::vector &out_file); + + Status UpdateValueIF(const std::vector &in_file, const std::vector &key, const std::vector &value, + const std::vector &out_file); + Status UpdateValueIF(const std::vector &in_file, const std::vector &key, const bool &value, + const std::vector &out_file); + Status UpdateValueIF(const std::vector &in_file, const std::vector &key, const int8_t &value, + const std::vector &out_file); + Status UpdateValueIF(const std::vector &in_file, const std::vector &key, const uint8_t &value, + const std::vector &out_file); + Status UpdateValueIF(const std::vector &in_file, const std::vector &key, const int16_t &value, + const std::vector &out_file); + Status UpdateValueIF(const std::vector &in_file, const std::vector &key, const uint16_t &value, + const std::vector &out_file); + Status UpdateValueIF(const std::vector &in_file, const std::vector &key, const int32_t &value, + const std::vector &out_file); + Status UpdateValueIF(const std::vector &in_file, const std::vector &key, const uint32_t &value, + const std::vector &out_file); + Status UpdateValueIF(const std::vector &in_file, const std::vector &key, const int64_t &value, + const std::vector &out_file); + Status UpdateValueIF(const std::vector &in_file, const std::vector &key, const uint64_t &value, + const std::vector &out_file); + Status UpdateValueIF(const std::vector &in_file, const std::vector &key, const float &value, + const std::vector &out_file); + Status UpdateValueIF(const std::vector &in_file, const std::vector &key, const double &value, + const std::vector &out_file); + Status RemoveKeyIF(const std::vector &in_file, const std::vector &key, const std::vector &out_file); +}; +} // namespace dataset +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATA_HELPER_H_ diff --git a/mindspore/ccsrc/minddata/dataset/include/datasets.h b/mindspore/ccsrc/minddata/dataset/include/datasets.h index e1e61e91736..53d38573a1a 100644 --- a/mindspore/ccsrc/minddata/dataset/include/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/include/datasets.h @@ -30,11 +30,10 @@ #include #include -#include - #include "include/api/dual_abi_helper.h" #include "include/api/types.h" #include "minddata/dataset/include/iterator.h" +#include "minddata/dataset/include/json_fwd.hpp" #include "minddata/dataset/include/samplers.h" #include "minddata/dataset/include/text.h" #include "minddata/dataset/include/type_id.h" diff --git a/mindspore/ccsrc/minddata/dataset/include/json_fwd.hpp b/mindspore/ccsrc/minddata/dataset/include/json_fwd.hpp new file mode 100644 index 00000000000..efb8d3e9282 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/include/json_fwd.hpp @@ -0,0 +1,74 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ +#define INCLUDE_NLOHMANN_JSON_FWD_HPP_ + +#include // int64_t, uint64_t +#include // map +#include // allocator +#include // string +#include // vector + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann { +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer; + +template