From 1f11ccce06c71bd5f79a9b86d5593a72af21bbca Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 14 Oct 2020 22:26:33 -0400 Subject: [PATCH] Removed log_adapter dependency for minddata lite Brute force fix? Added missing import Added removal of new cache functionality Fix batch Remove crc and cache optimizer Remove affine Remove cut mix batch Added securec.h Removed unused crc import Fixed minor arm64 compile issues for tensor ops Removed excess code from de_tensor Added vision import remove random device ops Removed commented line Added extra layer to switch adapter logic Added securec Added float fix --- .../ccsrc/minddata/dataset/api/de_tensor.cc | 20 +++------------ .../dataset/api/python/de_pipeline.cc | 2 +- .../minddata/dataset/core/config_manager.cc | 6 ++++- .../ccsrc/minddata/dataset/core/data_type.cc | 4 +++ .../ccsrc/minddata/dataset/core/tensor.h | 6 ++++- .../minddata/dataset/core/tensor_shape.cc | 4 +++ .../dataset/engine/cache/cache_grpc_server.cc | 6 ++++- .../minddata/dataset/engine/data_schema.cc | 4 +++ .../dataset/engine/datasetops/barrier_op.cc | 1 - .../engine/datasetops/cache_lookup_op.cc | 4 +++ .../dataset/engine/datasetops/cache_op.cc | 4 +++ .../dataset/engine/datasetops/dataset_op.cc | 6 +++++ .../dataset/engine/datasetops/dataset_op.h | 2 ++ .../engine/datasetops/epoch_ctrl_op.cc | 4 +++ .../dataset/engine/datasetops/filter_op.cc | 4 +++ .../engine/datasetops/map_op/map_op.cc | 4 +++ .../dataset/engine/datasetops/project_op.cc | 4 +++ .../dataset/engine/datasetops/rename_op.cc | 4 +++ .../dataset/engine/datasetops/repeat_op.cc | 4 +++ .../dataset/engine/datasetops/shuffle_op.cc | 4 +++ .../dataset/engine/datasetops/skip_op.cc | 4 +++ .../engine/datasetops/source/mindrecord_op.cc | 4 +++ .../dataset/engine/datasetops/zip_op.cc | 4 +++ .../minddata/dataset/engine/execution_tree.cc | 4 ++- .../dataset/engine/gnn/graph_shared_memory.cc | 4 +++ .../dataset/engine/gnn/grpc_async_server.cc | 4 +++ .../minddata/dataset/engine/perf/profiling.cc | 4 +++ .../minddata/dataset/engine/tdt/tdt_plugin.cc | 4 +++ .../ccsrc/minddata/dataset/include/tensor.h | 4 +++ .../dataset/kernels/image/affine_op.cc | 13 +++++----- .../dataset/kernels/image/cutmix_batch_op.cc | 2 +- .../ccsrc/minddata/dataset/text/vocab.cc | 4 +++ .../ccsrc/minddata/dataset/util/arena.cc | 2 +- .../minddata/dataset/util/btree_iterator.tpp | 3 ++- .../ccsrc/minddata/dataset/util/buddy.cc | 3 ++- .../minddata/dataset/util/circular_pool.cc | 2 +- .../minddata/dataset/util/data_helper.cc | 4 +-- .../ccsrc/minddata/dataset/util/data_helper.h | 2 +- .../minddata/dataset/util/intrp_service.h | 3 +-- mindspore/ccsrc/minddata/dataset/util/list.h | 2 +- mindspore/ccsrc/minddata/dataset/util/lock.cc | 2 +- .../ccsrc/minddata/dataset/util/log_adapter.h | 25 +++++++++++++++++++ mindspore/ccsrc/minddata/dataset/util/path.cc | 3 ++- mindspore/ccsrc/minddata/dataset/util/queue.h | 3 ++- .../ccsrc/minddata/dataset/util/random.h | 2 +- .../ccsrc/minddata/dataset/util/services.cc | 1 + .../dataset/util/storage_container.cc | 2 +- .../minddata/dataset/util/storage_manager.cc | 2 +- mindspore/ccsrc/minddata/dataset/util/task.cc | 2 +- mindspore/ccsrc/minddata/dataset/util/task.h | 2 +- mindspore/lite/minddata/CMakeLists.txt | 15 ++++++++--- .../lite/test/ut/src/dataset/eager_test.cc | 2 ++ 52 files changed, 182 insertions(+), 52 deletions(-) create mode 100644 mindspore/ccsrc/minddata/dataset/util/log_adapter.h diff --git a/mindspore/ccsrc/minddata/dataset/api/de_tensor.cc b/mindspore/ccsrc/minddata/dataset/api/de_tensor.cc index 190cc4efe98..0db2f1fcec3 100644 --- a/mindspore/ccsrc/minddata/dataset/api/de_tensor.cc +++ b/mindspore/ccsrc/minddata/dataset/api/de_tensor.cc @@ -14,14 +14,13 @@ * limitations under the License. */ -#include "minddata/dataset/include/de_tensor.h" -#include "minddata/dataset/include/type_id.h" #include "minddata/dataset/core/constants.h" #include "minddata/dataset/core/data_type.h" +#include "minddata/dataset/include/de_tensor.h" +#include "minddata/dataset/include/type_id.h" #include "mindspore/core/ir/dtype/type_id.h" -#include "utils/hashing.h" #include "mindspore/lite/internal/include/ms_tensor.h" -#include "mindspore/core/utils/convert_utils_base.h" +#include "utils/hashing.h" namespace mindspore { namespace tensor { @@ -124,19 +123,6 @@ int DETensor::ElementsNum() const { return this->tensor_impl_->Size(); } -std::size_t DETensor::hash() const { - MS_ASSERT(this->tensor_impl_ != nullptr); - auto shape = this->shape(); - std::size_t hash_value = std::hash{}(SizeToInt(this->data_type())); - hash_value = hash_combine(hash_value, std::hash{}(shape.size())); - // hash all elements may costly, so only take at most 4 elements into account based on - // some experiments. - for (size_t i = 0; (i < shape.size()) && (i < 4); ++i) { - hash_value = hash_combine(hash_value, (std::hash{}(shape[i]))); - } - return hash_value; -} - size_t DETensor::Size() const { MS_ASSERT(this->tensor_impl_ != nullptr); return this->tensor_impl_->SizeInBytes(); diff --git a/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc b/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc index 68cba69db3b..45d65288a23 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc @@ -41,6 +41,7 @@ #include "minddata/dataset/engine/datasetops/source/text_file_op.h" #include "minddata/dataset/engine/datasetops/source/voc_op.h" #include "minddata/dataset/kernels/py_func_op.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/random.h" #include "minddata/dataset/util/status.h" #include "minddata/mindrecord/include/shard_category.h" @@ -51,7 +52,6 @@ #include "minddata/mindrecord/include/shard_shuffle.h" #include "minddata/mindrecord/include/shard_writer.h" #include "pybind11/stl.h" -#include "utils/log_adapter.h" #include "utils/ms_utils.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/core/config_manager.cc b/mindspore/ccsrc/minddata/dataset/core/config_manager.cc index 44901b81199..aa03200cdc7 100644 --- a/mindspore/ccsrc/minddata/dataset/core/config_manager.cc +++ b/mindspore/ccsrc/minddata/dataset/core/config_manager.cc @@ -20,7 +20,11 @@ #include #include -#include "mindspore/core/utils/log_adapter.h" +#ifndef ENABLE_ANDROID +#include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif #include "minddata/dataset/util/system_pool.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/core/data_type.cc b/mindspore/ccsrc/minddata/dataset/core/data_type.cc index b5641e3105c..77cf2f45a58 100644 --- a/mindspore/ccsrc/minddata/dataset/core/data_type.cc +++ b/mindspore/ccsrc/minddata/dataset/core/data_type.cc @@ -18,7 +18,11 @@ #include "minddata/dataset/core/pybind_support.h" #endif +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor.h b/mindspore/ccsrc/minddata/dataset/core/tensor.h index 27b1a815fb6..349f45a8f3b 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor.h +++ b/mindspore/ccsrc/minddata/dataset/core/tensor.h @@ -21,7 +21,11 @@ #include #include #include "./securec.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif #if defined(_WIN32) || defined(_WIN64) #undef HAVE_STDDEF_H #undef HAVE_STDLIB_H @@ -33,12 +37,12 @@ #include "pybind11/stl.h" #endif -#include "utils/ms_utils.h" #include "minddata/dataset/core/constants.h" #include "minddata/dataset/core/data_type.h" #include "minddata/dataset/core/tensor_helpers.h" #include "minddata/dataset/core/tensor_shape.h" #include "minddata/dataset/util/status.h" +#include "utils/ms_utils.h" #ifndef ENABLE_ANDROID #include "proto/example.pb.h" #else diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc b/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc index 51fa874ea75..6a0fd22c338 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc +++ b/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc @@ -20,7 +20,11 @@ #include #include "utils/ms_utils.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif #include "minddata/dataset/core/constants.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_server.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_server.cc index 485d1a38060..892880f0ea6 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_server.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_server.cc @@ -13,11 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "minddata/dataset/engine/cache/cache_grpc_server.h" #include +#include "minddata/dataset/engine/cache/cache_grpc_server.h" #include "minddata/dataset/engine/cache/cache_server.h" #include "minddata/dataset/util/path.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { CacheServerGreeterImpl::CacheServerGreeterImpl(int32_t port, int32_t shared_memory_sz_in_gb) diff --git a/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc b/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc index 6db8ec5614b..bffb87f0c60 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc @@ -26,7 +26,11 @@ #include "utils/ms_utils.h" #include "minddata/dataset/util/status.h" #include "minddata/dataset/core/tensor_shape.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc index f77d39e3e5a..f240c9bc4c7 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc @@ -21,7 +21,6 @@ #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/global_context.h" -#include "utils/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc index 0e38ad63333..1cdaec76f80 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc @@ -19,8 +19,12 @@ #include "minddata/dataset/core/constants.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/execution_tree.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" #include "utils/system/crc32c.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc index 652d78bfbc6..04abeb989ee 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc @@ -26,7 +26,11 @@ #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/util/task_manager.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc index a24b4395a9c..9e59236d019 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc @@ -30,8 +30,12 @@ #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/opt/pass.h" +#ifndef ENABLE_ANDROID #include "utils/system/crc32c.h" #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { @@ -385,6 +389,7 @@ Status DatasetOp::FetchRemoveSampler(std::shared_ptr *sampler) { return Status::OK(); } +#ifndef ENABLE_ANDROID uint32_t DatasetOp::GenerateCRC(const std::shared_ptr &op) { std::stringstream ss; op->tree_->Print(ss, op); @@ -431,6 +436,7 @@ uint32_t DatasetOp::GenerateCRC(const std::shared_ptr &op) { uint32_t cache_crc = system::Crc32c::GetMaskCrc32cValue(ss_str.c_str(), ss_str.length()); return cache_crc; } +#endif void DatasetOp::UpdateRepeatAndEpochCounter() { op_current_repeats_++; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h index f99c0603d6a..00e2200f4cb 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h @@ -338,8 +338,10 @@ class DatasetOp : public std::enable_shared_from_this { /// \return Status error code Status FetchRemoveSampler(std::shared_ptr *sampler); +#ifndef ENABLE_ANDROID // Computes a CRC value for the operator static uint32_t GenerateCRC(const std::shared_ptr &op); +#endif /// \brief A helper templated function for casting "this" pointer to shared_ptr /// Similar to shared_from_this, except this one will give you the derived class as shared_ptr diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc index 1343fd46086..b8c94a01c2c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc @@ -22,7 +22,11 @@ #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/opt/pass.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc index 446cd80299a..d9eb0640412 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc @@ -29,7 +29,11 @@ #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/kernels/tensor_op.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif #include "minddata/dataset/util/task_manager.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc index f8c628647ff..0fa1efec4b0 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc @@ -31,7 +31,11 @@ #include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/kernels/tensor_op.h" #include "minddata/dataset/util/task_manager.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc index 01fcab877cc..b88578b32fa 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc @@ -26,7 +26,11 @@ #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/engine/opt/pass.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc index deb4027436b..b3f73b37b77 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc @@ -25,7 +25,11 @@ #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/opt/pass.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc index 0a8591f5b13..557698a3969 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc @@ -23,7 +23,11 @@ #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/opt/pass.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc index 282f57b8a9a..1259cfcb1cd 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc @@ -34,7 +34,11 @@ #include "minddata/dataset/util/random.h" #include "minddata/dataset/util/status.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc index 012e23b46a1..c85af73e2ca 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc @@ -24,7 +24,11 @@ #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/engine/opt/pass.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc index 6ba6a132f44..f9ddfb68de5 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc @@ -30,7 +30,11 @@ #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/engine/opt/pass.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc index 3b1fec02e0b..0ab8406153e 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc @@ -22,7 +22,11 @@ #include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/global_context.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc b/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc index 8a435749976..42660b45997 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc @@ -25,8 +25,8 @@ #ifndef ENABLE_ANDROID #include "minddata/dataset/engine/opt/pre/cache_transform_pass.h" #include "minddata/dataset/engine/opt/post/repeat_pass.h" -#endif #include "minddata/dataset/engine/opt/pre/cache_error_pass.h" +#endif #include "minddata/dataset/engine/opt/pre/epoch_injection_pass.h" #include "mindspore/ccsrc/minddata/dataset/engine/opt/optional/tensor_op_fusion_pass.h" #include "minddata/dataset/engine/perf/profiling.h" @@ -236,7 +236,9 @@ Status ExecutionTree::PrepareTreePreAction() { std::vector> pre_actions; // Construct pre actions MS_LOG(INFO) << "Running pre pass loops."; +#ifndef ENABLE_ANDROID pre_actions.push_back(std::make_unique()); +#endif pre_actions.push_back(std::make_unique()); pre_actions.push_back(std::make_unique()); #ifndef ENABLE_ANDROID diff --git a/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_shared_memory.cc b/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_shared_memory.cc index 54e6eda0d28..019f1513fa9 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_shared_memory.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_shared_memory.cc @@ -18,7 +18,11 @@ #include +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/gnn/grpc_async_server.cc b/mindspore/ccsrc/minddata/dataset/engine/gnn/grpc_async_server.cc index 02c967e442f..2202f2f75b3 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/gnn/grpc_async_server.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/gnn/grpc_async_server.cc @@ -18,7 +18,11 @@ #include #include "minddata/dataset/util/task_manager.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/perf/profiling.cc b/mindspore/ccsrc/minddata/dataset/engine/perf/profiling.cc index 82a4b6818c5..f3d4a39acb2 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/perf/profiling.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/perf/profiling.cc @@ -24,7 +24,11 @@ #include "minddata/dataset/engine/perf/connector_size.h" #include "minddata/dataset/engine/perf/connector_throughput.h" #include "minddata/dataset/engine/perf/dataset_iterator_tracing.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.cc b/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.cc index dd57fa7ea49..5f4834c80fc 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.cc @@ -15,7 +15,11 @@ */ #include "minddata/dataset/engine/tdt/tdt_plugin.h" #include "utils/ms_utils.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif #include "minddata/dataset/engine/perf/profiling.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/include/tensor.h b/mindspore/ccsrc/minddata/dataset/include/tensor.h index b2fe352c1df..6d43f18d31e 100644 --- a/mindspore/ccsrc/minddata/dataset/include/tensor.h +++ b/mindspore/ccsrc/minddata/dataset/include/tensor.h @@ -21,7 +21,11 @@ #include #include #include "./securec.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif #if defined(_WIN32) || defined(_WIN64) #undef HAVE_STDDEF_H #undef HAVE_STDLIB_H diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/affine_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/affine_op.cc index 1ea1ea434c3..90403abfc91 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/affine_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/affine_op.cc @@ -74,12 +74,13 @@ Status AffineOp::Compute(const std::shared_ptr &input, std::shared_ptrmat().cols - 1) / 2.0); float_t cy = ((input_cv->mat().rows - 1) / 2.0); // Calculate RSS - std::vector matrix{scale_ * cos(degrees + shear_y) / cos(shear_y), - scale_ * (-1 * cos(degrees + shear_y) * tan(shear_x) / cos(shear_y) - sin(degrees)), - 0, - scale_ * sin(degrees + shear_y) / cos(shear_y), - scale_ * (-1 * sin(degrees + shear_y) * tan(shear_x) / cos(shear_y) + cos(degrees)), - 0}; + std::vector matrix{ + static_cast(scale_ * cos(degrees + shear_y) / cos(shear_y)), + static_cast(scale_ * (-1 * cos(degrees + shear_y) * tan(shear_x) / cos(shear_y) - sin(degrees))), + 0, + static_cast(scale_ * sin(degrees + shear_y) / cos(shear_y)), + static_cast(scale_ * (-1 * sin(degrees + shear_y) * tan(shear_x) / cos(shear_y) + cos(degrees))), + 0}; // Compute T * C * RSS * C^-1 matrix[2] = (1 - matrix[0]) * cx - matrix[1] * cy + translation_x; matrix[5] = (1 - matrix[4]) * cy - matrix[3] * cx + translation_y; diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/cutmix_batch_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/cutmix_batch_op.cc index d4166545995..6480faa41f9 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/cutmix_batch_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/cutmix_batch_op.cc @@ -182,7 +182,7 @@ Status CutMixBatchOp::Compute(const TensorRow &input, TensorRow *output) { void CutMixBatchOp::Print(std::ostream &out) const { out << "CutMixBatchOp: " - << "image_batch_format: " << image_batch_format_ << "alpha: " << alpha_ << ", probability: " << prob_ << "\n"; + << "\n"; } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/text/vocab.cc b/mindspore/ccsrc/minddata/dataset/text/vocab.cc index 3ef1c399dd3..b0f7349a712 100644 --- a/mindspore/ccsrc/minddata/dataset/text/vocab.cc +++ b/mindspore/ccsrc/minddata/dataset/text/vocab.cc @@ -20,7 +20,11 @@ #include #include "minddata/dataset/text/vocab.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/arena.cc b/mindspore/ccsrc/minddata/dataset/util/arena.cc index 6bd6d295ffb..de563cf1dbc 100644 --- a/mindspore/ccsrc/minddata/dataset/util/arena.cc +++ b/mindspore/ccsrc/minddata/dataset/util/arena.cc @@ -16,9 +16,9 @@ #include "minddata/dataset/util/arena.h" #include #include +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/system_pool.h" #include "./securec.h" -#include "utils/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/btree_iterator.tpp b/mindspore/ccsrc/minddata/dataset/util/btree_iterator.tpp index 3e72013911b..890b1890961 100644 --- a/mindspore/ccsrc/minddata/dataset/util/btree_iterator.tpp +++ b/mindspore/ccsrc/minddata/dataset/util/btree_iterator.tpp @@ -15,7 +15,8 @@ #ifndef DATASET_UTIL_BTREE_ITERATOR_H_ #define DATASET_UTIL_BTREE_ITERATOR_H_ -#include "utils/log_adapter.h" +#include "./securec.h" +#include "minddata/dataset/util/log_adapter.h" #include "btree.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/util/buddy.cc b/mindspore/ccsrc/minddata/dataset/util/buddy.cc index 0cc2baed5f2..bbeeaff0db0 100644 --- a/mindspore/ccsrc/minddata/dataset/util/buddy.cc +++ b/mindspore/ccsrc/minddata/dataset/util/buddy.cc @@ -16,9 +16,10 @@ #include "minddata/dataset/util/buddy.h" #include #include + #include "minddata/dataset/util/memory_pool.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/system_pool.h" -#include "utils/log_adapter.h" #include "./securec.h" inline uint64_t BitLeftShift(uint64_t v, uint64_t n) { return (v << n); } diff --git a/mindspore/ccsrc/minddata/dataset/util/circular_pool.cc b/mindspore/ccsrc/minddata/dataset/util/circular_pool.cc index 5f57d596eb0..539c39cc77e 100644 --- a/mindspore/ccsrc/minddata/dataset/util/circular_pool.cc +++ b/mindspore/ccsrc/minddata/dataset/util/circular_pool.cc @@ -19,8 +19,8 @@ #include #include #include "./securec.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/system_pool.h" -#include "utils/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/data_helper.cc b/mindspore/ccsrc/minddata/dataset/util/data_helper.cc index ac821c2214e..49c70f884b4 100644 --- a/mindspore/ccsrc/minddata/dataset/util/data_helper.cc +++ b/mindspore/ccsrc/minddata/dataset/util/data_helper.cc @@ -23,11 +23,11 @@ #include #include -#include "minddata/dataset/util/status.h" #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/core/tensor_shape.h" -#include "utils/log_adapter.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/path.h" +#include "minddata/dataset/util/status.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/data_helper.h b/mindspore/ccsrc/minddata/dataset/util/data_helper.h index 19a0195958d..bf2e7f10a41 100644 --- a/mindspore/ccsrc/minddata/dataset/util/data_helper.h +++ b/mindspore/ccsrc/minddata/dataset/util/data_helper.h @@ -30,7 +30,7 @@ #include "minddata/dataset/core/data_type.h" #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/core/tensor_shape.h" -#include "utils/log_adapter.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/path.h" #include "minddata/dataset/util/status.h" diff --git a/mindspore/ccsrc/minddata/dataset/util/intrp_service.h b/mindspore/ccsrc/minddata/dataset/util/intrp_service.h index 2aa6987e6bd..13c8b43673b 100644 --- a/mindspore/ccsrc/minddata/dataset/util/intrp_service.h +++ b/mindspore/ccsrc/minddata/dataset/util/intrp_service.h @@ -23,12 +23,11 @@ #include #include "minddata/dataset/util/allocator.h" #include "minddata/dataset/util/intrp_resource.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/service.h" #include "minddata/dataset/util/services.h" #include "minddata/dataset/util/status.h" -#include "utils/log_adapter.h" - namespace mindspore { namespace dataset { using SvcAllocator = Allocator>; diff --git a/mindspore/ccsrc/minddata/dataset/util/list.h b/mindspore/ccsrc/minddata/dataset/util/list.h index b7eb8ebd014..06c1d069f5f 100644 --- a/mindspore/ccsrc/minddata/dataset/util/list.h +++ b/mindspore/ccsrc/minddata/dataset/util/list.h @@ -19,7 +19,7 @@ #include #include -#include "utils/log_adapter.h" +#include "minddata/dataset/util/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/lock.cc b/mindspore/ccsrc/minddata/dataset/util/lock.cc index 5302196a46c..fe4a9c2192b 100644 --- a/mindspore/ccsrc/minddata/dataset/util/lock.cc +++ b/mindspore/ccsrc/minddata/dataset/util/lock.cc @@ -14,7 +14,7 @@ * limitations under the License. */ #include "minddata/dataset/util/lock.h" -#include "utils/log_adapter.h" +#include "minddata/dataset/util/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/log_adapter.h b/mindspore/ccsrc/minddata/dataset/util/log_adapter.h new file mode 100644 index 00000000000..38dfafd3dcd --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/util/log_adapter.h @@ -0,0 +1,25 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_LOG_ADAPTER_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_LOG_ADAPTER_H_ + +#ifndef ENABLE_ANDROID +#include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif + +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_LOG_ADAPTER_H_ diff --git a/mindspore/ccsrc/minddata/dataset/util/path.cc b/mindspore/ccsrc/minddata/dataset/util/path.cc index 97416be6a80..df61ccda279 100644 --- a/mindspore/ccsrc/minddata/dataset/util/path.cc +++ b/mindspore/ccsrc/minddata/dataset/util/path.cc @@ -22,8 +22,9 @@ #include #include +#include "./securec.h" #include "utils/ms_utils.h" -#include "utils/log_adapter.h" +#include "minddata/dataset/util/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/queue.h b/mindspore/ccsrc/minddata/dataset/util/queue.h index 417a1ab507c..2f2e0c70066 100644 --- a/mindspore/ccsrc/minddata/dataset/util/queue.h +++ b/mindspore/ccsrc/minddata/dataset/util/queue.h @@ -24,9 +24,10 @@ #include #include +#include "./securec.h" #include "utils/ms_utils.h" -#include "utils/log_adapter.h" #include "minddata/dataset/util/allocator.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/services.h" #include "minddata/dataset/util/cond_var.h" #include "minddata/dataset/util/task_manager.h" diff --git a/mindspore/ccsrc/minddata/dataset/util/random.h b/mindspore/ccsrc/minddata/dataset/util/random.h index 7e8dd48e880..4d4e0b0d32d 100644 --- a/mindspore/ccsrc/minddata/dataset/util/random.h +++ b/mindspore/ccsrc/minddata/dataset/util/random.h @@ -28,7 +28,7 @@ #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/global_context.h" -#include "utils/log_adapter.h" +#include "minddata/dataset/util/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/services.cc b/mindspore/ccsrc/minddata/dataset/util/services.cc index 7958aad2242..c6e0f38d24f 100644 --- a/mindspore/ccsrc/minddata/dataset/util/services.cc +++ b/mindspore/ccsrc/minddata/dataset/util/services.cc @@ -22,6 +22,7 @@ #include #endif #include +#include "./securec.h" #include "minddata/dataset/util/circular_pool.h" #include "minddata/dataset/util/random.h" #include "minddata/dataset/util/task_manager.h" diff --git a/mindspore/ccsrc/minddata/dataset/util/storage_container.cc b/mindspore/ccsrc/minddata/dataset/util/storage_container.cc index b0855512cdc..cca10b40c46 100644 --- a/mindspore/ccsrc/minddata/dataset/util/storage_container.cc +++ b/mindspore/ccsrc/minddata/dataset/util/storage_container.cc @@ -20,9 +20,9 @@ #include #include #include "utils/ms_utils.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/path.h" #include "minddata/dataset/util/status.h" -#include "utils/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/storage_manager.cc b/mindspore/ccsrc/minddata/dataset/util/storage_manager.cc index 8c99e5a91e9..bb70198fd1e 100644 --- a/mindspore/ccsrc/minddata/dataset/util/storage_manager.cc +++ b/mindspore/ccsrc/minddata/dataset/util/storage_manager.cc @@ -20,9 +20,9 @@ #include #include #include "utils/ms_utils.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/path.h" #include "minddata/dataset/util/services.h" -#include "utils/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/task.cc b/mindspore/ccsrc/minddata/dataset/util/task.cc index 9cbaa7074bc..77336e56aea 100644 --- a/mindspore/ccsrc/minddata/dataset/util/task.cc +++ b/mindspore/ccsrc/minddata/dataset/util/task.cc @@ -15,8 +15,8 @@ */ #include "minddata/dataset/util/task.h" #include "utils/ms_utils.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/task_manager.h" -#include "utils/log_adapter.h" #if defined(__ANDROID__) || defined(ANDROID) #include "minddata/dataset/util/services.h" #endif diff --git a/mindspore/ccsrc/minddata/dataset/util/task.h b/mindspore/ccsrc/minddata/dataset/util/task.h index 951beb9a4c0..b41861ecc0e 100644 --- a/mindspore/ccsrc/minddata/dataset/util/task.h +++ b/mindspore/ccsrc/minddata/dataset/util/task.h @@ -29,10 +29,10 @@ #include #include "minddata/dataset/util/intrp_resource.h" #include "minddata/dataset/util/list.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/memory_pool.h" #include "minddata/dataset/util/services.h" #include "minddata/dataset/util/wait_post.h" -#include "utils/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/lite/minddata/CMakeLists.txt b/mindspore/lite/minddata/CMakeLists.txt index fe767f665ef..dbc0d1c158c 100644 --- a/mindspore/lite/minddata/CMakeLists.txt +++ b/mindspore/lite/minddata/CMakeLists.txt @@ -118,8 +118,15 @@ if (BUILD_MINDDATA STREQUAL "full") list(REMOVE_ITEM MINDDATA_ENGINE_OPT_PRE_SRC_FILES "${MINDDATA_DIR}/engine/opt/pre/cache_transform_pass.cc" + "${MINDDATA_DIR}/engine/opt/pre/cache_error_pass.cc" ) + #list(REMOVE_ITEM MINDDATA_KERNELS_IMAGE_SRC_FILES + #"${MINDDATA_DIR}/kernels/image/affine_op.cc" + #"${MINDDATA_DIR}/kernels/image/cutmix_batch_op.cc" + #"${MINDDATA_DIR}/kernels/image/random_affine_op.cc" + #) + include_directories("${CMAKE_BINARY_DIR}/minddata/dataset/engine/cache") if (BUILD_MINDDATA_EXAMPLE AND (PLATFORM_ARM32 OR PLATFORM_ARM64)) @@ -149,8 +156,6 @@ if (BUILD_MINDDATA STREQUAL "full") ${MINDDATA_EXAMPLE_SRC} ${CMAKE_CURRENT_SOURCE_DIR}/../src/common/log_adapter.cc ${CORE_DIR}/utils/ms_utils.cc - ${CORE_DIR}/gvar/logging_level.cc - ${CCSRC_DIR}/utils/system/crc32c.cc ) target_link_libraries(minddata-lite @@ -189,6 +194,10 @@ elseif (BUILD_MINDDATA STREQUAL "lite") ${MINDDATA_DIR}/api/execute.cc ) + list(REMOVE_ITEM MINDDATA_KERNELS_IMAGE_SRC_FILES + "${MINDDATA_DIR}/kernels/image/bounding_box_augment_op.cc" + "${MINDDATA_DIR}/kernels/image/cut_out_op.cc" + ) add_library(minddata-lite SHARED ${MINDDATA_CORE_SRC_FILES} ${MINDDATA_KERNELS_SRC_FILES} @@ -198,8 +207,6 @@ elseif (BUILD_MINDDATA STREQUAL "lite") ${MINDDATA_DIR}/util/memory_pool.cc ${MINDDATA_DIR}/util/path.cc ${MINDDATA_DIR}/api/transforms.cc - ${CORE_DIR}/utils/log_adapter.cc - ${CORE_DIR}/gvar/logging_level.cc ) target_link_libraries(minddata-lite diff --git a/mindspore/lite/test/ut/src/dataset/eager_test.cc b/mindspore/lite/test/ut/src/dataset/eager_test.cc index cf3d02a040a..04412678f2c 100644 --- a/mindspore/lite/test/ut/src/dataset/eager_test.cc +++ b/mindspore/lite/test/ut/src/dataset/eager_test.cc @@ -19,6 +19,8 @@ #include "./securec.h" #include "minddata/dataset/include/tensor.h" #include "minddata/dataset/include/datasets.h" +#include "minddata/dataset/include/transforms.h" +#include "minddata/dataset/include/vision.h" #include "minddata/dataset/include/execute.h" #include "minddata/dataset/util/path.h"