From 482a710836bd58f5e0ab91f9f3e5c2d9ecea9d58 Mon Sep 17 00:00:00 2001 From: YangLuo Date: Wed, 7 Jul 2021 18:02:09 +0800 Subject: [PATCH] Move optional_helper from abi_helper to dataset --- include/api/dual_abi_helper.h | 13 ------------- mindspore/ccsrc/minddata/dataset/api/text.cc | 10 +++++++--- .../dataset/engine/ir/cache/dataset_cache_impl.h | 9 +++++++-- .../minddata/dataset/include/dataset/datasets.h | 7 +++++-- .../ccsrc/minddata/dataset/include/dataset/text.h | 9 +++++++-- 5 files changed, 26 insertions(+), 22 deletions(-) diff --git a/include/api/dual_abi_helper.h b/include/api/dual_abi_helper.h index d2823abd9ce..0dd4ac23f6b 100644 --- a/include/api/dual_abi_helper.h +++ b/include/api/dual_abi_helper.h @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -32,18 +31,6 @@ inline std::vector StringToChar(const std::string &s) { return std::vector inline std::string CharToString(const std::vector &c) { return std::string(c.begin(), c.end()); } -inline std::optional> OptionalStringToChar(const std::optional &s) { - if (s == std::nullopt) return std::nullopt; - std::optional> ret = std::vector(s->begin(), s->end()); - return ret; -} - -inline std::optional OptionalCharToString(const std::optional> &c) { - if (c == std::nullopt) return std::nullopt; - std::optional ret = std::string(c->begin(), c->end()); - return ret; -} - inline std::pair, int32_t> PairStringToChar(const std::pair &s) { return std::pair, int32_t>(std::vector(s.first.begin(), s.first.end()), s.second); } diff --git a/mindspore/ccsrc/minddata/dataset/api/text.cc b/mindspore/ccsrc/minddata/dataset/api/text.cc index bee04fe44be..f1cb9007a74 100644 --- a/mindspore/ccsrc/minddata/dataset/api/text.cc +++ b/mindspore/ccsrc/minddata/dataset/api/text.cc @@ -212,9 +212,13 @@ Status JiebaTokenizer::ParserFile(const std::string &file_path, struct Lookup::Data { Data(const std::shared_ptr &vocab, const std::optional> &unknown_token, mindspore::DataType data_type) - : vocab_(vocab), - unknown_token_(OptionalCharToString(unknown_token)), - data_type_(dataset::MSTypeToDEType(static_cast(data_type))) {} + : vocab_(vocab), data_type_(dataset::MSTypeToDEType(static_cast(data_type))) { + if (unknown_token == std::nullopt) { + unknown_token_ = std::nullopt; + } else { + unknown_token_ = std::string(unknown_token->begin(), unknown_token->end()); + } + } std::shared_ptr vocab_; std::optional unknown_token_; dataset::DataType data_type_; diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache_impl.h b/mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache_impl.h index 287d026ef17..9c515372ed6 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache_impl.h +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache_impl.h @@ -49,10 +49,15 @@ class DatasetCacheImpl : public DatasetCache { : session_id_(id), cache_mem_sz_(mem_sz), spill_(spill), - hostname_(OptionalCharToString(hostname)), port_(std::move(port)), num_connections_(std::move(num_connections)), - prefetch_sz_(std::move(prefetch_sz)) {} + prefetch_sz_(std::move(prefetch_sz)) { + if (hostname == std::nullopt) { + hostname_ = std::nullopt; + } else { + hostname_ = std::string(hostname->begin(), hostname->end()); + } + } /// Method to initialize the DatasetCache by creating an instance of a CacheClient /// \return Status Error code diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h b/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h index 539a6994839..f6f44b32bb9 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h @@ -1726,8 +1726,11 @@ inline std::shared_ptr CreateDatasetCache(session_id_type id, uint std::optional port = std::nullopt, std::optional num_connections = std::nullopt, std::optional prefetch_sz = std::nullopt) { - return CreateDatasetCacheCharIF(id, mem_sz, spill, OptionalStringToChar(hostname), port, num_connections, - prefetch_sz); + std::optional> hostname_c = std::nullopt; + if (hostname != std::nullopt) { + hostname_c = std::vector(hostname->begin(), hostname->end()); + } + return CreateDatasetCacheCharIF(id, mem_sz, spill, hostname_c, port, num_connections, prefetch_sz); } /// \brief Function to create a ZipDataset. diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/text.h b/mindspore/ccsrc/minddata/dataset/include/dataset/text.h index 561c1f86ea1..a25c07ff846 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/text.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/text.h @@ -214,8 +214,13 @@ class Lookup final : public TensorTransform { /// \param[in] data_type mindspore::DataType of the tensor after lookup; must be numeric, including bool. /// (default=mindspore::DataType::kNumberTypeInt32). explicit Lookup(const std::shared_ptr &vocab, const std::optional &unknown_token = {}, - mindspore::DataType data_type = mindspore::DataType::kNumberTypeInt32) - : Lookup(vocab, OptionalStringToChar(unknown_token), data_type) {} + mindspore::DataType data_type = mindspore::DataType::kNumberTypeInt32) { + std::optional> unknown_token_c = std::nullopt; + if (unknown_token != std::nullopt) { + unknown_token_c = std::vector(unknown_token->begin(), unknown_token->end()); + } + new (this) Lookup(vocab, unknown_token_c, data_type); + } explicit Lookup(const std::shared_ptr &vocab, const std::optional> &unknown_token, mindspore::DataType data_type = mindspore::DataType::kNumberTypeInt32);