Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Move optional_helper from abi_helper to dataset

This commit is contained in:
YangLuo 2021-07-07 18:29:45 +08:00
parent 3d04d0362b
commit f1f8e6d855
5 changed files with 26 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
include/api/dual_abi_helper.h
View File

@ -20,7 +20,6 @@
#include <iterator>
#include <map>
#include <memory>
#include <optional>
#include <string>
#include <set>
#include <unordered_map>
@ -32,18 +31,6 @@ inline std::vector<char> StringToChar(const std::string &s) { return std::vector
inline std::string CharToString(const std::vector<char> &c) { return std::string(c.begin(), c.end()); }
inline std::optional<std::vector<char>> OptionalStringToChar(const std::optional<std::string> &s) {
if (s == std::nullopt) return std::nullopt;
std::optional<std::vector<char>> ret = std::vector<char>(s->begin(), s->end());
return ret;
}
inline std::optional<std::string> OptionalCharToString(const std::optional<std::vector<char>> &c) {
if (c == std::nullopt) return std::nullopt;
std::optional<std::string> ret = std::string(c->begin(), c->end());
return ret;
}
inline std::pair<std::vector<char>, int32_t> PairStringToChar(const std::pair<std::string, int32_t> &s) {
return std::pair<std::vector<char>, int32_t>(std::vector<char>(s.first.begin(), s.first.end()), s.second);
}

10
mindspore/ccsrc/minddata/dataset/api/text.cc
View File

@ -212,9 +212,13 @@ Status JiebaTokenizer::ParserFile(const std::string &file_path,
struct Lookup::Data {
Data(const std::shared_ptr<Vocab> &vocab, const std::optional<std::vector<char>> &unknown_token,
mindspore::DataType data_type)
: vocab_(vocab),
unknown_token_(OptionalCharToString(unknown_token)),
data_type_(dataset::MSTypeToDEType(static_cast<TypeId>(data_type))) {}
: vocab_(vocab), data_type_(dataset::MSTypeToDEType(static_cast<TypeId>(data_type))) {
if (unknown_token == std::nullopt) {
unknown_token_ = std::nullopt;
} else {
unknown_token_ = std::string(unknown_token->begin(), unknown_token->end());
}
}
std::shared_ptr<Vocab> vocab_;
std::optional<std::string> unknown_token_;
dataset::DataType data_type_;

9
mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache_impl.h vendored
View File

@ -49,10 +49,15 @@ class DatasetCacheImpl : public DatasetCache {
: session_id_(id),
cache_mem_sz_(mem_sz),
spill_(spill),
hostname_(OptionalCharToString(hostname)),
port_(std::move(port)),
num_connections_(std::move(num_connections)),
prefetch_sz_(std::move(prefetch_sz)) {}
prefetch_sz_(std::move(prefetch_sz)) {
if (hostname == std::nullopt) {
hostname_ = std::nullopt;
} else {
hostname_ = std::string(hostname->begin(), hostname->end());
}
}
/// Method to initialize the DatasetCache by creating an instance of a CacheClient
/// \return Status Error code

7
mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h
View File

@ -1726,8 +1726,11 @@ inline std::shared_ptr<DatasetCache> CreateDatasetCache(session_id_type id, uint
std::optional<int32_t> port = std::nullopt,
std::optional<int32_t> num_connections = std::nullopt,
std::optional<int32_t> prefetch_sz = std::nullopt) {
return CreateDatasetCacheCharIF(id, mem_sz, spill, OptionalStringToChar(hostname), port, num_connections,
prefetch_sz);
std::optional<std::vector<char>> hostname_c = std::nullopt;
if (hostname != std::nullopt) {
hostname_c = std::vector<char>(hostname->begin(), hostname->end());
}
return CreateDatasetCacheCharIF(id, mem_sz, spill, hostname_c, port, num_connections, prefetch_sz);
}
/// \brief Function to create a ZipDataset.

9
mindspore/ccsrc/minddata/dataset/include/dataset/text.h
View File

@ -214,8 +214,13 @@ class Lookup final : public TensorTransform {
/// \param[in] data_type mindspore::DataType of the tensor after lookup; must be numeric, including bool.
/// (default=mindspore::DataType::kNumberTypeInt32).
explicit Lookup(const std::shared_ptr<Vocab> &vocab, const std::optional<std::string> &unknown_token = {},
mindspore::DataType data_type = mindspore::DataType::kNumberTypeInt32)
: Lookup(vocab, OptionalStringToChar(unknown_token), data_type) {}
mindspore::DataType data_type = mindspore::DataType::kNumberTypeInt32) {
std::optional<std::vector<char>> unknown_token_c = std::nullopt;
if (unknown_token != std::nullopt) {
unknown_token_c = std::vector<char>(unknown_token->begin(), unknown_token->end());
}
new (this) Lookup(vocab, unknown_token_c, data_type);
}
explicit Lookup(const std::shared_ptr<Vocab> &vocab, const std::optional<std::vector<char>> &unknown_token,
mindspore::DataType data_type = mindspore::DataType::kNumberTypeInt32);