forked from mindspore-Ecosystem/mindspore
!9208 Add four new c++ api of text
From: @shenwei41 Reviewed-by: @heleiwang Signed-off-by:
This commit is contained in:
Gitee
commit
c41a21d2c2
|
|
@ -17,12 +17,20 @@
|
|||
#include <unistd.h>
|
||||
|
||||
#include "minddata/dataset/include/text.h"
|
||||
#ifndef _WIN32
|
||||
#include "minddata/dataset/text/kernels/case_fold_op.h"
|
||||
#endif
|
||||
#include "minddata/dataset/text/kernels/jieba_tokenizer_op.h"
|
||||
#include "minddata/dataset/text/kernels/lookup_op.h"
|
||||
#include "minddata/dataset/text/kernels/ngram_op.h"
|
||||
#ifndef _WIN32
|
||||
#include "minddata/dataset/text/kernels/normalize_utf8_op.h"
|
||||
#endif
|
||||
#include "minddata/dataset/text/kernels/sentence_piece_tokenizer_op.h"
|
||||
#include "minddata/dataset/text/kernels/sliding_window_op.h"
|
||||
#include "minddata/dataset/text/kernels/unicode_char_tokenizer_op.h"
|
||||
#ifndef _WIN32
|
||||
#include "minddata/dataset/text/kernels/unicode_script_tokenizer_op.h"
|
||||
#include "minddata/dataset/text/kernels/whitespace_tokenizer_op.h"
|
||||
#endif
|
||||
#include "minddata/dataset/util/path.h"
|
||||
|
|
@ -36,6 +44,14 @@ namespace text {
|
|||
// FUNCTIONS TO CREATE TEXT OPERATIONS
|
||||
// (In alphabetical order)
|
||||
|
||||
#ifndef _WIN32
|
||||
std::shared_ptr<CaseFoldOperation> CaseFold() {
|
||||
auto op = std::make_shared<CaseFoldOperation>();
|
||||
|
||||
return op->ValidateParams() ? op : nullptr;
|
||||
}
|
||||
#endif
|
||||
|
||||
std::shared_ptr<JiebaTokenizerOperation> JiebaTokenizer(const std::string &hmm_path, const std::string &mp_path,
|
||||
const JiebaMode &mode, bool with_offsets) {
|
||||
auto op = std::make_shared<JiebaTokenizerOperation>(hmm_path, mp_path, mode, with_offsets);
|
||||
|
|
@ -58,6 +74,14 @@ std::shared_ptr<NgramOperation> Ngram(const std::vector<int32_t> &ngrams,
|
|||
return op->ValidateParams() ? op : nullptr;
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
std::shared_ptr<NormalizeUTF8Operation> NormalizeUTF8(NormalizeForm normalize_form) {
|
||||
auto op = std::make_shared<NormalizeUTF8Operation>(normalize_form);
|
||||
|
||||
return op->ValidateParams() ? op : nullptr;
|
||||
}
|
||||
#endif
|
||||
|
||||
std::shared_ptr<SentencePieceTokenizerOperation> SentencePieceTokenizer(
|
||||
const std::shared_ptr<SentencePieceVocab> &vocab, SPieceTokenizerOutType out_type) {
|
||||
auto op = std::make_shared<SentencePieceTokenizerOperation>(vocab, out_type);
|
||||
|
|
@ -78,7 +102,19 @@ std::shared_ptr<SlidingWindowOperation> SlidingWindow(const int32_t width, const
|
|||
return op->ValidateParams() ? op : nullptr;
|
||||
}
|
||||
|
||||
std::shared_ptr<UnicodeCharTokenizerOperation> UnicodeCharTokenizer(bool with_offsets) {
|
||||
auto op = std::make_shared<UnicodeCharTokenizerOperation>(with_offsets);
|
||||
|
||||
return op->ValidateParams() ? op : nullptr;
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
std::shared_ptr<UnicodeScriptTokenizerOperation> UnicodeScriptTokenizer(bool keep_whitespace, bool with_offsets) {
|
||||
auto op = std::make_shared<UnicodeScriptTokenizerOperation>(keep_whitespace, with_offsets);
|
||||
|
||||
return op->ValidateParams() ? op : nullptr;
|
||||
}
|
||||
|
||||
std::shared_ptr<WhitespaceTokenizerOperation> WhitespaceTokenizer(bool with_offsets) {
|
||||
auto op = std::make_shared<WhitespaceTokenizerOperation>(with_offsets);
|
||||
|
||||
|
|
@ -116,6 +152,16 @@ Status ValidateTokenizerDirParam(const std::string &tokenizer_name, const std::s
|
|||
|
||||
// (In alphabetical order)
|
||||
|
||||
#ifndef _WIN32
|
||||
// CaseFoldOperation
|
||||
Status CaseFoldOperation::ValidateParams() { return Status::OK(); }
|
||||
|
||||
std::shared_ptr<TensorOp> CaseFoldOperation::Build() {
|
||||
std::shared_ptr<CaseFoldOp> tensor_op = std::make_shared<CaseFoldOp>();
|
||||
return tensor_op;
|
||||
}
|
||||
#endif
|
||||
|
||||
// JiebaTokenizerOperation
|
||||
JiebaTokenizerOperation::JiebaTokenizerOperation(const std::string &hmm_path, const std::string &mp_path,
|
||||
const JiebaMode &mode, bool with_offsets)
|
||||
|
|
@ -220,6 +266,18 @@ std::shared_ptr<TensorOp> NgramOperation::Build() {
|
|||
return tensor_op;
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
// NormalizeUTF8Operation
|
||||
NormalizeUTF8Operation::NormalizeUTF8Operation(NormalizeForm normalize_form) : normalize_form_(normalize_form) {}
|
||||
|
||||
Status NormalizeUTF8Operation::ValidateParams() { return Status::OK(); }
|
||||
|
||||
std::shared_ptr<TensorOp> NormalizeUTF8Operation::Build() {
|
||||
std::shared_ptr<NormalizeUTF8Op> tensor_op = std::make_shared<NormalizeUTF8Op>(normalize_form_);
|
||||
return tensor_op;
|
||||
}
|
||||
#endif
|
||||
|
||||
// SentencePieceTokenizerOperation
|
||||
SentencePieceTokenizerOperation::SentencePieceTokenizerOperation(const std::shared_ptr<SentencePieceVocab> &vocab,
|
||||
SPieceTokenizerOutType out_type)
|
||||
|
|
@ -283,7 +341,29 @@ std::shared_ptr<TensorOp> SlidingWindowOperation::Build() {
|
|||
return tensor_op;
|
||||
}
|
||||
|
||||
// UnicodeCharTokenizerOperation
|
||||
UnicodeCharTokenizerOperation::UnicodeCharTokenizerOperation(bool with_offsets) : with_offsets_(with_offsets) {}
|
||||
|
||||
Status UnicodeCharTokenizerOperation::ValidateParams() { return Status::OK(); }
|
||||
|
||||
std::shared_ptr<TensorOp> UnicodeCharTokenizerOperation::Build() {
|
||||
std::shared_ptr<UnicodeCharTokenizerOp> tensor_op = std::make_shared<UnicodeCharTokenizerOp>(with_offsets_);
|
||||
return tensor_op;
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
// UnicodeScriptTokenizerOperation
|
||||
UnicodeScriptTokenizerOperation::UnicodeScriptTokenizerOperation(bool keep_whitespace, bool with_offsets)
|
||||
: keep_whitespace_(keep_whitespace), with_offsets_(with_offsets) {}
|
||||
|
||||
Status UnicodeScriptTokenizerOperation::ValidateParams() { return Status::OK(); }
|
||||
|
||||
std::shared_ptr<TensorOp> UnicodeScriptTokenizerOperation::Build() {
|
||||
std::shared_ptr<UnicodeScriptTokenizerOp> tensor_op =
|
||||
std::make_shared<UnicodeScriptTokenizerOp>(keep_whitespace_, with_offsets_);
|
||||
return tensor_op;
|
||||
}
|
||||
|
||||
// WhitespaceTokenizerOperation
|
||||
WhitespaceTokenizerOperation::WhitespaceTokenizerOperation(bool with_offsets) : with_offsets_(with_offsets) {}
|
||||
|
||||
|
|
|
|||
|
|
@ -59,6 +59,15 @@ enum class SPieceTokenizerOutType { kString = 0, kInt = 1 };
|
|||
// Possible values for SPieceTokenizerLoadType
|
||||
enum class SPieceTokenizerLoadType { kFile = 0, kModel = 1 };
|
||||
|
||||
// Possible values for NormalizeForm
|
||||
enum class NormalizeForm {
|
||||
kNone = 0,
|
||||
kNfc,
|
||||
kNfkc,
|
||||
kNfd,
|
||||
kNfkd,
|
||||
};
|
||||
|
||||
// convenience functions for 32bit int bitmask
|
||||
inline bool BitTest(uint32_t bits, uint32_t bitMask) { return (bits & bitMask) == bitMask; }
|
||||
|
||||
|
|
|
|||
|
|
@ -38,23 +38,41 @@ namespace dataset {
|
|||
namespace text {
|
||||
|
||||
// Char arrays storing name of corresponding classes (in alphabetical order)
|
||||
constexpr char kCaseFoldOperation[] = "CaseFold";
|
||||
constexpr char kJiebaTokenizerOperation[] = "JiebaTokenizer";
|
||||
constexpr char kLookupOperation[] = "Lookup";
|
||||
constexpr char kNgramOperation[] = "Ngram";
|
||||
constexpr char kNormalizeUTF8Operation[] = "NormalizeUTF8";
|
||||
constexpr char kSentencepieceTokenizerOperation[] = "SentencepieceTokenizer";
|
||||
constexpr char kSlidingWindowOperation[] = "SlidingWindow";
|
||||
constexpr char kUnicodeCharTokenizerOperation[] = "UnicodeCharTokenizer";
|
||||
constexpr char kUnicodeScriptTokenizerOperation[] = "UnicodeScriptTokenizer";
|
||||
constexpr char kWhitespaceTokenizerOperation[] = "WhitespaceTokenizer";
|
||||
|
||||
// Text Op classes (in alphabetical order)
|
||||
#ifndef _WIN32
|
||||
class CaseFoldOperation;
|
||||
#endif
|
||||
class JiebaTokenizerOperation;
|
||||
class LookupOperation;
|
||||
class NgramOperation;
|
||||
#ifndef _WIN32
|
||||
class NormalizeUTF8Operation;
|
||||
#endif
|
||||
class SentencePieceTokenizerOperation;
|
||||
class SlidingWindowOperation;
|
||||
class UnicodeCharTokenizerOperation;
|
||||
#ifndef _WIN32
|
||||
class UnicodeScriptTokenizerOperation;
|
||||
class WhitespaceTokenizerOperation;
|
||||
#endif
|
||||
|
||||
#ifndef _WIN32
|
||||
/// \brief Apply case fold operation on UTF-8 string tensor.
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<CaseFoldOperation> CaseFold();
|
||||
#endif
|
||||
|
||||
/// \brief Tokenize Chinese string into words based on dictionary.
|
||||
/// \param[in] hmm_path Dictionary file is used by HMMSegment algorithm. The dictionary can be obtained on the
|
||||
/// official website of cppjieba.
|
||||
|
|
@ -94,6 +112,21 @@ std::shared_ptr<NgramOperation> Ngram(const std::vector<int32_t> &ngrams,
|
|||
const std::pair<std::string, int32_t> &right_pad = {"", 0},
|
||||
const std::string &separator = " ");
|
||||
|
||||
#ifndef _WIN32
|
||||
/// \brief Apply normalize operation on UTF-8 string tensor.
|
||||
/// \param[in] normalize_form Valid values can be any of [NormalizeForm::kNone,NormalizeForm::kNfc,
|
||||
/// NormalizeForm::kNfkc,
|
||||
/// NormalizeForm::kNfd, NormalizeForm::kNfkd](default=NormalizeForm::kNfkc).
|
||||
/// See http://unicode.org/reports/tr15/ for details.
|
||||
/// - NormalizeForm.NONE, do nothing for input string tensor.
|
||||
/// - NormalizeForm.NFC, normalize with Normalization Form C.
|
||||
/// - NormalizeForm.NFKC, normalize with Normalization Form KC.
|
||||
/// - NormalizeForm.NFD, normalize with Normalization Form D.
|
||||
/// - NormalizeForm.NFKD, normalize with Normalization Form KD.
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<NormalizeUTF8Operation> NormalizeUTF8(NormalizeForm normalize_form = NormalizeForm::kNfkc);
|
||||
#endif
|
||||
|
||||
/// \brief Tokenize scalar token or 1-D tokens to tokens by sentencepiece.
|
||||
/// \param[in] vocab a SentencePieceVocab object.
|
||||
/// \param[in] out_type The type of output.
|
||||
|
|
@ -116,8 +149,20 @@ std::shared_ptr<SentencePieceTokenizerOperation> SentencePieceTokenizer(
|
|||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<SlidingWindowOperation> SlidingWindow(const int32_t width, const int32_t axis = 0);
|
||||
|
||||
/// \brief Tokenize a scalar tensor of UTF-8 string to Unicode characters.
|
||||
/// \param[in] with_offsets If or not output offsets of tokens (default=false).
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<UnicodeCharTokenizerOperation> UnicodeCharTokenizer(bool with_offsets = false);
|
||||
|
||||
#ifndef _WIN32
|
||||
/// \brief Tokenize a scalar tensor of UTF-8 string on ICU4C defined whitespaces
|
||||
/// \brief Tokenize a scalar tensor of UTF-8 string on Unicode script boundaries.
|
||||
/// \param[in] keep_whitespace If or not emit whitespace tokens (default=false).
|
||||
/// \param[in] with_offsets If or not output offsets of tokens (default=false).
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<UnicodeScriptTokenizerOperation> UnicodeScriptTokenizer(bool keep_whitespace = false,
|
||||
bool with_offsets = false);
|
||||
|
||||
/// \brief Tokenize a scalar tensor of UTF-8 string on ICU4C defined whitespaces.
|
||||
/// \param[in] with_offsets If or not output offsets of tokens (default=false).
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<WhitespaceTokenizerOperation> WhitespaceTokenizer(bool with_offsets = false);
|
||||
|
|
@ -125,6 +170,21 @@ std::shared_ptr<WhitespaceTokenizerOperation> WhitespaceTokenizer(bool with_offs
|
|||
|
||||
/* ####################################### Derived TensorOperation classes ################################# */
|
||||
|
||||
#ifndef _WIN32
|
||||
class CaseFoldOperation : public TensorOperation {
|
||||
public:
|
||||
CaseFoldOperation() = default;
|
||||
|
||||
~CaseFoldOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kCaseFoldOperation; }
|
||||
};
|
||||
#endif
|
||||
|
||||
class JiebaTokenizerOperation : public TensorOperation {
|
||||
public:
|
||||
explicit JiebaTokenizerOperation(const std::string &hmm_path, const std::string &mp_path, const JiebaMode &mode,
|
||||
|
|
@ -185,6 +245,24 @@ class NgramOperation : public TensorOperation {
|
|||
std::string separator_;
|
||||
};
|
||||
|
||||
#ifndef _WIN32
|
||||
class NormalizeUTF8Operation : public TensorOperation {
|
||||
public:
|
||||
explicit NormalizeUTF8Operation(NormalizeForm normalize_form);
|
||||
|
||||
~NormalizeUTF8Operation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kNormalizeUTF8Operation; }
|
||||
|
||||
private:
|
||||
NormalizeForm normalize_form_;
|
||||
};
|
||||
#endif
|
||||
|
||||
class SentencePieceTokenizerOperation : public TensorOperation {
|
||||
public:
|
||||
SentencePieceTokenizerOperation(const std::shared_ptr<SentencePieceVocab> &vocab, SPieceTokenizerOutType out_type);
|
||||
|
|
@ -223,7 +301,40 @@ class SlidingWindowOperation : public TensorOperation {
|
|||
int32_t axis_;
|
||||
};
|
||||
|
||||
class UnicodeCharTokenizerOperation : public TensorOperation {
|
||||
public:
|
||||
explicit UnicodeCharTokenizerOperation(bool with_offsets);
|
||||
|
||||
~UnicodeCharTokenizerOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kUnicodeCharTokenizerOperation; }
|
||||
|
||||
private:
|
||||
bool with_offsets_;
|
||||
};
|
||||
|
||||
#ifndef _WIN32
|
||||
class UnicodeScriptTokenizerOperation : public TensorOperation {
|
||||
public:
|
||||
explicit UnicodeScriptTokenizerOperation(bool keep_whitespace, bool with_offsets);
|
||||
|
||||
~UnicodeScriptTokenizerOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
std::string Name() const override { return kUnicodeScriptTokenizerOperation; }
|
||||
|
||||
private:
|
||||
bool keep_whitespace_;
|
||||
bool with_offsets_;
|
||||
};
|
||||
|
||||
class WhitespaceTokenizerOperation : public TensorOperation {
|
||||
public:
|
||||
explicit WhitespaceTokenizerOperation(bool with_offsets);
|
||||
|
|
|
|||
|
|
@ -24,13 +24,6 @@
|
|||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
enum class NormalizeForm {
|
||||
kNone = 0,
|
||||
kNfc,
|
||||
kNfkc,
|
||||
kNfd,
|
||||
kNfkd,
|
||||
};
|
||||
|
||||
class NormalizeUTF8Op : public TensorOp {
|
||||
public:
|
||||
|
|
|
|||
|
|
@ -34,6 +34,49 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
|
|||
protected:
|
||||
};
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestCaseFoldSuccess) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCaseFoldSuccess.";
|
||||
|
||||
// Create a TextFile dataset
|
||||
std::string data_file = datasets_root_path_ + "/testTokenizerData/1.txt";
|
||||
std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create casefold operation on ds
|
||||
std::shared_ptr<TensorOperation> casefold = text::CaseFold();
|
||||
EXPECT_NE(casefold, nullptr);
|
||||
|
||||
// Create Map operation on ds
|
||||
ds = ds->Map({casefold}, {"text"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
std::vector<std::string> expected = {"welcome to beijing!", "北京欢迎您!", "我喜欢english!", " "};
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
auto ind = row["text"];
|
||||
std::shared_ptr<Tensor> expected_tensor;
|
||||
Tensor::CreateScalar(expected[i], &expected_tensor);
|
||||
EXPECT_EQ(*ind, *expected_tensor);
|
||||
iter->GetNextRow(&row);
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 4);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestJiebaTokenizerSuccess) {
|
||||
// Testing the parameter of JiebaTokenizer interface when the mode is JiebaMode::kMp and the with_offsets is false.
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestJiebaTokenizerSuccess.";
|
||||
|
|
@ -472,6 +515,514 @@ TEST_F(MindDataTestPipeline, TestTextOperationName) {
|
|||
EXPECT_EQ(correct_name, sentence_piece_tokenizer_op->Name());
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestNormalizeUTF8Success) {
|
||||
// Testing the parameter of NormalizeUTF8 interface when the normalize_form is NormalizeForm::kNfkc.
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestNormalizeUTF8Success.";
|
||||
|
||||
// Create a TextFile dataset
|
||||
std::string data_file = datasets_root_path_ + "/testTokenizerData/normalize.txt";
|
||||
std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create normalizeutf8 operation on ds
|
||||
std::shared_ptr<TensorOperation> normalizeutf8 = text::NormalizeUTF8(NormalizeForm::kNfkc);
|
||||
EXPECT_NE(normalizeutf8, nullptr);
|
||||
|
||||
// Create Map operation on ds
|
||||
ds = ds->Map({normalizeutf8}, {"text"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
std::vector<std::string> expected = {"ṩ", "ḍ̇", "q̣̇", "fi", "25", "ṩ"};
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
auto ind = row["text"];
|
||||
std::shared_ptr<Tensor> expected_tensor;
|
||||
Tensor::CreateScalar(expected[i], &expected_tensor);
|
||||
EXPECT_EQ(*ind, *expected_tensor);
|
||||
iter->GetNextRow(&row);
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 6);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestNormalizeUTF8Success1) {
|
||||
// Testing the parameter of NormalizeUTF8 interface when the normalize_form is NormalizeForm::kNfc.
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestNormalizeUTF8Success1.";
|
||||
|
||||
// Create a TextFile dataset
|
||||
std::string data_file = datasets_root_path_ + "/testTokenizerData/normalize.txt";
|
||||
std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create normalizeutf8 operation on ds
|
||||
std::shared_ptr<TensorOperation> normalizeutf8 = text::NormalizeUTF8(NormalizeForm::kNfc);
|
||||
EXPECT_NE(normalizeutf8, nullptr);
|
||||
|
||||
// Create Map operation on ds
|
||||
ds = ds->Map({normalizeutf8}, {"text"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
std::vector<std::string> expected = {"ṩ", "ḍ̇", "q̣̇", "fi", "2⁵", "ẛ̣"};
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
auto ind = row["text"];
|
||||
std::shared_ptr<Tensor> expected_tensor;
|
||||
Tensor::CreateScalar(expected[i], &expected_tensor);
|
||||
EXPECT_EQ(*ind, *expected_tensor);
|
||||
iter->GetNextRow(&row);
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 6);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestNormalizeUTF8Success2) {
|
||||
// Testing the parameter of NormalizeUTF8 interface when the normalize_form is NormalizeForm::kNfd.
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestNormalizeUTF8Success2.";
|
||||
|
||||
// Create a TextFile dataset
|
||||
std::string data_file = datasets_root_path_ + "/testTokenizerData/normalize.txt";
|
||||
std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create normalizeutf8 operation on ds
|
||||
std::shared_ptr<TensorOperation> normalizeutf8 = text::NormalizeUTF8(NormalizeForm::kNfd);
|
||||
EXPECT_NE(normalizeutf8, nullptr);
|
||||
|
||||
// Create Map operation on ds
|
||||
ds = ds->Map({normalizeutf8}, {"text"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
std::vector<std::string> expected = {"ṩ", "ḍ̇", "q̣̇", "fi", "2⁵", "ẛ̣"};
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
auto ind = row["text"];
|
||||
std::shared_ptr<Tensor> expected_tensor;
|
||||
Tensor::CreateScalar(expected[i], &expected_tensor);
|
||||
EXPECT_EQ(*ind, *expected_tensor);
|
||||
iter->GetNextRow(&row);
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 6);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestNormalizeUTF8Success3) {
|
||||
// Testing the parameter of NormalizeUTF8 interface when the normalize_form is NormalizeForm::kNfkd.
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestNormalizeUTF8Success3.";
|
||||
|
||||
// Create a TextFile dataset
|
||||
std::string data_file = datasets_root_path_ + "/testTokenizerData/normalize.txt";
|
||||
std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create normalizeutf8 operation on ds
|
||||
std::shared_ptr<TensorOperation> normalizeutf8 = text::NormalizeUTF8(NormalizeForm::kNfkd);
|
||||
EXPECT_NE(normalizeutf8, nullptr);
|
||||
|
||||
// Create Map operation on ds
|
||||
ds = ds->Map({normalizeutf8}, {"text"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
std::vector<std::string> expected = {"ṩ", "ḍ̇", "q̣̇", "fi", "25", "ṩ"};
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
auto ind = row["text"];
|
||||
std::shared_ptr<Tensor> expected_tensor;
|
||||
Tensor::CreateScalar(expected[i], &expected_tensor);
|
||||
EXPECT_EQ(*ind, *expected_tensor);
|
||||
iter->GetNextRow(&row);
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 6);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestUnicodeCharTokenizerSuccess) {
|
||||
// Testing the parameter of UnicodeCharTokenizer interface when the with_offsets is default.
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUnicodeCharTokenizerSuccess.";
|
||||
|
||||
// Create a TextFile dataset
|
||||
std::string data_file = datasets_root_path_ + "/testTokenizerData/1.txt";
|
||||
std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create unicodechar_tokenizer operation on ds
|
||||
std::shared_ptr<TensorOperation> unicodechar_tokenizer = text::UnicodeCharTokenizer();
|
||||
EXPECT_NE(unicodechar_tokenizer, nullptr);
|
||||
|
||||
// Create Map operation on ds
|
||||
ds = ds->Map({unicodechar_tokenizer}, {"text"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
std::vector<std::vector<std::string>> expected = {
|
||||
{"W", "e", "l", "c", "o", "m", "e", " ", "t", "o", " ", "B", "e", "i", "j", "i", "n", "g", "!"},
|
||||
{"北", "京", "欢", "迎", "您", "！"},
|
||||
{"我", "喜", "欢", "E", "n", "g", "l", "i", "s", "h", "!"},
|
||||
{" ", " "}};
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
auto ind = row["text"];
|
||||
std::shared_ptr<Tensor> expected_tensor;
|
||||
int x = expected[i].size();
|
||||
Tensor::CreateFromVector(expected[i], TensorShape({x}), &expected_tensor);
|
||||
EXPECT_EQ(*ind, *expected_tensor);
|
||||
iter->GetNextRow(&row);
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 4);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestUnicodeCharTokenizerSuccess1) {
|
||||
// Testing the parameter of UnicodeCharTokenizer interface when the with_offsets is true.
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUnicodeCharTokenizerSuccess1.";
|
||||
|
||||
// Create a TextFile dataset
|
||||
std::string data_file = datasets_root_path_ + "/testTokenizerData/1.txt";
|
||||
std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create unicodechar_tokenizer operation on ds
|
||||
std::shared_ptr<TensorOperation> unicodechar_tokenizer = text::UnicodeCharTokenizer(true);
|
||||
EXPECT_NE(unicodechar_tokenizer, nullptr);
|
||||
|
||||
// Create Map operation on ds
|
||||
ds = ds->Map({unicodechar_tokenizer}, {"text"}, {"token", "offsets_start", "offsets_limit"},
|
||||
{"token", "offsets_start", "offsets_limit"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
std::vector<std::vector<std::string>> expected = {
|
||||
{"W", "e", "l", "c", "o", "m", "e", " ", "t", "o", " ", "B", "e", "i", "j", "i", "n", "g", "!"},
|
||||
{"北", "京", "欢", "迎", "您", "！"},
|
||||
{"我", "喜", "欢", "E", "n", "g", "l", "i", "s", "h", "!"},
|
||||
{" ", " "}};
|
||||
|
||||
std::vector<std::vector<uint32_t>> expected_offsets_start = {
|
||||
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18},
|
||||
{0, 3, 6, 9, 12, 15},
|
||||
{0, 3, 6, 9, 10, 11, 12, 13, 14, 15, 16},
|
||||
{0, 1}};
|
||||
std::vector<std::vector<uint32_t>> expected_offsets_limit = {
|
||||
{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19},
|
||||
{3, 6, 9, 12, 15, 18},
|
||||
{3, 6, 9, 10, 11, 12, 13, 14, 15, 16, 17},
|
||||
{1, 2}};
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
auto ind = row["offsets_start"];
|
||||
auto ind1 = row["offsets_limit"];
|
||||
auto token = row["token"];
|
||||
std::shared_ptr<Tensor> expected_tensor;
|
||||
std::shared_ptr<Tensor> expected_tensor_offsets_start;
|
||||
std::shared_ptr<Tensor> expected_tensor_offsets_limit;
|
||||
int x = expected[i].size();
|
||||
Tensor::CreateFromVector(expected[i], TensorShape({x}), &expected_tensor);
|
||||
Tensor::CreateFromVector(expected_offsets_start[i], TensorShape({x}), &expected_tensor_offsets_start);
|
||||
Tensor::CreateFromVector(expected_offsets_limit[i], TensorShape({x}), &expected_tensor_offsets_limit);
|
||||
EXPECT_EQ(*ind, *expected_tensor_offsets_start);
|
||||
EXPECT_EQ(*ind1, *expected_tensor_offsets_limit);
|
||||
EXPECT_EQ(*token, *expected_tensor);
|
||||
|
||||
iter->GetNextRow(&row);
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 4);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestUnicodeScriptTokenizerSuccess) {
|
||||
// Testing the parameter of UnicodeScriptTokenizer interface when the with_offsets and the keep_whitespace is default.
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUnicodeScriptTokenizerSuccess.";
|
||||
|
||||
// Create a TextFile dataset
|
||||
std::string data_file = datasets_root_path_ + "/testTokenizerData/1.txt";
|
||||
std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create unicodescript_tokenizer operation on ds
|
||||
std::shared_ptr<TensorOperation> unicodescript_tokenizer = text::UnicodeScriptTokenizer();
|
||||
EXPECT_NE(unicodescript_tokenizer, nullptr);
|
||||
|
||||
// Create Map operation on ds
|
||||
ds = ds->Map({unicodescript_tokenizer}, {"text"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
std::vector<std::vector<std::string>> expected = {
|
||||
{"Welcome", "to", "Beijing", "!"}, {"北京欢迎您", "！"}, {"我喜欢", "English", "!"}, {""}};
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
auto ind = row["text"];
|
||||
std::shared_ptr<Tensor> expected_tensor;
|
||||
int x = expected[i].size();
|
||||
Tensor::CreateFromVector(expected[i], TensorShape({x}), &expected_tensor);
|
||||
EXPECT_EQ(*ind, *expected_tensor);
|
||||
iter->GetNextRow(&row);
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 4);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestUnicodeScriptTokenizerSuccess1) {
|
||||
// Testing the parameter of UnicodeScriptTokenizer interface when the keep_whitespace is true and the with_offsets is
|
||||
// false.
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUnicodeScriptTokenizerSuccess1.";
|
||||
|
||||
// Create a TextFile dataset
|
||||
std::string data_file = datasets_root_path_ + "/testTokenizerData/1.txt";
|
||||
std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create unicodescript_tokenizer operation on ds
|
||||
std::shared_ptr<TensorOperation> unicodescript_tokenizer = text::UnicodeScriptTokenizer(true);
|
||||
EXPECT_NE(unicodescript_tokenizer, nullptr);
|
||||
|
||||
// Create Map operation on ds
|
||||
ds = ds->Map({unicodescript_tokenizer}, {"text"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
std::vector<std::vector<std::string>> expected = {
|
||||
{"Welcome", " ", "to", " ", "Beijing", "!"}, {"北京欢迎您", "！"}, {"我喜欢", "English", "!"}, {" "}};
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
auto ind = row["text"];
|
||||
std::shared_ptr<Tensor> expected_tensor;
|
||||
int x = expected[i].size();
|
||||
Tensor::CreateFromVector(expected[i], TensorShape({x}), &expected_tensor);
|
||||
EXPECT_EQ(*ind, *expected_tensor);
|
||||
iter->GetNextRow(&row);
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 4);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestUnicodeScriptTokenizerSuccess2) {
|
||||
// Testing the parameter of UnicodeScriptTokenizer interface when the keep_whitespace is false and the with_offsets is
|
||||
// true.
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUnicodeScriptTokenizerSuccess2.";
|
||||
|
||||
// Create a TextFile dataset
|
||||
std::string data_file = datasets_root_path_ + "/testTokenizerData/1.txt";
|
||||
std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create unicodescript_tokenizer operation on ds
|
||||
std::shared_ptr<TensorOperation> unicodescript_tokenizer = text::UnicodeScriptTokenizer(false, true);
|
||||
EXPECT_NE(unicodescript_tokenizer, nullptr);
|
||||
|
||||
// Create Map operation on ds
|
||||
ds = ds->Map({unicodescript_tokenizer}, {"text"}, {"token", "offsets_start", "offsets_limit"},
|
||||
{"token", "offsets_start", "offsets_limit"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
std::vector<std::vector<std::string>> expected = {
|
||||
{"Welcome", "to", "Beijing", "!"}, {"北京欢迎您", "！"}, {"我喜欢", "English", "!"}, {""}};
|
||||
|
||||
std::vector<std::vector<uint32_t>> expected_offsets_start = {{0, 8, 11, 18}, {0, 15}, {0, 9, 16}, {0}};
|
||||
std::vector<std::vector<uint32_t>> expected_offsets_limit = {{7, 10, 18, 19}, {15, 18}, {9, 16, 17}, {0}};
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
auto ind = row["offsets_start"];
|
||||
auto ind1 = row["offsets_limit"];
|
||||
auto token = row["token"];
|
||||
std::shared_ptr<Tensor> expected_tensor;
|
||||
std::shared_ptr<Tensor> expected_tensor_offsets_start;
|
||||
std::shared_ptr<Tensor> expected_tensor_offsets_limit;
|
||||
int x = expected[i].size();
|
||||
Tensor::CreateFromVector(expected[i], TensorShape({x}), &expected_tensor);
|
||||
Tensor::CreateFromVector(expected_offsets_start[i], TensorShape({x}), &expected_tensor_offsets_start);
|
||||
Tensor::CreateFromVector(expected_offsets_limit[i], TensorShape({x}), &expected_tensor_offsets_limit);
|
||||
EXPECT_EQ(*ind, *expected_tensor_offsets_start);
|
||||
EXPECT_EQ(*ind1, *expected_tensor_offsets_limit);
|
||||
EXPECT_EQ(*token, *expected_tensor);
|
||||
|
||||
iter->GetNextRow(&row);
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 4);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestUnicodeScriptTokenizerSuccess3) {
|
||||
// Testing the parameter of UnicodeScriptTokenizer interface when the keep_whitespace is true and the with_offsets is
|
||||
// true.
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUnicodeScriptTokenizerSuccess3.";
|
||||
|
||||
// Create a TextFile dataset
|
||||
std::string data_file = datasets_root_path_ + "/testTokenizerData/1.txt";
|
||||
std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create unicodescript_tokenizer operation on ds
|
||||
std::shared_ptr<TensorOperation> unicodescript_tokenizer = text::UnicodeScriptTokenizer(true, true);
|
||||
EXPECT_NE(unicodescript_tokenizer, nullptr);
|
||||
|
||||
// Create Map operation on ds
|
||||
ds = ds->Map({unicodescript_tokenizer}, {"text"}, {"token", "offsets_start", "offsets_limit"},
|
||||
{"token", "offsets_start", "offsets_limit"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
std::vector<std::vector<std::string>> expected = {
|
||||
{"Welcome", " ", "to", " ", "Beijing", "!"}, {"北京欢迎您", "！"}, {"我喜欢", "English", "!"}, {" "}};
|
||||
|
||||
std::vector<std::vector<uint32_t>> expected_offsets_start = {{0, 7, 8, 10, 11, 18}, {0, 15}, {0, 9, 16}, {0}};
|
||||
std::vector<std::vector<uint32_t>> expected_offsets_limit = {{7, 8, 10, 11, 18, 19}, {15, 18}, {9, 16, 17}, {2}};
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
auto ind = row["offsets_start"];
|
||||
auto ind1 = row["offsets_limit"];
|
||||
auto token = row["token"];
|
||||
std::shared_ptr<Tensor> expected_tensor;
|
||||
std::shared_ptr<Tensor> expected_tensor_offsets_start;
|
||||
std::shared_ptr<Tensor> expected_tensor_offsets_limit;
|
||||
int x = expected[i].size();
|
||||
Tensor::CreateFromVector(expected[i], TensorShape({x}), &expected_tensor);
|
||||
Tensor::CreateFromVector(expected_offsets_start[i], TensorShape({x}), &expected_tensor_offsets_start);
|
||||
Tensor::CreateFromVector(expected_offsets_limit[i], TensorShape({x}), &expected_tensor_offsets_limit);
|
||||
EXPECT_EQ(*ind, *expected_tensor_offsets_start);
|
||||
EXPECT_EQ(*ind1, *expected_tensor_offsets_limit);
|
||||
EXPECT_EQ(*token, *expected_tensor);
|
||||
|
||||
iter->GetNextRow(&row);
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 4);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestWhitespaceTokenizerSuccess) {
|
||||
// Testing the parameter of WhitespaceTokenizer interface when the with_offsets is default.
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestWhitespaceTokenizerSuccess.";
|
||||
|
|
|
|||
Loading…
Reference in New Issue