forked from mindspore-Ecosystem/mindspore
57 lines
1.7 KiB
ReStructuredText
57 lines
1.7 KiB
ReStructuredText
mindspore.dataset.text
|
|
======================
|
|
|
|
.. automodule:: mindspore.dataset.text
|
|
|
|
Transforms
|
|
----------
|
|
|
|
.. msnoteautosummary::
|
|
:toctree: dataset_text
|
|
:nosignatures:
|
|
:template: classtemplate.rst
|
|
|
|
mindspore.dataset.text.BasicTokenizer
|
|
mindspore.dataset.text.BertTokenizer
|
|
mindspore.dataset.text.CaseFold
|
|
mindspore.dataset.text.FilterWikipediaXML
|
|
mindspore.dataset.text.JiebaTokenizer
|
|
mindspore.dataset.text.Lookup
|
|
mindspore.dataset.text.Ngram
|
|
mindspore.dataset.text.NormalizeUTF8
|
|
mindspore.dataset.text.PythonTokenizer
|
|
mindspore.dataset.text.RegexReplace
|
|
mindspore.dataset.text.RegexTokenizer
|
|
mindspore.dataset.text.SentencePieceTokenizer
|
|
mindspore.dataset.text.SlidingWindow
|
|
mindspore.dataset.text.ToNumber
|
|
mindspore.dataset.text.ToVectors
|
|
mindspore.dataset.text.TruncateSequencePair
|
|
mindspore.dataset.text.UnicodeCharTokenizer
|
|
mindspore.dataset.text.UnicodeScriptTokenizer
|
|
mindspore.dataset.text.WhitespaceTokenizer
|
|
mindspore.dataset.text.WordpieceTokenizer
|
|
|
|
|
|
Utilities
|
|
---------
|
|
|
|
.. msnoteautosummary::
|
|
:toctree: dataset_text
|
|
:nosignatures:
|
|
:template: classtemplate.rst
|
|
|
|
mindspore.dataset.text.CharNGram
|
|
mindspore.dataset.text.FastText
|
|
mindspore.dataset.text.GloVe
|
|
mindspore.dataset.text.JiebaMode
|
|
mindspore.dataset.text.NormalizeForm
|
|
mindspore.dataset.text.SentencePieceModel
|
|
mindspore.dataset.text.SentencePieceVocab
|
|
mindspore.dataset.text.SPieceTokenizerLoadType
|
|
mindspore.dataset.text.SPieceTokenizerOutType
|
|
mindspore.dataset.text.Vectors
|
|
mindspore.dataset.text.Vocab
|
|
mindspore.dataset.text.to_bytes
|
|
mindspore.dataset.text.to_str
|