Index

A

add_vocab() (meguru_tokenizer.vocab.Vocab method)

add_vocabs() (meguru_tokenizer.vocab.Vocab method)

B

BaseVocab (class in meguru_tokenizer.vocab)

build_vocab() (meguru_tokenizer.vocab.Vocab method)

D

decode() (meguru_tokenizer.base_tokenizer.Tokenizer method) (meguru_tokenizer.sentencepiece_tokenizer.SentencePieceTokenizer method) (meguru_tokenizer.sudachi_tokenizer.SudachiTokenizer method) (meguru_tokenizer.Tokenizer method) (meguru_tokenizer.whitespace_tokenizer.WhitespaceTokenizer method)

dump_vocab() (meguru_tokenizer.vocab.Vocab method)

E

encode() (meguru_tokenizer.base_tokenizer.Tokenizer method) (meguru_tokenizer.sentencepiece_tokenizer.SentencePieceTokenizer method) (meguru_tokenizer.sudachi_tokenizer.SudachiTokenizer method) (meguru_tokenizer.Tokenizer method) (meguru_tokenizer.whitespace_tokenizer.WhitespaceTokenizer method)

I

idx2word() (meguru_tokenizer.sentencepiece_tokenizer.SentencePieceVocab method) (meguru_tokenizer.vocab.BaseVocab method) (meguru_tokenizer.vocab.Vocab method)

L

languages (meguru_tokenizer.base_tokenizer.Tokenizer attribute) (meguru_tokenizer.sentencepiece_tokenizer.SentencePieceTokenizer attribute) (meguru_tokenizer.sudachi_tokenizer.SudachiTokenizer attribute) (meguru_tokenizer.Tokenizer attribute) (meguru_tokenizer.whitespace_tokenizer.LooseWhitespaceTokenizer attribute) (meguru_tokenizer.whitespace_tokenizer.WhitespaceTokenizer attribute)

load_sp_model() (meguru_tokenizer.sentencepiece_tokenizer.SentencePieceTokenizer method) load_vocab() (meguru_tokenizer.vocab.Vocab method) LooseWhitespaceTokenizer (class in meguru_tokenizer.whitespace_tokenizer)

M

meguru_tokenizer module meguru_tokenizer.base_tokenizer module meguru_tokenizer.process module meguru_tokenizer.process.noise_pytorch module meguru_tokenizer.process.noise_tf module meguru_tokenizer.sentencepiece_tokenizer module meguru_tokenizer.sudachi_tokenizer module

meguru_tokenizer.vocab module meguru_tokenizer.whitespace_tokenizer module module meguru_tokenizer meguru_tokenizer.base_tokenizer meguru_tokenizer.process meguru_tokenizer.process.noise_pytorch meguru_tokenizer.process.noise_tf meguru_tokenizer.sentencepiece_tokenizer meguru_tokenizer.sudachi_tokenizer meguru_tokenizer.vocab meguru_tokenizer.whitespace_tokenizer

N

Noiser (class in meguru_tokenizer.process.noise_pytorch) (class in meguru_tokenizer.process.noise_tf)

noisy() (meguru_tokenizer.process.noise_pytorch.Noiser method) (meguru_tokenizer.process.noise_tf.Noiser method)

S

SentencePieceTokenizer (class in meguru_tokenizer.sentencepiece_tokenizer)

SentencePieceVocab (class in meguru_tokenizer.sentencepiece_tokenizer) SudachiTokenizer (class in meguru_tokenizer.sudachi_tokenizer)

T

tokenize() (meguru_tokenizer.base_tokenizer.Tokenizer method) (meguru_tokenizer.sentencepiece_tokenizer.SentencePieceTokenizer method) (meguru_tokenizer.sudachi_tokenizer.SudachiTokenizer method) (meguru_tokenizer.Tokenizer method) (meguru_tokenizer.whitespace_tokenizer.LooseWhitespaceTokenizer method) (meguru_tokenizer.whitespace_tokenizer.WhitespaceTokenizer method) tokenize_list() (meguru_tokenizer.base_tokenizer.Tokenizer method) (meguru_tokenizer.sentencepiece_tokenizer.SentencePieceTokenizer method) (meguru_tokenizer.sudachi_tokenizer.SudachiTokenizer method) (meguru_tokenizer.Tokenizer method) (meguru_tokenizer.whitespace_tokenizer.WhitespaceTokenizer method)

Tokenizer (class in meguru_tokenizer) (class in meguru_tokenizer.base_tokenizer) tokenizer (meguru_tokenizer.base_tokenizer.Tokenizer attribute) (meguru_tokenizer.Tokenizer attribute) train_sp() (meguru_tokenizer.sentencepiece_tokenizer.SentencePieceTokenizer method)

V

Vocab (class in meguru_tokenizer.vocab) vocab_size() (meguru_tokenizer.base_tokenizer.Tokenizer method) (meguru_tokenizer.sentencepiece_tokenizer.SentencePieceTokenizer method) (meguru_tokenizer.sudachi_tokenizer.SudachiTokenizer method) (meguru_tokenizer.Tokenizer method) (meguru_tokenizer.whitespace_tokenizer.WhitespaceTokenizer method)

W

WhitespaceTokenizer (class in meguru_tokenizer.whitespace_tokenizer) word2idx() (meguru_tokenizer.sentencepiece_tokenizer.SentencePieceVocab method) (meguru_tokenizer.vocab.BaseVocab method) (meguru_tokenizer.vocab.Vocab method) word_blank() (meguru_tokenizer.process.noise_pytorch.Noiser method) (meguru_tokenizer.process.noise_tf.Noiser method)

word_drop() (meguru_tokenizer.process.noise_pytorch.Noiser method) (meguru_tokenizer.process.noise_tf.Noiser method) word_shuffle() (meguru_tokenizer.process.noise_pytorch.Noiser method) (meguru_tokenizer.process.noise_tf.Noiser method) word_substitute() (meguru_tokenizer.process.noise_pytorch.Noiser method) (meguru_tokenizer.process.noise_tf.Noiser method)