Trait dataflow::tokenization::Tokenizer

pub trait Tokenizer: Debug + Send + Sync {
    fn load() -> Self;
    fn tokenize(&self, string: &str) -> Vec<String>;
    fn batch_tokenize(&self, strings: Vec<String>) -> Vec<Vec<String>>;
    fn untokenize(&self, tokens: Vec<String>) -> String;
    fn batch_untokenize(&self, tokens: Vec<Vec<String>>) -> Vec<String>;
}

A trait to implement for all tokenizers, contains basic tokenizing and untokenizing functions

Required methods

fn load() -> Self

Load the tokenizer

fn tokenize(&self, string: &str) -> Vec<String>

Tokenize a single string

fn batch_tokenize(&self, strings: Vec<String>) -> Vec<Vec<String>>

Tokenize a batch of strings

fn untokenize(&self, tokens: Vec<String>) -> String

Untokenize a single string

fn batch_untokenize(&self, tokens: Vec<Vec<String>>) -> Vec<String>

Untokenize a batch of strings

Implementors

impl Tokenizer for AlphabetTokenizer

impl Tokenizer for BPETokenizer

impl Tokenizer for WhitespaceTokenizer

impl Tokenizer for WordpieceTokenizer