Trait Tokenizer 

pub trait Tokenizer: Send + Sync {
    // Required methods
    fn train(&mut self, corpus: &[&str]) -> Result<()>;
    fn encode(&self, text: &str) -> Result<Vec<TokenId>>;
    fn decode(&self, ids: &[TokenId]) -> Result<String>;
    fn vocab_size(&self) -> usize;
    fn is_trained(&self) -> bool;
    fn id_to_token(&self, id: TokenId) -> Option<&str>;
    fn token_to_id(&self, token: &str) -> Option<TokenId>;
}

Tokenizer trait

Required Methods

fn train(&mut self, corpus: &[&str]) -> Result<()>

Train the tokenizer on a corpus

fn encode(&self, text: &str) -> Result<Vec<TokenId>>

Encode text to token IDs

fn decode(&self, ids: &[TokenId]) -> Result<String>

Decode token IDs to text

fn vocab_size(&self) -> usize

Get vocabulary size

fn is_trained(&self) -> bool

Check if tokenizer is trained

fn id_to_token(&self, id: TokenId) -> Option<&str>

Get token for ID

fn token_to_id(&self, token: &str) -> Option<TokenId>

Get ID for token

Implementors

impl Tokenizer for BPETokenizer

impl Tokenizer for CharTokenizer