Trait TokenCounter 

pub trait TokenCounter: Send + Sync {
    // Required methods
    fn count(&self, text: &str) -> usize;
    fn tokenize(&self, text: &str) -> Vec<u32>;
    fn decode(&self, tokens: &[u32]) -> String;
    fn model(&self) -> TokenizerModel;
    fn is_exact(&self) -> bool;

    // Provided method
    fn count_for_model(&self, text: &str, model: TokenizerModel) -> usize { ... }
}

Trait for token counting implementations

Required Methods

fn count(&self, text: &str) -> usize

Count tokens in text

fn tokenize(&self, text: &str) -> Vec<u32>

Tokenize text (returns token IDs)

fn decode(&self, tokens: &[u32]) -> String

Decode tokens back to text

fn model(&self) -> TokenizerModel

Get the model being used

fn is_exact(&self) -> bool

Check if this counter uses exact tokenization

Provided Methods

fn count_for_model(&self, text: &str, model: TokenizerModel) -> usize

Count tokens with model hint

Implementors

impl TokenCounter for ExactTokenCounter

impl TokenCounter for HeuristicTokenCounter