Struct HeuristicTokenCounter 

pub struct HeuristicTokenCounter { /* private fields */ }

Fast heuristic-based token counter

Implementations

impl HeuristicTokenCounter

pub fn new() -> Self

Create with default settings

pub fn for_model(model: TokenizerModel) -> Self

Create for specific model

Trait Implementations

impl Default for HeuristicTokenCounter

fn default() -> Self

Returns the “default value” for a type.

impl TokenCounter for HeuristicTokenCounter

fn count(&self, text: &str) -> usize

Count tokens in text

fn tokenize(&self, text: &str) -> Vec<u32>

Tokenize text (returns token IDs)

fn decode(&self, _tokens: &[u32]) -> String

Decode tokens back to text

fn model(&self) -> TokenizerModel

Get the model being used

fn is_exact(&self) -> bool

Check if this counter uses exact tokenization

fn count_for_model(&self, text: &str, model: TokenizerModel) -> usize

Count tokens with model hint