Trait Tokenizer 

pub trait Tokenizer: Send + Sync {
    // Required method
    fn count_tokens(&self, text: &str) -> usize;
}

Token counter abstraction

Required Methods

fn count_tokens(&self, text: &str) -> usize

Estimate how many model tokens the input text will consume.

Trait Implementations

impl Tokenizer for Box<dyn Tokenizer>

fn count_tokens(&self, text: &str) -> usize

Estimate how many model tokens the input text will consume.

Implementations on Foreign Types

impl Tokenizer for Box<dyn Tokenizer>

fn count_tokens(&self, text: &str) -> usize

Implementors

impl Tokenizer for HeuristicTokenizer

impl Tokenizer for SimpleTokenizer