Struct TokenCounter 

pub struct TokenCounter { /* private fields */ }

Implementations

impl TokenCounter

pub fn new() -> Result<Self>

pub fn from_model(model_name: &str) -> Result<Self>

pub fn model_name(&self) -> &str

pub fn count_tokens(&self, content: &str) -> usize

pub fn count_tokens_parallel<'a, I>(&self, contents: I) -> Vec<usize>

where I: ParallelIterator<Item = &'a str>,

pub fn analyze_batch<'a, I>(&self, contents: I) -> (usize, f64)

where I: ParallelIterator<Item = &'a str>,

Trait Implementations

impl Default for TokenCounter

fn default() -> Self

Returns the “default value” for a type.