Struct TokenEstimator 

pub struct TokenEstimator { /* private fields */ }

Token estimator for mixed-language text

Implementations

impl TokenEstimator

pub const fn new(cjk_factor: f32, latin_factor: f32) -> Self

Create a new token estimator with custom factors

Example

use llm_token::estimation::TokenEstimator;

let estimator = TokenEstimator::new(0.8, 0.3);

pub fn estimate(text: &str) -> usize

Estimate token count for text using default factors

Example

use llm_token::estimation::TokenEstimator;

let tokens = TokenEstimator::estimate("Hello world");

pub fn estimate_with_config(&self, text: &str) -> usize

Estimate token count for text with this estimator’s configuration

Uses fast path for ASCII-only text, otherwise processes character by character.

pub fn estimate_text(&self, text: &str) -> usize

Estimate tokens for text (instance method)

Convenience method that delegates to estimate_with_config.

pub fn is_cjk(ch: char) -> bool

Check if character is CJK (Chinese, Japanese, Korean)

pub fn find_split_point(&self, text: &str, max_tokens: usize) -> usize

Find split point that fits within token limit

Returns the byte offset where to split.

Trait Implementations

impl Clone for TokenEstimator

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Copy for TokenEstimator

impl Debug for TokenEstimator

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for TokenEstimator

fn default() -> Self

Returns the “default value” for a type.