Struct TokenEstimator 

pub struct TokenEstimator { /* private fields */ }

Token estimator

Implementations

impl TokenEstimator

pub fn new() -> Self

Create a new estimator with default config

pub fn with_config(config: TokenEstimatorConfig) -> Self

Create with custom config

pub fn estimate_value(&self, value: &SochValue) -> usize

Estimate tokens for a single value.

Applies safety_margin to the raw estimate to reduce the risk of exceeding the LLM context window with non-Latin or structured text.

pub fn estimate_row(&self, values: &[SochValue]) -> usize

Estimate tokens for a row (multiple values)

pub fn estimate_header( &self, table: &str, columns: &[String], row_count: usize, ) -> usize

Estimate tokens for a table header

pub fn estimate_table( &self, table: &str, columns: &[String], rows: &[Vec<SochValue>], ) -> usize

Estimate tokens for a complete TOON table

pub fn estimate_text(&self, text: &str) -> usize

Estimate tokens for plain text

pub fn truncate_to_tokens(&self, text: &str, max_tokens: usize) -> String

Truncate text to fit within token budget

Uses binary search to find the optimal truncation point.

Trait Implementations

impl Default for TokenEstimator

fn default() -> Self

Returns the “default value” for a type.