Struct TokenEstimatorConfig 

pub struct TokenEstimatorConfig {
    pub int_factor: f32,
    pub float_factor: f32,
    pub string_factor: f32,
    pub hex_factor: f32,
    pub bytes_per_token: f32,
    pub separator_tokens: usize,
    pub newline_tokens: usize,
    pub header_tokens: usize,
}

Token estimation configuration

Fields

int_factor: f32

Multiplier for integer values

float_factor: f32

Multiplier for float values

string_factor: f32

Multiplier for string values

hex_factor: f32

Multiplier for binary (hex) values

bytes_per_token: f32

Bytes per token (approximate)

separator_tokens: usize

Separator cost in tokens

newline_tokens: usize

Newline cost in tokens

header_tokens: usize

Header overhead tokens

Implementations

impl TokenEstimatorConfig

pub fn gpt4() -> Self

Create config tuned for GPT-4 tokenizer

pub fn claude() -> Self

Create config tuned for Claude tokenizer

pub fn conservative() -> Self

Create config with high precision (conservative)

Trait Implementations

impl Clone for TokenEstimatorConfig

fn clone(&self) -> TokenEstimatorConfig

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenEstimatorConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for TokenEstimatorConfig

fn default() -> Self

Returns the “default value” for a type.