Struct Tokenizer 

pub struct Tokenizer { /* private fields */ }

Tokenizer for counting and encoding tokens

Implementations

impl Tokenizer

pub fn new(model: TokenizerModel) -> Result<Self>

Create a new tokenizer with the specified model

pub fn default_model() -> Result<Self>

Create a tokenizer with the default model (cl100k_base)

pub fn model(&self) -> TokenizerModel

Get the tokenizer model

pub fn count_tokens(&self, text: &str) -> usize

Count the number of tokens in the text

pub fn encode(&self, text: &str) -> Vec<u32>

Encode text into token IDs

pub fn decode(&self, tokens: &[u32]) -> Result<String>

Decode token IDs back to text

pub fn truncate(&self, text: &str, max_tokens: usize) -> Result<String>

Truncate text to a maximum number of tokens

pub fn exceeds_limit(&self, text: &str, limit: usize) -> bool

Check if text exceeds a token limit

impl Tokenizer

pub fn stats(&self, text: &str) -> TokenStats

Calculate token statistics for text

Trait Implementations

impl Debug for Tokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for Tokenizer

fn default() -> Self

Returns the “default value” for a type.