Struct TokenCounter 

pub struct TokenCounter { /* private fields */ }

Token counter with caching and batch support

For repeated counting with the same encoding, this struct provides a cleaner interface than the free functions.

Example

use m2m::tokenizer::TokenCounter;
use m2m::models::Encoding;

let counter = TokenCounter::new(Encoding::O200kBase);

let tokens1 = counter.count("Hello, world!");
let tokens2 = counter.count("Another message");
let total = counter.count_many(&["Hello", "World"]);

Implementations

impl TokenCounter

pub fn new(encoding: Encoding) -> Self

Create a new token counter with the specified encoding

pub fn default_encoding() -> Self

Create a token counter for the default encoding (cl100k_base)

pub fn for_model(model: &str) -> Self

Create a token counter for a specific model

pub fn count(&self, text: &str) -> usize

Count tokens in text

pub fn count_many(&self, texts: &[&str]) -> usize

Count tokens in multiple texts

pub fn count_json(&self, value: &Value) -> usize

Count tokens in JSON value (serialized)

pub fn encoding(&self) -> Encoding

Get the encoding used by this counter

Trait Implementations

impl Default for TokenCounter

fn default() -> Self

Returns the “default value” for a type.