llm-token 0.1.0

Token counting and handling for LLM messages
Documentation 
1
2
3
4
5
6
7
8
9

// Token sizing: pure text estimation and provider-shaped request counting.
pub mod count;
pub mod estimation;

pub use count::{
    estimate_image_tokens, estimate_message_tokens, estimate_messages, estimate_request_tokens,
    estimate_tool_declarations,
};
pub use estimation::{estimate_tokens, TokenEstimator};