supercode_interchange/
token_estimator.rs

1//! Provider-independent token estimates over canonical session messages.
2//!
3//! This module is deliberately smaller than `tokens`: request guards that
4//! account for provider tool schemas remain runtime concerns, while reduction
5//! and interchange only need deterministic text/message estimates.
6
7use crate::message::ChatMessage;
8
9/// Deterministic token estimate: `ceil(utf8_bytes / 4)`.
10pub fn estimate_tokens(value: &str) -> u64 {
11    (value.len() as u64).div_ceil(4)
12}
13
14/// Estimate the serialized wire size of canonical messages.
15pub fn estimate_view_tokens(messages: &[ChatMessage]) -> u64 {
16    messages.iter().map(estimate_message_tokens).sum()
17}
18
19fn estimate_message_tokens(message: &ChatMessage) -> u64 {
20    match serde_json::to_string(message) {
21        Ok(wire) => estimate_tokens(&wire),
22        Err(_) => estimate_tokens(&format!("{message:?}")),
23    }
24}
25
26/// Render `n` with comma thousands separators.
27pub fn format_commas(n: usize) -> String {
28    let digits = n.to_string();
29    let bytes = digits.as_bytes();
30    let mut output = String::with_capacity(bytes.len() + bytes.len() / 3);
31    for (index, byte) in bytes.iter().enumerate() {
32        if index > 0 && (bytes.len() - index) % 3 == 0 {
33            output.push(',');
34        }
35        output.push(*byte as char);
36    }
37    output
38}
supercode_interchange/token_estimator.rs

supercode_interchange/
token_estimator.rs
