Skip to main content

prompt_token_counter/
lib.rs

1//! # prompt-token-counter
2//!
3//! Approximate token counts for LLM messages, system prompts, and tools.
4//!
5//! Most agent code needs to answer: "Will this request fit in the model's
6//! context window?" The honest answers are vendor-specific (tiktoken for
7//! OpenAI, Anthropic's counts API for Claude, sentencepiece for Llama).
8//! The cheap-and-acceptable answer is `chars/4`, which is within ~15% for
9//! English prose.
10//!
11//! [`TokenCounter`] wraps either approach behind one API.
12//!
13//! ## Quick example
14//!
15//! ```
16//! use prompt_token_counter::TokenCounter;
17//! use serde_json::json;
18//!
19//! let counter = TokenCounter::new();
20//! assert_eq!(counter.count("Hello world"), 3); // 11/4 ceil
21//!
22//! let messages = vec![
23//!     json!({"role": "system", "content": "You are helpful."}),
24//!     json!({"role": "user", "content": [{"type": "text", "text": "hi"}]}),
25//! ];
26//! let _ = counter.count_messages(&messages);
27//! ```
28//!
29//! ## BYO tokenizer
30//!
31//! ```
32//! use prompt_token_counter::TokenCounter;
33//! // Pretend tokenizer: one token per char.
34//! let counter = TokenCounter::new().with_tokenizer(|s: &str| s.chars().count());
35//! assert_eq!(counter.count("hello"), 5);
36//! ```
37//!
38//! ## Budget check
39//!
40//! ```
41//! use prompt_token_counter::{TokenCounter, FitsRequest, SystemSpec};
42//! use serde_json::json;
43//!
44//! let counter = TokenCounter::new();
45//! let result = counter.fits(FitsRequest {
46//!     context_window: 200_000,
47//!     system: Some(SystemSpec::Text("...".into())),
48//!     messages: &[json!({"role": "user", "content": "..."})],
49//!     tools: &[],
50//!     reserved_output: 4096,
51//! });
52//! assert!(result.fits);
53//! ```
54
55#![deny(missing_docs)]
56
57use serde_json::Value;
58
59/// Default token estimate: `ceil(len / 4)`. Within ~15% for English prose.
60pub fn chars_per_4_estimate(text: &str) -> usize {
61    if text.is_empty() {
62        return 0;
63    }
64    let n = text.len();
65    (n + 3) / 4
66}
67
68/// Outcome of [`TokenCounter::fits`].
69#[derive(Debug, Clone, PartialEq, Eq)]
70pub struct BudgetResult {
71    /// Whether `used <= context_window - reserved_output`.
72    pub fits: bool,
73    /// Total tokens summed across system + messages + tools.
74    pub used: usize,
75    /// `effective_budget - used`. Signed so callers can see how far over they are.
76    pub remaining: i64,
77    /// Echo of the input.
78    pub context_window: usize,
79    /// Echo of the input.
80    pub reserved_output: usize,
81}
82
83/// System-prompt shape passed to [`TokenCounter::fits`].
84///
85/// Anthropic accepts both a flat string and an array of content blocks;
86/// this enum mirrors that.
87pub enum SystemSpec {
88    /// Plain string system prompt.
89    Text(String),
90    /// Anthropic-shape content blocks.
91    Blocks(Vec<Value>),
92}
93
94/// Named-args bundle for [`TokenCounter::fits`].
95///
96/// All fields are inputs. `messages` and `tools` accept borrowed slices so the
97/// caller can keep ownership of large vectors.
98pub struct FitsRequest<'a> {
99    /// Model context window in tokens.
100    pub context_window: usize,
101    /// Optional system prompt (text or content-block list).
102    pub system: Option<SystemSpec>,
103    /// Conversation messages. Pass `&[]` if none.
104    pub messages: &'a [Value],
105    /// Tool definitions. Pass `&[]` if none.
106    pub tools: &'a [Value],
107    /// Tokens to subtract from `context_window` before checking fit.
108    pub reserved_output: usize,
109}
110
111type TokenizeFn = Box<dyn Fn(&str) -> usize + Send + Sync>;
112
113/// Approximate token counts for LLM messages, system prompts, and tools.
114///
115/// Construct with [`TokenCounter::new`], then chain builder methods.
116pub struct TokenCounter {
117    tokenizer: Option<TokenizeFn>,
118    per_message_overhead: u32,
119    per_tool_overhead: u32,
120}
121
122impl Default for TokenCounter {
123    fn default() -> Self {
124        Self::new()
125    }
126}
127
128impl std::fmt::Debug for TokenCounter {
129    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
130        f.debug_struct("TokenCounter")
131            .field("tokenizer", &self.tokenizer.as_ref().map(|_| "<fn>"))
132            .field("per_message_overhead", &self.per_message_overhead)
133            .field("per_tool_overhead", &self.per_tool_overhead)
134            .finish()
135    }
136}
137
138impl TokenCounter {
139    /// New counter with `chars/4` heuristic, 4 tokens per message, 8 per tool.
140    pub fn new() -> Self {
141        Self {
142            tokenizer: None,
143            per_message_overhead: 4,
144            per_tool_overhead: 8,
145        }
146    }
147
148    /// Install a BYO tokenizer. The closure receives a string slice and must
149    /// return the token count. Wrap tiktoken, Anthropic's count-tokens
150    /// endpoint, sentencepiece, etc.
151    pub fn with_tokenizer<F>(mut self, f: F) -> Self
152    where
153        F: Fn(&str) -> usize + Send + Sync + 'static,
154    {
155        self.tokenizer = Some(Box::new(f));
156        self
157    }
158
159    /// Override the per-message overhead (default 4).
160    pub fn with_per_message_overhead(mut self, n: u32) -> Self {
161        self.per_message_overhead = n;
162        self
163    }
164
165    /// Override the per-tool overhead (default 8).
166    pub fn with_per_tool_overhead(mut self, n: u32) -> Self {
167        self.per_tool_overhead = n;
168        self
169    }
170
171    /// Tokens in a single string. Routes through the installed tokenizer if
172    /// any, otherwise falls back to [`chars_per_4_estimate`].
173    pub fn count(&self, text: &str) -> usize {
174        if text.is_empty() {
175            return 0;
176        }
177        match &self.tokenizer {
178            Some(f) => f(text),
179            None => chars_per_4_estimate(text),
180        }
181    }
182
183    /// Tokens in a list of role+content messages.
184    ///
185    /// Handles plain string `content` and Anthropic-shape arrays of content
186    /// blocks (text, image, tool_use, tool_result, document). Each message
187    /// adds `per_message_overhead` tokens of framing.
188    pub fn count_messages(&self, messages: &[Value]) -> usize {
189        let mut total: usize = 0;
190        for msg in messages {
191            total += self.per_message_overhead as usize;
192            let content = msg.get("content");
193            match content {
194                Some(Value::String(s)) => total += self.count(s),
195                Some(Value::Array(blocks)) => {
196                    for block in blocks {
197                        total += self.count_block(block);
198                    }
199                }
200                Some(Value::Null) | None => {}
201                Some(other) => total += self.count(&value_repr(other)),
202            }
203        }
204        total
205    }
206
207    /// Tokens in a list of tool definitions.
208    ///
209    /// Walks `name`, `description`, and the JSON-serialized schema. Accepts
210    /// both Anthropic-shape `input_schema` and OpenAI-shape `parameters`.
211    /// Each tool adds `per_tool_overhead` tokens.
212    pub fn count_tools(&self, tools: &[Value]) -> usize {
213        let mut total: usize = 0;
214        for tool in tools {
215            total += self.per_tool_overhead as usize;
216            let name = tool
217                .get("name")
218                .and_then(Value::as_str)
219                .unwrap_or("");
220            let desc = tool
221                .get("description")
222                .and_then(Value::as_str)
223                .unwrap_or("");
224            // Anthropic uses `input_schema`; OpenAI uses `parameters`.
225            // Prefer input_schema; fall back to parameters; fall back to {}.
226            let schema_owner = tool
227                .get("input_schema")
228                .or_else(|| tool.get("parameters"))
229                .cloned()
230                .unwrap_or(Value::Object(Default::default()));
231            total += self.count(name);
232            total += self.count(desc);
233            let schema_str = serde_json::to_string(&schema_owner)
234                .unwrap_or_else(|_| value_repr(&schema_owner));
235            total += self.count(&schema_str);
236        }
237        total
238    }
239
240    /// Tally system + messages + tools and check against the budget.
241    ///
242    /// `reserved_output` is subtracted from `context_window` before the
243    /// check. `remaining` is `i64` so the caller can see how far over the
244    /// budget the request is when it doesn't fit.
245    pub fn fits(&self, req: FitsRequest<'_>) -> BudgetResult {
246        let FitsRequest {
247            context_window,
248            system,
249            messages,
250            tools,
251            reserved_output,
252        } = req;
253
254        let mut used: usize = 0;
255        match system {
256            Some(SystemSpec::Text(s)) => used += self.count(&s),
257            Some(SystemSpec::Blocks(blocks)) => {
258                for block in &blocks {
259                    used += self.count_block(block);
260                }
261            }
262            None => {}
263        }
264        used += self.count_messages(messages);
265        used += self.count_tools(tools);
266
267        let effective_budget = context_window.saturating_sub(reserved_output);
268        let remaining = effective_budget as i64 - used as i64;
269        BudgetResult {
270            fits: used <= effective_budget,
271            used,
272            remaining,
273            context_window,
274            reserved_output,
275        }
276    }
277
278    /// Count a single content block. Public for callers building custom
279    /// block streams; mirrors the dispatch used in `count_messages`.
280    fn count_block(&self, block: &Value) -> usize {
281        if let Value::String(s) = block {
282            return self.count(s);
283        }
284        let obj = match block.as_object() {
285            Some(o) => o,
286            None => return self.count(&value_repr(block)),
287        };
288        let btype = obj.get("type").and_then(Value::as_str);
289        match btype {
290            Some("text") => {
291                let t = obj.get("text").and_then(Value::as_str).unwrap_or("");
292                self.count(t)
293            }
294            // Flat overhead — real vendor cost is image-size-specific.
295            Some("image") => 256,
296            Some("tool_use") => {
297                let name = obj.get("name").and_then(Value::as_str).unwrap_or("");
298                let input = obj.get("input").cloned().unwrap_or(Value::Null);
299                let input_str = serde_json::to_string(&input)
300                    .unwrap_or_else(|_| value_repr(&input));
301                self.count(name) + self.count(&input_str) + 8
302            }
303            Some("tool_result") => {
304                let content = obj.get("content");
305                let inner = match content {
306                    Some(Value::String(s)) => self.count(s),
307                    Some(Value::Array(blocks)) => {
308                        blocks.iter().map(|b| self.count_block(b)).sum()
309                    }
310                    Some(Value::Null) | None => 0,
311                    Some(other) => self.count(&value_repr(other)),
312                };
313                inner + 4
314            }
315            // Placeholder flat overhead, like the Python lib.
316            Some("document") => 1024,
317            _ => self.count(&value_repr(block)),
318        }
319    }
320}
321
322/// Minimal repr for unknown JSON shapes. Mirrors Python's `repr(dict)`
323/// closely enough that `chars/4` lands in the same ballpark.
324fn value_repr(v: &Value) -> String {
325    v.to_string()
326}