prompt_token_counter/lib.rs
1//! # prompt-token-counter
2//!
3//! Approximate token counts for LLM messages, system prompts, and tools.
4//!
5//! Most agent code needs to answer: "Will this request fit in the model's
6//! context window?" The honest answers are vendor-specific (tiktoken for
7//! OpenAI, Anthropic's counts API for Claude, sentencepiece for Llama).
8//! The cheap-and-acceptable answer is `chars/4`, which is within ~15% for
9//! English prose.
10//!
11//! [`TokenCounter`] wraps either approach behind one API.
12//!
13//! ## Quick example
14//!
15//! ```
16//! use prompt_token_counter::TokenCounter;
17//! use serde_json::json;
18//!
19//! let counter = TokenCounter::new();
20//! assert_eq!(counter.count("Hello world"), 3); // 11/4 ceil
21//!
22//! let messages = vec![
23//! json!({"role": "system", "content": "You are helpful."}),
24//! json!({"role": "user", "content": [{"type": "text", "text": "hi"}]}),
25//! ];
26//! let _ = counter.count_messages(&messages);
27//! ```
28//!
29//! ## BYO tokenizer
30//!
31//! ```
32//! use prompt_token_counter::TokenCounter;
33//! // Pretend tokenizer: one token per char.
34//! let counter = TokenCounter::new().with_tokenizer(|s: &str| s.chars().count());
35//! assert_eq!(counter.count("hello"), 5);
36//! ```
37//!
38//! ## Budget check
39//!
40//! ```
41//! use prompt_token_counter::{TokenCounter, FitsRequest, SystemSpec};
42//! use serde_json::json;
43//!
44//! let counter = TokenCounter::new();
45//! let result = counter.fits(FitsRequest {
46//! context_window: 200_000,
47//! system: Some(SystemSpec::Text("...".into())),
48//! messages: &[json!({"role": "user", "content": "..."})],
49//! tools: &[],
50//! reserved_output: 4096,
51//! });
52//! assert!(result.fits);
53//! ```
54
55#![deny(missing_docs)]
56
57use serde_json::Value;
58
59/// Default token estimate: `ceil(len / 4)`. Within ~15% for English prose.
60pub fn chars_per_4_estimate(text: &str) -> usize {
61 if text.is_empty() {
62 return 0;
63 }
64 let n = text.len();
65 (n + 3) / 4
66}
67
68/// Outcome of [`TokenCounter::fits`].
69#[derive(Debug, Clone, PartialEq, Eq)]
70pub struct BudgetResult {
71 /// Whether `used <= context_window - reserved_output`.
72 pub fits: bool,
73 /// Total tokens summed across system + messages + tools.
74 pub used: usize,
75 /// `effective_budget - used`. Signed so callers can see how far over they are.
76 pub remaining: i64,
77 /// Echo of the input.
78 pub context_window: usize,
79 /// Echo of the input.
80 pub reserved_output: usize,
81}
82
83/// System-prompt shape passed to [`TokenCounter::fits`].
84///
85/// Anthropic accepts both a flat string and an array of content blocks;
86/// this enum mirrors that.
87pub enum SystemSpec {
88 /// Plain string system prompt.
89 Text(String),
90 /// Anthropic-shape content blocks.
91 Blocks(Vec<Value>),
92}
93
94/// Named-args bundle for [`TokenCounter::fits`].
95///
96/// All fields are inputs. `messages` and `tools` accept borrowed slices so the
97/// caller can keep ownership of large vectors.
98pub struct FitsRequest<'a> {
99 /// Model context window in tokens.
100 pub context_window: usize,
101 /// Optional system prompt (text or content-block list).
102 pub system: Option<SystemSpec>,
103 /// Conversation messages. Pass `&[]` if none.
104 pub messages: &'a [Value],
105 /// Tool definitions. Pass `&[]` if none.
106 pub tools: &'a [Value],
107 /// Tokens to subtract from `context_window` before checking fit.
108 pub reserved_output: usize,
109}
110
111type TokenizeFn = Box<dyn Fn(&str) -> usize + Send + Sync>;
112
113/// Approximate token counts for LLM messages, system prompts, and tools.
114///
115/// Construct with [`TokenCounter::new`], then chain builder methods.
116pub struct TokenCounter {
117 tokenizer: Option<TokenizeFn>,
118 per_message_overhead: u32,
119 per_tool_overhead: u32,
120}
121
122impl Default for TokenCounter {
123 fn default() -> Self {
124 Self::new()
125 }
126}
127
128impl std::fmt::Debug for TokenCounter {
129 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
130 f.debug_struct("TokenCounter")
131 .field("tokenizer", &self.tokenizer.as_ref().map(|_| "<fn>"))
132 .field("per_message_overhead", &self.per_message_overhead)
133 .field("per_tool_overhead", &self.per_tool_overhead)
134 .finish()
135 }
136}
137
138impl TokenCounter {
139 /// New counter with `chars/4` heuristic, 4 tokens per message, 8 per tool.
140 pub fn new() -> Self {
141 Self {
142 tokenizer: None,
143 per_message_overhead: 4,
144 per_tool_overhead: 8,
145 }
146 }
147
148 /// Install a BYO tokenizer. The closure receives a string slice and must
149 /// return the token count. Wrap tiktoken, Anthropic's count-tokens
150 /// endpoint, sentencepiece, etc.
151 pub fn with_tokenizer<F>(mut self, f: F) -> Self
152 where
153 F: Fn(&str) -> usize + Send + Sync + 'static,
154 {
155 self.tokenizer = Some(Box::new(f));
156 self
157 }
158
159 /// Override the per-message overhead (default 4).
160 pub fn with_per_message_overhead(mut self, n: u32) -> Self {
161 self.per_message_overhead = n;
162 self
163 }
164
165 /// Override the per-tool overhead (default 8).
166 pub fn with_per_tool_overhead(mut self, n: u32) -> Self {
167 self.per_tool_overhead = n;
168 self
169 }
170
171 /// Tokens in a single string. Routes through the installed tokenizer if
172 /// any, otherwise falls back to [`chars_per_4_estimate`].
173 pub fn count(&self, text: &str) -> usize {
174 if text.is_empty() {
175 return 0;
176 }
177 match &self.tokenizer {
178 Some(f) => f(text),
179 None => chars_per_4_estimate(text),
180 }
181 }
182
183 /// Tokens in a list of role+content messages.
184 ///
185 /// Handles plain string `content` and Anthropic-shape arrays of content
186 /// blocks (text, image, tool_use, tool_result, document). Each message
187 /// adds `per_message_overhead` tokens of framing.
188 pub fn count_messages(&self, messages: &[Value]) -> usize {
189 let mut total: usize = 0;
190 for msg in messages {
191 total += self.per_message_overhead as usize;
192 let content = msg.get("content");
193 match content {
194 Some(Value::String(s)) => total += self.count(s),
195 Some(Value::Array(blocks)) => {
196 for block in blocks {
197 total += self.count_block(block);
198 }
199 }
200 Some(Value::Null) | None => {}
201 Some(other) => total += self.count(&value_repr(other)),
202 }
203 }
204 total
205 }
206
207 /// Tokens in a list of tool definitions.
208 ///
209 /// Walks `name`, `description`, and the JSON-serialized schema. Accepts
210 /// both Anthropic-shape `input_schema` and OpenAI-shape `parameters`.
211 /// Each tool adds `per_tool_overhead` tokens.
212 pub fn count_tools(&self, tools: &[Value]) -> usize {
213 let mut total: usize = 0;
214 for tool in tools {
215 total += self.per_tool_overhead as usize;
216 let name = tool
217 .get("name")
218 .and_then(Value::as_str)
219 .unwrap_or("");
220 let desc = tool
221 .get("description")
222 .and_then(Value::as_str)
223 .unwrap_or("");
224 // Anthropic uses `input_schema`; OpenAI uses `parameters`.
225 // Prefer input_schema; fall back to parameters; fall back to {}.
226 let schema_owner = tool
227 .get("input_schema")
228 .or_else(|| tool.get("parameters"))
229 .cloned()
230 .unwrap_or(Value::Object(Default::default()));
231 total += self.count(name);
232 total += self.count(desc);
233 let schema_str = serde_json::to_string(&schema_owner)
234 .unwrap_or_else(|_| value_repr(&schema_owner));
235 total += self.count(&schema_str);
236 }
237 total
238 }
239
240 /// Tally system + messages + tools and check against the budget.
241 ///
242 /// `reserved_output` is subtracted from `context_window` before the
243 /// check. `remaining` is `i64` so the caller can see how far over the
244 /// budget the request is when it doesn't fit.
245 pub fn fits(&self, req: FitsRequest<'_>) -> BudgetResult {
246 let FitsRequest {
247 context_window,
248 system,
249 messages,
250 tools,
251 reserved_output,
252 } = req;
253
254 let mut used: usize = 0;
255 match system {
256 Some(SystemSpec::Text(s)) => used += self.count(&s),
257 Some(SystemSpec::Blocks(blocks)) => {
258 for block in &blocks {
259 used += self.count_block(block);
260 }
261 }
262 None => {}
263 }
264 used += self.count_messages(messages);
265 used += self.count_tools(tools);
266
267 let effective_budget = context_window.saturating_sub(reserved_output);
268 let remaining = effective_budget as i64 - used as i64;
269 BudgetResult {
270 fits: used <= effective_budget,
271 used,
272 remaining,
273 context_window,
274 reserved_output,
275 }
276 }
277
278 /// Count a single content block. Public for callers building custom
279 /// block streams; mirrors the dispatch used in `count_messages`.
280 fn count_block(&self, block: &Value) -> usize {
281 if let Value::String(s) = block {
282 return self.count(s);
283 }
284 let obj = match block.as_object() {
285 Some(o) => o,
286 None => return self.count(&value_repr(block)),
287 };
288 let btype = obj.get("type").and_then(Value::as_str);
289 match btype {
290 Some("text") => {
291 let t = obj.get("text").and_then(Value::as_str).unwrap_or("");
292 self.count(t)
293 }
294 // Flat overhead — real vendor cost is image-size-specific.
295 Some("image") => 256,
296 Some("tool_use") => {
297 let name = obj.get("name").and_then(Value::as_str).unwrap_or("");
298 let input = obj.get("input").cloned().unwrap_or(Value::Null);
299 let input_str = serde_json::to_string(&input)
300 .unwrap_or_else(|_| value_repr(&input));
301 self.count(name) + self.count(&input_str) + 8
302 }
303 Some("tool_result") => {
304 let content = obj.get("content");
305 let inner = match content {
306 Some(Value::String(s)) => self.count(s),
307 Some(Value::Array(blocks)) => {
308 blocks.iter().map(|b| self.count_block(b)).sum()
309 }
310 Some(Value::Null) | None => 0,
311 Some(other) => self.count(&value_repr(other)),
312 };
313 inner + 4
314 }
315 // Placeholder flat overhead, like the Python lib.
316 Some("document") => 1024,
317 _ => self.count(&value_repr(block)),
318 }
319 }
320}
321
322/// Minimal repr for unknown JSON shapes. Mirrors Python's `repr(dict)`
323/// closely enough that `chars/4` lands in the same ballpark.
324fn value_repr(v: &Value) -> String {
325 v.to_string()
326}