Skip to main content

rig_memory/
tokens.rs

1//! Estimating how many tokens a message, and a request built to send
2//! ([`request_estimate`]), cost.
3
4use rig_core::completion::{CompletionRequest, Message, ToolDefinition, Usage};
5use rig_core::conversation::MessageId;
6use rig_core::json_utils::json_len;
7use serde::{Deserialize, Serialize};
8
9/// Estimates token cost from UTF-8 lengths, rounded up per text-bearing part.
10/// Tool calls count name bytes plus serialized argument bytes. Attachments and
11/// message overhead have fixed costs.
12///
13/// ```
14/// use rig_core::completion::Message;
15/// use rig_memory::HeuristicTokenCounter;
16///
17/// let counter = HeuristicTokenCounter::default();
18/// assert_eq!(counter.count(&Message::user("12345678")), 2 + 4);
19/// ```
20#[derive(Debug, Clone, Copy)]
21pub struct HeuristicTokenCounter {
22    bytes_per_token: f32,
23    per_message_overhead: usize,
24    per_attachment_tokens: usize,
25}
26
27impl HeuristicTokenCounter {
28    fn bytes_to_tokens(&self, bytes: usize) -> usize {
29        // Rounded up, so a non-empty input costs at least one token.
30        let tokens = (bytes as f32) / self.bytes_per_token;
31        tokens.ceil() as usize
32    }
33
34    /// About how many tokens the definition of a tool costs as a request
35    /// sends it: its name, description and parameters' JSON.
36    pub fn count_tool(&self, tool: &ToolDefinition) -> usize {
37        let bytes = tool.name.len().saturating_add(tool.description.len());
38        self.bytes_to_tokens(bytes.saturating_add(json_len(&tool.parameters)))
39            .saturating_add(self.per_message_overhead)
40    }
41
42    /// About how many tokens one item of a user message costs, without the
43    /// per-message overhead.
44    pub(crate) fn count_user(&self, content: &rig_core::message::UserContent) -> usize {
45        use rig_core::message::UserContent;
46        match content {
47            UserContent::Text(text) => self.bytes_to_tokens(text.text.len()),
48            UserContent::ToolResult(result) => result
49                .content
50                .iter()
51                .map(|c| match c {
52                    rig_core::message::ToolResultContent::Text(t) => {
53                        self.bytes_to_tokens(t.text.len())
54                    }
55                    rig_core::message::ToolResultContent::Json { value } => {
56                        self.bytes_to_tokens(json_len(value))
57                    }
58                    rig_core::message::ToolResultContent::Image(_) => self.per_attachment_tokens,
59                })
60                .fold(0, usize::saturating_add),
61            UserContent::Image(_)
62            | UserContent::Audio(_)
63            | UserContent::Video(_)
64            | UserContent::Document(_) => self.per_attachment_tokens,
65        }
66    }
67
68    /// About how many tokens one item of an assistant message costs, without
69    /// the per-message overhead.
70    fn count_assistant(&self, content: &rig_core::message::AssistantContent) -> usize {
71        use rig_core::message::AssistantContent;
72        match content {
73            AssistantContent::Text(text) => self.bytes_to_tokens(text.text.len()),
74            AssistantContent::Reasoning(reasoning) => self.bytes_to_tokens(reasoning.text.len()),
75            AssistantContent::ToolCall(call) => {
76                let name_bytes = call.function.name.len();
77                let args_bytes = json_len(&call.function.arguments);
78                self.bytes_to_tokens(name_bytes + args_bytes)
79            }
80            AssistantContent::Image(_) => self.per_attachment_tokens,
81            // A provider-only item is sent back to the model that made it.
82            AssistantContent::Opaque(opaque) => self.bytes_to_tokens(json_len(&opaque.item)),
83            other => self.bytes_to_tokens(json_len(other)),
84        }
85    }
86}
87
88/// About how many tokens an image, or another attachment, costs.
89pub const ATTACHMENT_TOKENS: usize = 1_200;
90
91impl Default for HeuristicTokenCounter {
92    /// Uses four bytes per token, four tokens per message, and
93    /// [`ATTACHMENT_TOKENS`] per attachment.
94    fn default() -> Self {
95        Self {
96            bytes_per_token: 4.0,
97            per_message_overhead: 4,
98            per_attachment_tokens: ATTACHMENT_TOKENS,
99        }
100    }
101}
102
103impl HeuristicTokenCounter {
104    /// About how many tokens `message` costs.
105    pub fn count(&self, message: &Message) -> usize {
106        let content_tokens: usize = match message {
107            Message::User { content } => content
108                .iter()
109                .map(|c| self.count_user(c))
110                .fold(0, usize::saturating_add),
111            Message::Assistant(turn) => turn
112                .content
113                .iter()
114                .map(|c| self.count_assistant(c))
115                .fold(0, usize::saturating_add),
116            Message::System { content } => self.bytes_to_tokens(content.len()),
117        };
118        content_tokens.saturating_add(self.per_message_overhead)
119    }
120}
121
122/// The usage a provider reported for an agent's last reply, and the last
123/// entry of the conversation it counts: the reply, or the request's last
124/// message when the reply added none. It anchors the estimate of the next
125/// request: the reported tokens plus an estimate of the messages after it.
126#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
127pub struct Anchor {
128    /// What the provider reported. Its input counts every token the
129    /// request sent, the tool definitions included.
130    pub usage: Usage,
131    /// The last entry it counts.
132    pub through: MessageId,
133}
134
135/// The estimate of a request sent as it is: its messages and tools, by the
136/// default [`HeuristicTokenCounter`]. Its documents are not counted.
137///
138/// ```
139/// use rig_core::completion::{CompletionRequest, Message};
140/// use rig_memory::request_estimate;
141///
142/// let request = CompletionRequest::new(Message::user("12345678"));
143/// assert_eq!(request_estimate(&request), 2 + 4);
144/// ```
145pub fn request_estimate(request: &CompletionRequest) -> u64 {
146    let counter = HeuristicTokenCounter::default();
147    let tools = request.tools.iter().map(|tool| counter.count_tool(tool));
148    let messages = request
149        .chat_history
150        .iter()
151        .map(|message| counter.count(message));
152    tools.chain(messages).fold(0, usize::saturating_add) as u64
153}
154
155#[cfg(test)]
156mod tests;