1#[derive(Debug, Clone, PartialEq, Eq)]
6pub struct ChatMessage {
7 pub role: String,
8 pub content: String,
9}
10
11pub trait TokenCounter {
18 fn id(&self) -> &str;
19 fn revision(&self) -> &str;
20 fn count(&self, text: &str) -> u32;
21 fn count_parts(&self, parts: &[&str]) -> u32 {
22 parts
23 .iter()
24 .map(|part| self.count(part))
25 .fold(0_u32, u32::saturating_add)
26 }
27 fn count_message(&self, message: &ChatMessage) -> u32 {
28 self.count_parts(&[&message.role, &message.content])
29 }
30 fn count_tool_schema(&self, schema: &blazingly_json::Value) -> u32 {
31 self.count(&schema.to_string())
32 }
33}
34
35#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
40pub struct CharDiv4Counter;
41
42impl TokenCounter for CharDiv4Counter {
43 fn id(&self) -> &'static str {
44 "char_div4"
45 }
46
47 fn revision(&self) -> &'static str {
48 "v1"
49 }
50
51 fn count(&self, text: &str) -> u32 {
52 let tokens = text.chars().count().div_ceil(4).max(1);
53 u32::try_from(tokens).unwrap_or(u32::MAX)
54 }
55}
56
57#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
63pub struct ConservativeCounter;
64
65impl TokenCounter for ConservativeCounter {
66 fn id(&self) -> &'static str {
67 "conservative"
68 }
69
70 fn revision(&self) -> &'static str {
71 "v1"
72 }
73
74 fn count(&self, text: &str) -> u32 {
75 let mut tokens = 0_u32;
76 let mut ascii_run = 0_u32;
77 for character in text.chars() {
78 if character.is_whitespace() {
79 flush_ascii(&mut tokens, &mut ascii_run);
80 continue;
81 }
82 if character.is_ascii_alphanumeric() {
83 ascii_run = ascii_run.saturating_add(1);
84 continue;
85 }
86 flush_ascii(&mut tokens, &mut ascii_run);
87 tokens = tokens.saturating_add(1);
88 }
89 flush_ascii(&mut tokens, &mut ascii_run);
90 tokens.max(1)
91 }
92}
93
94fn flush_ascii(tokens: &mut u32, ascii_run: &mut u32) {
95 if *ascii_run == 0 {
96 return;
97 }
98 *tokens = tokens.saturating_add(ascii_run.div_ceil(3));
99 *ascii_run = 0;
100}
101
102#[must_use]
105pub fn estimate_tokens(value: &str) -> u32 {
106 CharDiv4Counter.count(value)
107}
108
109pub const RUNTIME_COUNTER: ConservativeCounter = ConservativeCounter;
110
111#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
112#[serde(rename_all = "camelCase")]
113pub struct TokenBreakdown {
114 pub counter_id: String,
115 pub tokenizer_revision: String,
116 pub candidate_tokens: u32,
118 pub selected_before_dedup_tokens: u32,
120 pub delivered_tokens: u32,
122 pub budget_omitted_tokens: u32,
124 pub dedup_saved_tokens: u32,
126 pub rendering_overhead_tokens: u32,
128 #[serde(default)]
131 pub estimated_tokens: u32,
132 #[serde(default, skip_serializing_if = "Option::is_none")]
134 pub actual_provider_tokens: Option<u32>,
135 #[serde(default, skip_serializing_if = "Option::is_none")]
137 pub estimation_error: Option<i32>,
138 #[serde(default)]
140 pub wire_tokens: u32,
141 #[serde(default)]
143 pub schema_tokens: u32,
144 #[serde(default)]
146 pub instruction_tokens: u32,
147 #[serde(default)]
149 pub evidence_tokens: u32,
150 #[serde(default)]
152 pub output_tokens: u32,
153}
154
155impl TokenBreakdown {
156 #[must_use]
157 pub fn for_counter(counter: &dyn TokenCounter) -> Self {
158 Self {
159 counter_id: counter.id().to_owned(),
160 tokenizer_revision: counter.revision().to_owned(),
161 candidate_tokens: 0,
162 selected_before_dedup_tokens: 0,
163 delivered_tokens: 0,
164 budget_omitted_tokens: 0,
165 dedup_saved_tokens: 0,
166 rendering_overhead_tokens: 0,
167 estimated_tokens: 0,
168 actual_provider_tokens: None,
169 estimation_error: None,
170 wire_tokens: 0,
171 schema_tokens: 0,
172 instruction_tokens: 0,
173 evidence_tokens: 0,
174 output_tokens: 0,
175 }
176 }
177}
178
179#[cfg(test)]
180mod tests {
181 use super::*;
182
183 #[test]
184 fn conservative_overestimates_cyrillic_against_char_div4() {
185 let text = "Предыдущая попытка молча пропустила архив";
186 assert!(
187 ConservativeCounter.count(text) > CharDiv4Counter.count(text),
188 "cyrillic must not be treated as four chars per token"
189 );
190 }
191
192 #[test]
193 fn two_russian_sentences_do_not_share_a_div4_cost_with_code() {
194 let rust = "fn finish_block()";
195 let russian = "предыдущая попытка";
196 assert_eq!(CharDiv4Counter.count(rust), CharDiv4Counter.count(russian));
197 assert!(ConservativeCounter.count(russian) > ConservativeCounter.count(rust));
198 }
199
200 #[test]
201 fn message_and_schema_counts_are_conservative() {
202 let message = ChatMessage {
203 role: "user".to_owned(),
204 content: "предыдущая попытка".to_owned(),
205 };
206 assert!(
207 ConservativeCounter.count_message(&message) > CharDiv4Counter.count_message(&message)
208 );
209 let schema = blazingly_json::json!({
210 "type": "object",
211 "properties": { "id": { "type": "string" } }
212 });
213 assert!(ConservativeCounter.count_tool_schema(&schema) >= 1);
214 }
215}