Skip to main content

cortex_context/
tokens.rs

1//! Runtime token counting. The four-character rule is a comparative unit,
2//! not a safety bound.
3
4/// One chat turn for [`TokenCounter::count_message`].
5#[derive(Debug, Clone, PartialEq, Eq)]
6pub struct ChatMessage {
7    pub role: String,
8    pub content: String,
9}
10
11/// Counts tokens for a budget. Implementations name themselves so a ledger
12/// can say which counter produced a number.
13///
14/// Vendor tokenizers (OpenAI/Codex, Hugging Face JSON for Qwen, a
15/// Claude-calibrated table) plug in here. Runtime compile uses
16/// [`ConservativeCounter`] until one of those is wired for the active model.
17pub trait TokenCounter {
18    fn id(&self) -> &str;
19    fn revision(&self) -> &str;
20    fn count(&self, text: &str) -> u32;
21    fn count_parts(&self, parts: &[&str]) -> u32 {
22        parts
23            .iter()
24            .map(|part| self.count(part))
25            .fold(0_u32, u32::saturating_add)
26    }
27    fn count_message(&self, message: &ChatMessage) -> u32 {
28        self.count_parts(&[&message.role, &message.content])
29    }
30    fn count_tool_schema(&self, schema: &blazingly_json::Value) -> u32 {
31        self.count(&schema.to_string())
32    }
33}
34
35/// Historical comparative unit: `chars.div_ceil(4)`.
36///
37/// Fine for ranking two packets of the same language on the same bench.
38/// Unsafe as a remaining-context bound for Cyrillic, CJK, JSON, or code.
39#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
40pub struct CharDiv4Counter;
41
42impl TokenCounter for CharDiv4Counter {
43    fn id(&self) -> &'static str {
44        "char_div4"
45    }
46
47    fn revision(&self) -> &'static str {
48        "v1"
49    }
50
51    fn count(&self, text: &str) -> u32 {
52        let tokens = text.chars().count().div_ceil(4).max(1);
53        u32::try_from(tokens).unwrap_or(u32::MAX)
54    }
55}
56
57/// Conservative overestimate for runtime budgets.
58///
59/// Non-Latin letters and punctuation cost one token each. ASCII words cost
60/// one token per three characters. This is not a vendor tokenizer; it is
61/// meant to refuse a packet before a model does.
62#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
63pub struct ConservativeCounter;
64
65impl TokenCounter for ConservativeCounter {
66    fn id(&self) -> &'static str {
67        "conservative"
68    }
69
70    fn revision(&self) -> &'static str {
71        "v1"
72    }
73
74    fn count(&self, text: &str) -> u32 {
75        let mut tokens = 0_u32;
76        let mut ascii_run = 0_u32;
77        for character in text.chars() {
78            if character.is_whitespace() {
79                flush_ascii(&mut tokens, &mut ascii_run);
80                continue;
81            }
82            if character.is_ascii_alphanumeric() {
83                ascii_run = ascii_run.saturating_add(1);
84                continue;
85            }
86            flush_ascii(&mut tokens, &mut ascii_run);
87            tokens = tokens.saturating_add(1);
88        }
89        flush_ascii(&mut tokens, &mut ascii_run);
90        tokens.max(1)
91    }
92}
93
94fn flush_ascii(tokens: &mut u32, ascii_run: &mut u32) {
95    if *ascii_run == 0 {
96        return;
97    }
98    *tokens = tokens.saturating_add(ascii_run.div_ceil(3));
99    *ascii_run = 0;
100}
101
102/// Comparative helper used by benches. Runtime compile uses
103/// [`ConservativeCounter`].
104#[must_use]
105pub fn estimate_tokens(value: &str) -> u32 {
106    CharDiv4Counter.count(value)
107}
108
109pub const RUNTIME_COUNTER: ConservativeCounter = ConservativeCounter;
110
111#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
112#[serde(rename_all = "camelCase")]
113pub struct TokenBreakdown {
114    pub counter_id: String,
115    pub tokenizer_revision: String,
116    /// Every candidate, before budget or dedup.
117    pub candidate_tokens: u32,
118    /// Included items counted on their original bodies.
119    pub selected_before_dedup_tokens: u32,
120    /// What the packet actually contains after dedup.
121    pub delivered_tokens: u32,
122    /// Whole items dropped to fit the budget. Not a saving.
123    pub budget_omitted_tokens: u32,
124    /// Repeated lines removed from included items. This is a saving.
125    pub dedup_saved_tokens: u32,
126    /// Headers and wrappers around the evidence bodies.
127    pub rendering_overhead_tokens: u32,
128    /// Same unit as [`Self::candidate_tokens`]; kept so a ledger can say
129    /// "estimated" without knowing which compile field to read.
130    #[serde(default)]
131    pub estimated_tokens: u32,
132    /// Filled when an executor later reports vendor usage.
133    #[serde(default, skip_serializing_if = "Option::is_none")]
134    pub actual_provider_tokens: Option<u32>,
135    /// `actual - estimated` once both exist. Positive means we under-counted.
136    #[serde(default, skip_serializing_if = "Option::is_none")]
137    pub estimation_error: Option<i32>,
138    /// Tokens on the wire after compile (packet body).
139    #[serde(default)]
140    pub wire_tokens: u32,
141    /// Tool/JSON schema tokens counted separately from evidence.
142    #[serde(default)]
143    pub schema_tokens: u32,
144    /// Instruction / wrapper tokens.
145    #[serde(default)]
146    pub instruction_tokens: u32,
147    /// Evidence bodies as selected, before wrappers.
148    #[serde(default)]
149    pub evidence_tokens: u32,
150    /// Reserved output budget. Compile does not consume this.
151    #[serde(default)]
152    pub output_tokens: u32,
153}
154
155impl TokenBreakdown {
156    #[must_use]
157    pub fn for_counter(counter: &dyn TokenCounter) -> Self {
158        Self {
159            counter_id: counter.id().to_owned(),
160            tokenizer_revision: counter.revision().to_owned(),
161            candidate_tokens: 0,
162            selected_before_dedup_tokens: 0,
163            delivered_tokens: 0,
164            budget_omitted_tokens: 0,
165            dedup_saved_tokens: 0,
166            rendering_overhead_tokens: 0,
167            estimated_tokens: 0,
168            actual_provider_tokens: None,
169            estimation_error: None,
170            wire_tokens: 0,
171            schema_tokens: 0,
172            instruction_tokens: 0,
173            evidence_tokens: 0,
174            output_tokens: 0,
175        }
176    }
177}
178
179#[cfg(test)]
180mod tests {
181    use super::*;
182
183    #[test]
184    fn conservative_overestimates_cyrillic_against_char_div4() {
185        let text = "Предыдущая попытка молча пропустила архив";
186        assert!(
187            ConservativeCounter.count(text) > CharDiv4Counter.count(text),
188            "cyrillic must not be treated as four chars per token"
189        );
190    }
191
192    #[test]
193    fn two_russian_sentences_do_not_share_a_div4_cost_with_code() {
194        let rust = "fn finish_block()";
195        let russian = "предыдущая попытка";
196        assert_eq!(CharDiv4Counter.count(rust), CharDiv4Counter.count(russian));
197        assert!(ConservativeCounter.count(russian) > ConservativeCounter.count(rust));
198    }
199
200    #[test]
201    fn message_and_schema_counts_are_conservative() {
202        let message = ChatMessage {
203            role: "user".to_owned(),
204            content: "предыдущая попытка".to_owned(),
205        };
206        assert!(
207            ConservativeCounter.count_message(&message) > CharDiv4Counter.count_message(&message)
208        );
209        let schema = blazingly_json::json!({
210            "type": "object",
211            "properties": { "id": { "type": "string" } }
212        });
213        assert!(ConservativeCounter.count_tool_schema(&schema) >= 1);
214    }
215}