synaptic_core/
token_counter.rs

1use crate::Message;
2
3/// Trait for counting tokens in text and messages.
4pub trait TokenCounter: Send + Sync {
5    /// Count the number of tokens in a text string.
6    fn count_text(&self, text: &str) -> usize;
7
8    /// Count the total number of tokens in a slice of messages.
9    /// Default implementation sums count_text(content) + 4 per message overhead.
10    fn count_messages(&self, messages: &[Message]) -> usize {
11        messages
12            .iter()
13            .map(|m| self.count_text(m.content()) + 4)
14            .sum()
15    }
16}
17
18/// Heuristic token counter that estimates ~4 characters per token.
19pub struct HeuristicTokenCounter;
20
21impl TokenCounter for HeuristicTokenCounter {
22    fn count_text(&self, text: &str) -> usize {
23        // ~4 chars per token, minimum 1 token for non-empty text
24        let count = text.len() / 4;
25        if text.is_empty() {
26            0
27        } else {
28            count.max(1)
29        }
30    }
31}
synaptic_core/token_counter.rs

synaptic_core/
token_counter.rs
