Skip to main content

fetchium_core/token/
counter.rs

1//! Token counter and budget tracking.
2//!
3//! PRD SS20: Token Budget System. Approximates GPT-4 tokenization using
4//! a whitespace+punctuation heuristic (~95% accuracy for English text).
5//! For exact counts, integrate tiktoken in Phase 5.
6
7/// Average characters per token for the heuristic estimator.
8const CHARS_PER_TOKEN: f64 = 4.0;
9
10/// Count tokens in a text string using the heuristic estimator.
11///
12/// Returns an approximate token count. For typical English text,
13/// this is within ~5% of GPT-4's tiktoken.
14pub fn count_tokens(text: &str) -> u32 {
15    if text.is_empty() {
16        return 0;
17    }
18    let char_estimate = (text.len() as f64 / CHARS_PER_TOKEN).ceil() as u32;
19
20    let word_count = text.split_whitespace().count();
21    let word_estimate = (word_count as f64 * 1.3).ceil() as u32;
22
23    (char_estimate + word_estimate) / 2
24}
25
26/// Quickly estimate tokens without any splitting (fastest path).
27pub fn estimate_tokens_fast(text: &str) -> u32 {
28    (text.len() as f64 / CHARS_PER_TOKEN).ceil() as u32
29}
30
31/// Count tokens in a JSON value (accounts for JSON syntax overhead).
32pub fn count_tokens_json(value: &serde_json::Value) -> u32 {
33    let json_str = serde_json::to_string(value).unwrap_or_default();
34    count_tokens(&json_str)
35}
36
37/// Budget tracker that monitors token consumption.
38#[derive(Debug, Clone)]
39pub struct TokenBudget {
40    /// Total token budget allocated.
41    pub total: u32,
42    /// Tokens consumed so far.
43    pub used: u32,
44}
45
46impl TokenBudget {
47    /// Create a new budget with the given total.
48    pub fn new(total: u32) -> Self {
49        Self { total, used: 0 }
50    }
51
52    /// Remaining tokens available.
53    pub fn remaining(&self) -> u32 {
54        self.total.saturating_sub(self.used)
55    }
56
57    /// Whether the budget has been exhausted.
58    pub fn is_exhausted(&self) -> bool {
59        self.used >= self.total
60    }
61
62    /// Try to consume tokens. Returns true if within budget.
63    pub fn try_consume(&mut self, tokens: u32) -> bool {
64        if self.used + tokens <= self.total {
65            self.used += tokens;
66            true
67        } else {
68            false
69        }
70    }
71
72    /// Force-consume tokens (may exceed budget).
73    pub fn consume(&mut self, tokens: u32) {
74        self.used += tokens;
75    }
76
77    /// How full the budget is (0.0 to 1.0+).
78    /// Returns 0.0 for a zero-token budget (no budget allocated = no utilization).
79    pub fn utilization(&self) -> f64 {
80        if self.total == 0 {
81            return 0.0;
82        }
83        self.used as f64 / self.total as f64
84    }
85
86    /// Force-consume tokens with overflow protection.
87    /// Returns the actual number of tokens consumed (may be less if u32 would overflow).
88    pub fn consume_checked(&mut self, tokens: u32) -> u32 {
89        let actual = self.used.saturating_add(tokens) - self.used;
90        self.used = self.used.saturating_add(tokens);
91        actual
92    }
93}
94
95#[cfg(test)]
96mod tests {
97    use super::*;
98
99    #[test]
100    fn count_tokens_empty() {
101        assert_eq!(count_tokens(""), 0);
102    }
103
104    #[test]
105    fn count_tokens_short_text() {
106        let tokens = count_tokens("Hello, world!");
107        assert!((2..=6).contains(&tokens), "got {tokens}");
108    }
109
110    #[test]
111    fn count_tokens_paragraph() {
112        let text = "Rust is a multi-paradigm, general-purpose programming language. \
113                    It emphasizes performance, type safety, and concurrency.";
114        let tokens = count_tokens(text);
115        assert!((15..=45).contains(&tokens), "got {tokens}");
116    }
117
118    #[test]
119    fn budget_tracking() {
120        let mut budget = TokenBudget::new(1000);
121        assert_eq!(budget.remaining(), 1000);
122        assert!(!budget.is_exhausted());
123
124        assert!(budget.try_consume(500));
125        assert_eq!(budget.remaining(), 500);
126        assert_eq!(budget.used, 500);
127
128        assert!(budget.try_consume(500));
129        assert!(budget.is_exhausted());
130
131        assert!(!budget.try_consume(1));
132    }
133
134    #[test]
135    fn budget_utilization() {
136        let mut budget = TokenBudget::new(100);
137        budget.consume(50);
138        assert!((budget.utilization() - 0.5).abs() < f64::EPSILON);
139    }
140
141    #[test]
142    fn fast_estimate_consistency() {
143        let text = "This is a test sentence with several words.";
144        let fast = estimate_tokens_fast(text);
145        let normal = count_tokens(text);
146        assert!((fast as i32 - normal as i32).unsigned_abs() < 10);
147    }
148}