fetchium_core/token/
counter.rs1const CHARS_PER_TOKEN: f64 = 4.0;
9
10pub fn count_tokens(text: &str) -> u32 {
15 if text.is_empty() {
16 return 0;
17 }
18 let char_estimate = (text.len() as f64 / CHARS_PER_TOKEN).ceil() as u32;
19
20 let word_count = text.split_whitespace().count();
21 let word_estimate = (word_count as f64 * 1.3).ceil() as u32;
22
23 (char_estimate + word_estimate) / 2
24}
25
26pub fn estimate_tokens_fast(text: &str) -> u32 {
28 (text.len() as f64 / CHARS_PER_TOKEN).ceil() as u32
29}
30
31pub fn count_tokens_json(value: &serde_json::Value) -> u32 {
33 let json_str = serde_json::to_string(value).unwrap_or_default();
34 count_tokens(&json_str)
35}
36
37#[derive(Debug, Clone)]
39pub struct TokenBudget {
40 pub total: u32,
42 pub used: u32,
44}
45
46impl TokenBudget {
47 pub fn new(total: u32) -> Self {
49 Self { total, used: 0 }
50 }
51
52 pub fn remaining(&self) -> u32 {
54 self.total.saturating_sub(self.used)
55 }
56
57 pub fn is_exhausted(&self) -> bool {
59 self.used >= self.total
60 }
61
62 pub fn try_consume(&mut self, tokens: u32) -> bool {
64 if self.used + tokens <= self.total {
65 self.used += tokens;
66 true
67 } else {
68 false
69 }
70 }
71
72 pub fn consume(&mut self, tokens: u32) {
74 self.used += tokens;
75 }
76
77 pub fn utilization(&self) -> f64 {
80 if self.total == 0 {
81 return 0.0;
82 }
83 self.used as f64 / self.total as f64
84 }
85
86 pub fn consume_checked(&mut self, tokens: u32) -> u32 {
89 let actual = self.used.saturating_add(tokens) - self.used;
90 self.used = self.used.saturating_add(tokens);
91 actual
92 }
93}
94
95#[cfg(test)]
96mod tests {
97 use super::*;
98
99 #[test]
100 fn count_tokens_empty() {
101 assert_eq!(count_tokens(""), 0);
102 }
103
104 #[test]
105 fn count_tokens_short_text() {
106 let tokens = count_tokens("Hello, world!");
107 assert!((2..=6).contains(&tokens), "got {tokens}");
108 }
109
110 #[test]
111 fn count_tokens_paragraph() {
112 let text = "Rust is a multi-paradigm, general-purpose programming language. \
113 It emphasizes performance, type safety, and concurrency.";
114 let tokens = count_tokens(text);
115 assert!((15..=45).contains(&tokens), "got {tokens}");
116 }
117
118 #[test]
119 fn budget_tracking() {
120 let mut budget = TokenBudget::new(1000);
121 assert_eq!(budget.remaining(), 1000);
122 assert!(!budget.is_exhausted());
123
124 assert!(budget.try_consume(500));
125 assert_eq!(budget.remaining(), 500);
126 assert_eq!(budget.used, 500);
127
128 assert!(budget.try_consume(500));
129 assert!(budget.is_exhausted());
130
131 assert!(!budget.try_consume(1));
132 }
133
134 #[test]
135 fn budget_utilization() {
136 let mut budget = TokenBudget::new(100);
137 budget.consume(50);
138 assert!((budget.utilization() - 0.5).abs() < f64::EPSILON);
139 }
140
141 #[test]
142 fn fast_estimate_consistency() {
143 let text = "This is a test sentence with several words.";
144 let fast = estimate_tokens_fast(text);
145 let normal = count_tokens(text);
146 assert!((fast as i32 - normal as i32).unsigned_abs() < 10);
147 }
148}