vtcode_commons/
tokens.rs

1//! Token estimation utilities
2
3/// Estimate token count from string (rough approximation)
4///
5/// Simple estimation: 1 token ≈ 4 characters.
6/// Returns a minimum of 1 for non-empty strings.
7#[inline]
8pub fn estimate_tokens(text: &str) -> usize {
9    if text.is_empty() {
10        return 0;
11    }
12    (text.len() / 4).max(1)
13}
14
15/// Truncate string to approximate token limit
16///
17/// Returns a truncated string that fits within the approximate token limit.
18/// Tries to truncate at word boundaries when possible to avoid mid-word cuts.
19pub fn truncate_to_tokens(text: &str, max_tokens: usize) -> String {
20    if max_tokens == 0 || text.is_empty() {
21        return String::new();
22    }
23
24    let max_chars = max_tokens * 4;
25    if text.len() <= max_chars {
26        return text.to_string();
27    }
28
29    // Try to truncate at a word boundary
30    let truncated = &text[..max_chars];
31    match truncated.rfind(' ') {
32        Some(last_space) if last_space > max_chars / 2 => {
33            let mut result = truncated[..last_space].to_string();
34            result.push_str("...");
35            result
36        }
37        _ => {
38            let mut result = truncated.to_string();
39            result.push_str("...");
40            result
41        }
42    }
43}
vtcode_commons/tokens.rs

vtcode_commons/
tokens.rs
