codetether-rlm 0.1.0

Recursive Language Model processing for CodeTether
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

//! Token estimation heuristics.

/// Estimate token count (roughly 2.8–4 chars per token depending on
/// whitespace density).
pub fn estimate_tokens(text: &str) -> usize {
    if text.is_empty() {
        return 0;
    }
    let len = text.len();
    let whitespace = text
        .as_bytes()
        .iter()
        .filter(|b| b.is_ascii_whitespace())
        .count();
    let ws_ratio = whitespace as f64 / len as f64;
    let chars_per_token = if ws_ratio < 0.05 {
        2.8
    } else if ws_ratio < 0.10 {
        3.2
    } else if ws_ratio < 0.20 {
        3.6
    } else {
        4.0
    };
    ((len as f64) / chars_per_token).ceil() as usize
}