newt_core/
tokens.rs

1//! Token-estimation settings — the chars-per-token heuristic as a config-backed
2//! value threaded explicitly through the estimators (no global state).
3//!
4//! Lives at the crate root (not under `agentic`) so [`crate::config`] can hold it
5//! without a backwards dependency, and so `newt-tui` can reach it for its own
6//! probe-side estimates.
7
8use serde::{Deserialize, Serialize};
9
10fn default_chars_per_token() -> usize {
11    4
12}
13
14/// How many characters of serialized JSON count as one token in the cheap
15/// no-tokenizer heuristic. Configured under `[context.estimation]` and threaded
16/// through `estimate_tokens` and friends.
17///
18/// The default (4) is the universal rough heuristic; the per-model calibration
19/// `ratio` (model-self-tuning, `docs/design/model-self-tuning.md` §2.3) scales the
20/// result on top. Exposed as a setting because a different model/tokenizer may
21/// pack a different number of chars per token — a smart default that adapts when
22/// an expert (human or LLM) tunes it.
23#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
24pub struct TokenEstimation {
25    #[serde(default = "default_chars_per_token")]
26    pub chars_per_token: usize,
27}
28
29impl Default for TokenEstimation {
30    fn default() -> Self {
31        Self {
32            chars_per_token: default_chars_per_token(),
33        }
34    }
35}
36
37impl TokenEstimation {
38    /// Construct from an explicit ratio (for tests / call sites that aren't
39    /// config-driven). A zero is clamped to 1 so division never panics.
40    pub fn new(chars_per_token: usize) -> Self {
41        Self {
42            chars_per_token: chars_per_token.max(1),
43        }
44    }
45
46    /// Token estimate for a `chars`-character string (ceiling-divide, so a
47    /// 1-char fragment never estimates to zero — err toward counting, 18.1).
48    pub fn tokens_for_chars(&self, chars: usize) -> usize {
49        chars.div_ceil(self.chars_per_token.max(1))
50    }
51
52    /// The inverse: a character budget for `tokens` tokens. Used to size a
53    /// summarizer input cap from a token budget.
54    pub fn chars_for_tokens(&self, tokens: usize) -> usize {
55        tokens.saturating_mul(self.chars_per_token.max(1))
56    }
57}
58
59#[cfg(test)]
60mod tests {
61    use super::*;
62
63    #[test]
64    fn default_is_four_chars_per_token() {
65        assert_eq!(TokenEstimation::default().chars_per_token, 4);
66    }
67
68    #[test]
69    fn ratio_drives_both_directions_consistently() {
70        let est = TokenEstimation::new(4);
71        assert_eq!(est.tokens_for_chars(8), 2);
72        assert_eq!(est.tokens_for_chars(9), 3, "ceiling-divide, never zero");
73        assert_eq!(est.chars_for_tokens(2), 8);
74        // A configured ratio changes estimate AND cap-sizing together — the whole
75        // point of one shared setting (#661 follow-up).
76        let wide = TokenEstimation::new(5);
77        assert_eq!(wide.tokens_for_chars(10), 2);
78        assert_eq!(wide.chars_for_tokens(2), 10);
79    }
80
81    #[test]
82    fn it_deserializes_from_the_context_estimation_table() {
83        let est: TokenEstimation = toml::from_str("chars_per_token = 5").unwrap();
84        assert_eq!(est.chars_per_token, 5);
85        // An omitted field falls back to the default.
86        assert_eq!(
87            toml::from_str::<TokenEstimation>("")
88                .unwrap()
89                .chars_per_token,
90            4
91        );
92    }
93
94    #[test]
95    fn zero_ratio_is_clamped_so_division_never_panics() {
96        assert_eq!(TokenEstimation::new(0).tokens_for_chars(3), 3);
97    }
98}
newt_core/tokens.rs

newt_core/
tokens.rs
