agentd/context/
tokens.rs

1// SPDX-License-Identifier: Apache-2.0
2//! Token **estimates** (RFC 0026 §5.2, §7). agentd never counts provider tokens
3//! itself — the provider's `usage` is the truth once a call returns — but the
4//! governor's reservation and the compaction trigger need a number *before*
5//! the call. The heuristic is the usual `chars / 4` (English prose ≈ 4 chars a
6//! token; JSON and code run denser, which errs on the safe side for a
7//! reservation) plus a small per-message overhead. Deliberately simple and
8//! dependency-free; the estimate is only ever compared against generous
9//! thresholds.
10
11use serde_json::Value;
12
13/// Per-message framing overhead (role, delimiters).
14pub const MESSAGE_OVERHEAD: u64 = 4;
15
16/// Estimate the tokens of a text.
17pub fn estimate(text: &str) -> u64 {
18    (text.chars().count() as u64).div_ceil(4)
19}
20
21/// Estimate the tokens of a JSON value (its compact serialization).
22pub fn estimate_value(v: &Value) -> u64 {
23    match v {
24        Value::String(s) => estimate(s),
25        Value::Null => 1,
26        other => estimate(&other.to_string()),
27    }
28}
29
30/// The default context window when the model is unknown (a conservative
31/// modern default; `intelligence.model_window` overrides).
32pub const DEFAULT_MODEL_WINDOW: u64 = 128_000;
33
34/// A best-effort window from the model name (kept tiny and obviously
35/// approximate; unknown models get the default).
36pub fn window_for_model(model: &str) -> u64 {
37    let m = model.to_ascii_lowercase();
38    if m.contains("claude") {
39        200_000
40    } else if m.contains("gpt-4o")
41        || m.contains("gpt-4.1")
42        || m.contains("gpt-5")
43        || m.starts_with("o1")
44        || m.starts_with("o3")
45        || m.starts_with("o4")
46    {
47        if m.contains("gpt-4.1") {
48            1_000_000
49        } else {
50            128_000
51        }
52    } else if m.contains("gemini") {
53        1_000_000
54    } else {
55        DEFAULT_MODEL_WINDOW
56    }
57}
58
59#[cfg(test)]
60mod tests {
61    use super::*;
62
63    #[test]
64    fn estimates_are_monotone_and_rounded_up() {
65        assert_eq!(estimate(""), 0);
66        assert_eq!(estimate("abcd"), 1);
67        assert_eq!(estimate("abcde"), 2);
68        assert!(estimate("a much longer sentence with many words") > estimate("short"));
69        assert_eq!(estimate_value(&serde_json::json!(null)), 1);
70        assert_eq!(estimate_value(&serde_json::json!("abcd")), 1);
71        assert!(estimate_value(&serde_json::json!({"k": "vvvv"})) >= 2);
72        assert_eq!(window_for_model("claude-sonnet-5"), 200_000);
73        assert_eq!(window_for_model("gpt-4.1-mini"), 1_000_000);
74        assert_eq!(window_for_model("something-else"), DEFAULT_MODEL_WINDOW);
75    }
76}
agentd/context/tokens.rs

agentd/context/
tokens.rs
