lean_ctx_proxy/
session_elide.rs

1// SPDX-License-Identifier: Apache-2.0
2//! Long-session elision for Anthropic Messages requests.
3//!
4//! In a long agent session most of every request is old tool output the model
5//! has already acted on. Once the client's `messages` serialize to more than
6//! [`THRESHOLD_BYTES`], every `tool_result` older than the live window whose
7//! text is longer than [`MIN_ELIDE_BYTES`] leaves as a one-line stub. The
8//! original goes to the content-addressed tee store (`{state}/tee/`, files
9//! 0600 in a private state directory) and the stub names that file, so the
10//! model gets the full text back with `cat` or its file read — Claude Code in
11//! governor mode has no MCP tools, so `ctx_expand` alone would not do.
12//!
13//! ## Why this rewrites inside the cached prefix
14//!
15//! Claude Code puts a cache marker on the last message, so the history pruner
16//! (which never touches the client-cached prefix, #448) never fires there.
17//! Elision deliberately does. It is bounded:
18//!
19//! - the window is the cache-aware staircase of
20//!   `history_prune::prune_boundary`: it advances only in whole strides of
21//!   16 messages and keeps at least 8, so the cached prefix breaks once per
22//!   stride, not every turn;
23//! - the stub is a pure function of the block's text (#498), so a block
24//!   elided once leaves with the same bytes on every later turn;
25//! - the size test reads the client's own messages, which only grow, so a
26//!   session that crossed the threshold stays across it;
27//! - it runs on every request of such a session, the person's prompt turns
28//!   included: a prompt turn then sends the same elided prefix as the tool
29//!   turn before it and reads the cache, instead of sending the history
30//!   whole and writing it again.
31//!
32//! Measured on Claude Code with Haiku 5.5, long sessions: cost x0.74
33//! (95 % CI 0.53–0.94) against native, 12/12 tasks solved on both arms. The
34//! one cache write per stride is cheaper than reading the full history on
35//! every turn of that stride.
36//!
37//! The determinism guard and the prefix replay know this exception: a
38//! forwarded prefix that first differs at a newly elided message is the
39//! sanctioned stride break ([`newly_elided`], [`message_is_elided`]).
40//!
41//! Under protection (a policy pack, governed or sovereign mode) the tee store
42//! holds nothing originless: a block leaves shortened only with a recovery
43//! copy bound to its lineage (`compress::shorten_with_copy`), otherwise it
44//! stays whole. Content never reaches a log.
45
46use serde_json::Value;
47
48/// Serialized size of the client's `messages` above which old tool output is
49/// elided (measured arm: 180 000 characters).
50pub const THRESHOLD_BYTES: usize = 180_000;
51
52/// Tool output at or below this size stays as it is: the stub would save
53/// little and the model often still quotes short results.
54pub const MIN_ELIDE_BYTES: usize = 2_000;
55
56/// How every elision stub opens; also what [`message_is_elided`] recognises.
57pub const STUB_OPEN: &str = "[lean-ctx: earlier tool output elided";
58
59/// Serialized size of `messages`, without building the string.
60fn serialized_len(messages: &[Value]) -> usize {
61    struct Count(usize);
62    impl std::io::Write for Count {
63        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
64            self.0 += buf.len();
65            Ok(buf.len())
66        }
67        fn flush(&mut self) -> std::io::Result<()> {
68            Ok(())
69        }
70    }
71    let mut count = Count(0);
72    let _ = serde_json::to_writer(&mut count, messages);
73    count.0
74}
75
76/// Elide old tool output in `messages` when they exceed `threshold` bytes.
77/// Returns the index of the last message that carries a stub, or `None` when
78/// nothing was elided.
79pub fn elide(messages: &mut [Value], threshold: usize) -> Option<usize> {
80    if serialized_len(messages) <= threshold {
81        return None;
82    }
83    let boundary = super::history_prune::prune_boundary(
84        lean_ctx::core::config::HistoryMode::CacheAware,
85        messages.len(),
86    );
87    let mut last = None;
88    for (index, message) in messages[..boundary].iter_mut().enumerate() {
89        if message.get("role").and_then(Value::as_str) != Some("user") {
90            continue;
91        }
92        let Some(blocks) = message.get_mut("content").and_then(Value::as_array_mut) else {
93            continue;
94        };
95        for block in blocks {
96            if elide_block(block) {
97                last = Some(index);
98            }
99        }
100    }
101    last
102}
103
104/// The text of a tool result that may be elided: a string, or an array made
105/// only of text items without a cache marker (an image or a nested marker
106/// keeps the block whole).
107fn elidable_text(content: &Value) -> Option<String> {
108    match content {
109        Value::String(text) => Some(text.clone()),
110        Value::Array(items) => {
111            let mut parts = Vec::with_capacity(items.len());
112            for item in items {
113                if item.get("type").and_then(Value::as_str) != Some("text")
114                    || item.get("cache_control").is_some()
115                {
116                    return None;
117                }
118                parts.push(item.get("text")?.as_str()?);
119            }
120            Some(parts.join("\n"))
121        }
122        _ => None,
123    }
124}
125
126/// Replace one `tool_result` block's content by its stub. Only `content`
127/// changes: `tool_use_id`, `is_error` and `cache_control` stay.
128fn elide_block(block: &mut Value) -> bool {
129    if block.get("type").and_then(Value::as_str) != Some("tool_result") {
130        return false;
131    }
132    let Some(original) = block.get("content").and_then(elidable_text) else {
133        return false;
134    };
135    if original.len() <= MIN_ELIDE_BYTES
136        || original.starts_with(STUB_OPEN)
137        || original.contains("<protect>")
138    {
139        return false;
140    }
141    let chars = original.chars().count();
142    let stub = if super::compress::protected() {
143        let call_id = block.get("tool_use_id").and_then(Value::as_str);
144        let _block = super::sticky_tools::block(call_id);
145        let Some(stub) = super::compress::shorten_with_copy(
146            &original,
147            &format!("{STUB_OPEN} ({chars} chars).]"),
148        ) else {
149            return false;
150        };
151        stub
152    } else {
153        // No recovery copy, no elision: the model keeps the text it had.
154        let Some(path) = super::ccr::persist_once(&original) else {
155            return false;
156        };
157        format!(
158            "{STUB_OPEN} ({chars} chars) to keep this long session lean; full text: {}]",
159            locator(std::path::Path::new(&path))
160        )
161    };
162    block["content"] = Value::String(stub);
163    true
164}
165
166/// How the stub names the archive without the account name: `cat ~/…` under
167/// the home directory, else the store-relative name that `ctx_expand`
168/// resolves. A pure function of the path and the home, so byte-stable.
169fn locator(path: &std::path::Path) -> String {
170    let home = shell_home(|key| std::env::var_os(key), cfg!(windows));
171    locator_in(path, home.as_deref())
172}
173
174/// The home directory the host shell expands `~` to: `HOME`, and on Windows,
175/// where `HOME` is usually unset, `USERPROFILE` (#2180). `var` is injected so
176/// tests never touch the process environment.
177fn shell_home(
178    var: impl Fn(&str) -> Option<std::ffi::OsString>,
179    windows: bool,
180) -> Option<std::ffi::OsString> {
181    let set = |key: &str| var(key).filter(|value| !value.is_empty());
182    set("HOME").or_else(|| if windows { set("USERPROFILE") } else { None })
183}
184
185/// The locator for `path` against `home`. The part after `~/` is joined with
186/// `/` (Windows paths included) and double-quoted when it holds a character a
187/// shell would split or glob on, such as a space (#2180): `cat ~/"My State/…"`
188/// reads the same file in Bash (Claude Code's shell, Git Bash on Windows) and
189/// in PowerShell, where `cat` is `Get-Content` and `~` is the home. A path a
190/// double quote cannot carry literally falls back to the `ctx_expand` name.
191fn locator_in(path: &std::path::Path, home: Option<&std::ffi::OsStr>) -> String {
192    if let Some(home) = home
193        && let Ok(rest) = path.strip_prefix(home)
194        && let Some(rest) = home_relative(rest)
195    {
196        return format!("cat ~/{rest}");
197    }
198    let name = path
199        .file_name()
200        .map_or_else(String::new, |n| n.to_string_lossy().into_owned());
201    format!("ctx_expand(id=\"{name}\") (LeanCTX state directory, tee/{name})")
202}
203
204/// `rest` as the shell-ready text after `~/`, or `None` when it cannot be
205/// written there unambiguously.
206fn home_relative(rest: &std::path::Path) -> Option<String> {
207    let mut parts = Vec::new();
208    for component in rest.components() {
209        match component {
210            std::path::Component::Normal(part) => parts.push(part.to_str()?),
211            _ => return None,
212        }
213    }
214    if parts.is_empty() {
215        return None;
216    }
217    let joined = parts.join("/");
218    let plain = |c: char| c.is_ascii_alphanumeric() || "._-/+@%,:=".contains(c);
219    if joined.chars().all(plain) {
220        return Some(joined);
221    }
222    // Inside double quotes Bash still expands `$`, `` ` ``, `\` and `"`, and
223    // PowerShell `$` and `` ` ``; `!` can trigger history expansion.
224    if joined
225        .chars()
226        .any(|c| "\"$`\\!".contains(c) || c.is_control())
227    {
228        return None;
229    }
230    Some(format!("\"{joined}\""))
231}
232
233/// Whether `message` carries a tool result that left as an elision stub.
234pub fn message_is_elided(message: &Value) -> bool {
235    message
236        .get("content")
237        .and_then(Value::as_array)
238        .is_some_and(|blocks| {
239            blocks.iter().any(|block| {
240                block.get("type").and_then(Value::as_str) == Some("tool_result")
241                    && block
242                        .get("content")
243                        .and_then(Value::as_str)
244                        .is_some_and(|text| text.starts_with(STUB_OPEN))
245            })
246        })
247}
248
249/// Whether the elision that reached message `last` is new against a forwarded
250/// request (`prefix_bytes`): that request covered message `last` and sent it
251/// without a stub. Elision only advances in whole strides, so the last elided
252/// message decides; a prefix replay must not restore the older bytes then.
253pub fn newly_elided(prefix_bytes: &[u8], last: usize) -> bool {
254    let Ok(previous) = serde_json::from_slice::<Value>(prefix_bytes) else {
255        return true;
256    };
257    previous
258        .get("messages")
259        .and_then(Value::as_array)
260        .and_then(|messages| messages.get(last))
261        .is_some_and(|message| !message_is_elided(message))
262}
263
264#[cfg(test)]
265mod tests {
266    use super::*;
267    use serde_json::json;
268
269    /// A tool loop: `turns` tool calls, each with a `size`-byte result.
270    fn session(turns: usize, size: usize) -> Vec<Value> {
271        let mut messages = vec![json!({"role": "user", "content": "fix the build"})];
272        for turn in 0..turns {
273            let id = format!("t{turn}");
274            messages.push(json!({"role": "assistant", "content": [
275                {"type": "tool_use", "id": id, "name": "Bash", "input": {"command": "make"}}
276            ]}));
277            let output = format!("output {turn} line\n").repeat(size / 15);
278            messages.push(json!({"role": "user", "content": [
279                {"type": "tool_result", "tool_use_id": id, "content": output}
280            ]}));
281        }
282        let last = messages.len() - 1;
283        messages[last]["content"][0]["cache_control"] = json!({"type": "ephemeral", "ttl": "1h"});
284        messages
285    }
286
287    #[test]
288    fn below_the_threshold_nothing_changes() {
289        let _data = lean_ctx::core::data_dir::isolated_data_dir();
290        let _community = lean_ctx::core::policy::runtime::TestPolicyOverride::set(None);
291        let mut messages = session(20, 5_000);
292        let before = messages.clone();
293        assert_eq!(elide(&mut messages, THRESHOLD_BYTES), None);
294        assert_eq!(messages, before);
295    }
296
297    #[test]
298    fn old_large_results_leave_as_a_stub_that_names_the_original() {
299        let _data = lean_ctx::core::data_dir::isolated_data_dir();
300        let _community = lean_ctx::core::policy::runtime::TestPolicyOverride::set(None);
301        let mut messages = session(30, 5_000);
302        // A small old result and a nested marker stay whole.
303        messages[2]["content"][0]["content"] = json!("ok");
304        messages[4]["content"][0]["content"] = json!([{"type": "text", "text": "x".repeat(5_000), "cache_control": {"type": "ephemeral"}}]);
305        let original = messages[6]["content"][0]["content"]
306            .as_str()
307            .unwrap()
308            .to_owned();
309        let before = messages.clone();
310
311        let last = elide(&mut messages, 100_000).expect("elided");
312        let boundary = (messages.len() - 8) / 16 * 16;
313        // The last tool-result turn before the stride boundary (odd indices
314        // are assistant turns).
315        assert_eq!(last, boundary - 2, "the window ends at the stride boundary");
316
317        assert_eq!(messages[2], before[2], "small result untouched");
318        assert_eq!(messages[4], before[4], "nested marker untouched");
319        let stub = messages[6]["content"][0]["content"].as_str().unwrap();
320        assert!(stub.starts_with(STUB_OPEN), "{stub}");
321        let path = if let Some(rest) = stub.split("cat ~/").nth(1) {
322            std::path::PathBuf::from(
323                shell_home(|key| std::env::var_os(key), cfg!(windows)).unwrap(),
324            )
325            .join(rest.trim_end_matches(']').trim_matches('"'))
326        } else {
327            let id = stub
328                .split("id=\"")
329                .nth(1)
330                .unwrap()
331                .split('"')
332                .next()
333                .unwrap();
334            lean_ctx::core::ccr::resolve_tee(id).expect("archive resolves")
335        };
336        if let Some(home) = std::env::var("HOME").ok().filter(|h| !h.is_empty()) {
337            assert!(!stub.contains(&home), "no account path in the stub: {stub}");
338        }
339        // The fixture holds no secret, so the redacted archive is the original.
340        assert_eq!(std::fs::read_to_string(&path).unwrap(), original);
341        #[cfg(unix)]
342        {
343            use std::os::unix::fs::PermissionsExt;
344            let mode = std::fs::metadata(&path).unwrap().permissions().mode();
345            assert_eq!(mode & 0o777, 0o600, "the archive is private");
346        }
347        assert!(message_is_elided(&messages[6]));
348        assert_eq!(
349            messages[boundary..],
350            before[boundary..],
351            "the live window, with its cache marker, is untouched"
352        );
353
354        // Same block, same stub — also after the TTL cleanup removed the
355        // archive, which the next pass writes again.
356        let once = messages.clone();
357        std::fs::remove_file(&path).unwrap();
358        let mut again = before.clone();
359        elide(&mut again, 100_000);
360        assert_eq!(again, once, "content-deterministic stub (#498)");
361        assert_eq!(std::fs::read_to_string(&path).unwrap(), original);
362    }
363
364    fn env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option<std::ffi::OsString> {
365        let pairs: Vec<(String, String)> = pairs
366            .iter()
367            .map(|(k, v)| ((*k).to_owned(), (*v).to_owned()))
368            .collect();
369        move |key| {
370            pairs
371                .iter()
372                .find(|(k, _)| k == key)
373                .map(|(_, v)| std::ffi::OsString::from(v))
374        }
375    }
376
377    #[test]
378    fn windows_falls_back_to_userprofile_when_home_is_unset() {
379        let profile = env(&[("USERPROFILE", "/Users/me")]);
380        assert_eq!(shell_home(&profile, true), Some("/Users/me".into()));
381        assert_eq!(
382            shell_home(&profile, false),
383            None,
384            "only Windows reads USERPROFILE"
385        );
386        let both = env(&[("HOME", "/home/me"), ("USERPROFILE", "/Users/me")]);
387        assert_eq!(
388            shell_home(&both, true),
389            Some("/home/me".into()),
390            "HOME wins"
391        );
392        let empty = env(&[("HOME", ""), ("USERPROFILE", "/Users/me")]);
393        assert_eq!(shell_home(&empty, true), Some("/Users/me".into()));
394        assert_eq!(shell_home(env(&[]), true), None);
395
396        let home = shell_home(&profile, true);
397        let path = std::path::Path::new("/Users/me/AppData/Local/lean-ctx/tee/proxy_ab12.log");
398        assert_eq!(
399            locator_in(path, home.as_deref()),
400            "cat ~/AppData/Local/lean-ctx/tee/proxy_ab12.log"
401        );
402    }
403
404    #[cfg(windows)]
405    #[test]
406    fn windows_paths_use_forward_slashes_after_the_tilde() {
407        let path = std::path::Path::new(r"C:\Users\me\AppData\Local\lean-ctx\tee\proxy_ab12.log");
408        assert_eq!(
409            locator_in(path, Some(std::ffi::OsStr::new(r"C:\Users\me"))),
410            "cat ~/AppData/Local/lean-ctx/tee/proxy_ab12.log"
411        );
412    }
413
414    #[test]
415    fn a_path_with_spaces_is_quoted_after_the_tilde() {
416        let home = Some(std::ffi::OsStr::new("/Users/me"));
417        let path = std::path::Path::new("/Users/me/My State/lean-ctx/tee/proxy_ab12.log");
418        let once = locator_in(path, home);
419        assert_eq!(once, "cat ~/\"My State/lean-ctx/tee/proxy_ab12.log\"");
420        assert_eq!(locator_in(path, home), once, "byte-stable (#498)");
421
422        let plain = std::path::Path::new("/Users/me/.local/state/lean-ctx/tee/proxy_ab12.log");
423        assert_eq!(
424            locator_in(plain, home),
425            "cat ~/.local/state/lean-ctx/tee/proxy_ab12.log",
426            "plain paths stay unquoted"
427        );
428
429        // A name a double quote cannot carry literally falls back to ctx_expand.
430        let dollar = std::path::Path::new("/Users/me/$state/lean-ctx/tee/proxy_ab12.log");
431        assert!(
432            locator_in(dollar, home).starts_with("ctx_expand(id=\"proxy_ab12.log\")"),
433            "{}",
434            locator_in(dollar, home)
435        );
436        // Outside the home: the store-relative name, no account path.
437        let outside = std::path::Path::new("/var/state/lean-ctx/tee/proxy_ab12.log");
438        assert_eq!(
439            locator_in(outside, home),
440            "ctx_expand(id=\"proxy_ab12.log\") (LeanCTX state directory, tee/proxy_ab12.log)"
441        );
442    }
443
444    #[test]
445    fn the_prefix_stays_byte_identical_within_a_stride() {
446        let _data = lean_ctx::core::data_dir::isolated_data_dir();
447        let _community = lean_ctx::core::policy::runtime::TestPolicyOverride::set(None);
448        // 61 and 63 messages share the boundary 48; 73 moves it to 64.
449        let mut turn = session(30, 5_000);
450        let mut next = session(31, 5_000);
451        let mut jump = session(36, 5_000);
452        elide(&mut turn, 100_000);
453        elide(&mut next, 100_000);
454        let last = elide(&mut jump, 100_000).unwrap();
455        let prefix = |m: &[Value]| serde_json::to_vec(&m[..48]).unwrap();
456        assert_eq!(prefix(&turn), prefix(&next));
457        assert_eq!(prefix(&turn), prefix(&jump));
458
459        let forwarded = serde_json::to_vec(&json!({"messages": next})).unwrap();
460        assert!(newly_elided(&forwarded, last), "the stride jump is new");
461        let forwarded = serde_json::to_vec(&json!({"messages": jump})).unwrap();
462        assert!(!newly_elided(&forwarded, last), "already sent elided");
463    }
464}
lean_ctx_proxy/session_elide.rs

lean_ctx_proxy/
session_elide.rs
