1use super::ccr;
2use crate::core::tokens::{COUNTING_FAMILY, TokenizerFamily, count_tokens_for};
3use crate::core::web::distill;
4
5const RESEARCH_PROSE_CAP: usize = 20_000;
9const RESEARCH_PROSE_CAP_ENV: &str = "LEAN_CTX_RESEARCH_PROSE_CAP";
10
11fn research_prose_cap() -> usize {
12 std::env::var(RESEARCH_PROSE_CAP_ENV)
13 .ok()
14 .and_then(|v| v.trim().parse::<usize>().ok())
15 .filter(|cap| *cap > 0)
16 .unwrap_or(RESEARCH_PROSE_CAP)
17}
18
19pub fn compress_tool_result(content: &str, tool_name: Option<&str>) -> String {
31 compress_tool_result_for(content, tool_name, COUNTING_FAMILY)
32}
33
34pub fn compress_tool_result_for(
35 content: &str,
36 tool_name: Option<&str>,
37 family: TokenizerFamily,
38) -> String {
39 let compressed = compress_inner(content, tool_name, family);
40 attach_ccr(content, compressed, CcrAudience::Local)
41}
42
43pub fn compress_tool_result_gateway(content: &str, tool_name: Option<&str>) -> String {
53 compress_tool_result_gateway_for(content, tool_name, COUNTING_FAMILY)
54}
55
56pub fn compress_tool_result_gateway_for(
57 content: &str,
58 tool_name: Option<&str>,
59 family: TokenizerFamily,
60) -> String {
61 let compressed = compress_inner(content, tool_name, family);
62 let clean = crate::core::protocol::strip_trailing_savings_footer(&compressed).to_string();
63 attach_ccr(content, clean, CcrAudience::Gateway)
64}
65
66#[derive(Clone, Copy, PartialEq)]
69enum CcrAudience {
70 Local,
71 Gateway,
72}
73
74fn attach_ccr(original: &str, result: String, audience: CcrAudience) -> String {
81 use super::sticky_tools;
82 if original.len() < ccr::MIN_TEE_BYTES
83 || original.len().saturating_sub(result.len()) < ccr::MIN_TEE_BYTES
84 {
85 return result;
86 }
87 match ccr::persist(original) {
88 Some(handle) if audience == CcrAudience::Gateway => {
96 sticky_tools::mark_ccr_active(0);
97 let hash = ccr::litellm_hash(original);
98 format!(
99 "{result}\n[lean-ctx CCR: full original elided to save tokens — call the \
100 retrieve tool with hash={hash}, or read {handle} locally]"
101 )
102 }
103 Some(handle) => {
104 sticky_tools::mark_ccr_active(0);
105 match ccr::inband_locator(&handle) {
106 Some(marker) => format!(
107 "{result}\n[lean-ctx: full original elided to save tokens — echo {marker} \
108 on your next turn to get the verbatim original spliced back inline]"
109 ),
110 None => format!(
111 "{result}\n[lean-ctx: full original at {handle} — read it directly (no MCP), or \
112 ctx_expand(id=\"{handle}\", head=N|search=\"…\"|json_path=\"…\") for a slice]"
113 ),
114 }
115 }
116 None => result,
117 }
118}
119
120fn compress_inner(content: &str, tool_name: Option<&str>, family: TokenizerFamily) -> String {
121 if content.trim().is_empty() || content.len() < 200 {
122 return content.to_string();
123 }
124
125 if tool_name.is_some_and(is_lean_ctx_tool) {
132 return content.to_string();
133 }
134
135 if crate::core::protect::has_markers(content) {
139 return crate::core::protect::compress_preserving(content, |seg| {
140 compress_inner(seg, tool_name, family)
141 });
142 }
143
144 if is_cited_research_output(content) {
145 return content.to_string();
146 }
147
148 if extract_command_hint(content).is_none()
149 && looks_like_prose(content)
150 && let Some(out) = squeeze_research_prose(content, family)
151 {
152 return out;
153 }
154
155 let cmd = infer_command(content, tool_name);
156
157 let generic_command = cmd.is_empty() || cmd == "shell";
164 if generic_command
165 && (output_looks_like_test_run(content) || output_looks_like_build_failure(content))
166 {
167 return crate::shell::compress::engine::preserve_verbatim_pub_for(content, family);
168 }
169
170 crate::shell::compress::engine::compress_if_beneficial_for(&cmd, content, family)
171}
172
173fn output_looks_like_test_run(content: &str) -> bool {
177 const NEEDLES: &[&str] = &[
178 "test result:", "short test summary info", " passed in ", " failed in ", "=== RUN", "--- FAIL:", "--- PASS:", "Test Suites:", " examples, ", "FAILED", ];
189 NEEDLES.iter().any(|n| content.contains(n))
190}
191
192fn output_looks_like_build_failure(content: &str) -> bool {
196 const NEEDLES: &[&str] = &[
197 "error[", ": error:", "fatal error:", "undefined reference to", "panicked at", "could not compile", "Traceback (most recent call last)", "AssertionError", "make: ***", "Build FAILED",
207 "BUILD FAILED",
208 "Segmentation fault",
209 ];
210 NEEDLES.iter().any(|n| content.contains(n))
211}
212
213fn is_cited_research_output(content: &str) -> bool {
216 content.contains("· Retrieved: ") && content.contains("\nSource: ")
217}
218
219const CODE_SYMBOLS: &str = "{}<>;=|\\$`";
221
222fn looks_like_prose(content: &str) -> bool {
225 let sample: String = content.chars().take(4000).collect();
226 let total = sample.chars().count();
227 if total < 600 {
228 return false;
229 }
230 let total_f = total as f32;
231 let alpha = sample.chars().filter(|c| c.is_alphabetic()).count() as f32;
232 let spaces = sample.chars().filter(|c| *c == ' ').count() as f32;
233 let symbols = sample.chars().filter(|c| CODE_SYMBOLS.contains(*c)).count() as f32;
234
235 if alpha / total_f < 0.6 || spaces / total_f < 0.12 || symbols / total_f > 0.06 {
236 return false;
237 }
238 if sample.matches(['.', '!', '?']).count() < 4 {
239 return false;
240 }
241
242 let non_empty: Vec<&str> = sample.lines().filter(|l| !l.trim().is_empty()).collect();
243 if non_empty.is_empty() {
244 return false;
245 }
246 let avg_len =
247 non_empty.iter().map(|l| l.chars().count()).sum::<usize>() as f32 / non_empty.len() as f32;
248 avg_len >= 40.0
249}
250
251fn squeeze_research_prose(content: &str, family: TokenizerFamily) -> Option<String> {
254 let before = count_tokens_for(content, family);
255 let squeezed = squeeze_research_prose_body(content);
256 if squeezed.trim().is_empty() {
257 return None;
258 }
259 let after = count_tokens_for(&squeezed, family);
260 if after + 2 >= before {
261 return None;
262 }
263 Some(crate::core::protocol::append_savings_with_info(
264 &squeezed,
265 before,
266 after,
267 Some("research"),
268 None,
269 ))
270}
271
272fn squeeze_research_prose_body(content: &str) -> String {
280 let cap = research_prose_cap();
281 if content.len() > cap {
282 return super::prose_ranker::squeeze(content, cap);
283 }
284 distill::squeeze_prose(content, cap)
285}
286
287fn is_lean_ctx_tool(name: &str) -> bool {
296 let bare = name
297 .rsplit("__")
298 .next()
299 .unwrap_or(name)
300 .rsplit([':', '/', '.'])
301 .next()
302 .unwrap_or(name);
303 bare.starts_with("ctx_") || name.starts_with("ctx_")
304}
305
306fn infer_command(content: &str, tool_name: Option<&str>) -> String {
307 if let Some(cmd) = extract_command_hint(content) {
308 return cmd;
309 }
310
311 if let Some(name) = tool_name {
312 let nl = name.to_lowercase();
313 if nl.contains("bash") || nl.contains("shell") || nl.contains("terminal") {
314 return "shell".to_string();
315 }
316 if nl.contains("search") || nl.contains("grep") || nl.contains("find") {
317 return "grep".to_string();
318 }
319 }
320
321 String::new()
322}
323
324fn extract_command_hint(content: &str) -> Option<String> {
325 for line in content.lines().take(3) {
326 let trimmed = line.trim();
327 if let Some(cmd) = trimmed.strip_prefix("$ ") {
328 return Some(cmd.to_string());
329 }
330 if let Some(cmd) = trimmed.strip_prefix("% ") {
331 return Some(cmd.to_string());
332 }
333 }
334 None
335}
336
337#[cfg(test)]
338mod tests {
339 use super::ccr;
340 use super::{
341 RESEARCH_PROSE_CAP, RESEARCH_PROSE_CAP_ENV, compress_tool_result,
342 compress_tool_result_gateway, extract_command_hint, infer_command, looks_like_prose,
343 output_looks_like_build_failure, output_looks_like_test_run, research_prose_cap,
344 };
345 use serial_test::serial;
346
347 #[test]
349 fn search_tool_result_is_compressed_without_corrupting_source() {
350 let raw = (0..60)
351 .map(|i| {
352 format!(
353 "src/h.go:{i}:func handler{i}(ctx context.Context) (api.Result, error) \
354 {{ return doWork(ctx) }}"
355 )
356 })
357 .collect::<Vec<_>>()
358 .join("\n");
359 let out = compress_tool_result(&raw, Some("search_files"));
360
361 assert!(
362 out.len() < raw.len(),
363 "search results must still compress ({} -> {})",
364 raw.len(),
365 out.len()
366 );
367 for keyword in ["context.Context", "(api.Result, error)", "return"] {
368 assert!(
369 out.contains(keyword),
370 "the terse dictionary rewrote `{keyword}` out of a search result:\n{out}"
371 );
372 }
373 }
374
375 #[test]
376 fn short_content_unchanged() {
377 let short = "hello world";
378 assert_eq!(compress_tool_result(short, None), short);
379 }
380
381 #[test]
382 fn empty_content_unchanged() {
383 assert_eq!(compress_tool_result("", None), "");
384 assert_eq!(compress_tool_result(" ", None), " ");
385 }
386
387 #[test]
388 fn command_hint_extraction() {
389 assert_eq!(
390 extract_command_hint("$ cargo build\nCompiling foo"),
391 Some("cargo build".to_string())
392 );
393 assert_eq!(extract_command_hint("no prefix here"), None);
394 }
395
396 #[test]
397 fn tool_name_inference() {
398 assert_eq!(infer_command("some text", Some("bash_execute")), "shell");
399 assert_eq!(infer_command("some text", Some("search_files")), "grep");
400 assert_eq!(infer_command("some text", Some("unknown_tool")), "");
401 }
402
403 #[test]
404 fn lean_ctx_tool_results_pass_through_verbatim() {
405 let raw = (1..=120)
409 .map(|i| format!("Line {i:04}: the quick brown fox jumps over the lazy dog"))
410 .collect::<Vec<_>>()
411 .join("\n");
412 assert!(raw.len() > 200);
413 for tool in [
416 "ctx_shell",
417 "ctx_read",
418 "ctx_search",
419 "ctx_grep",
420 "mcp__lean-ctx__ctx_shell",
421 "lean-ctx:ctx_read",
422 ] {
423 assert_eq!(
424 compress_tool_result(&raw, Some(tool)),
425 raw,
426 "{tool} output must pass through the proxy verbatim"
427 );
428 }
429 assert_ne!(
432 compress_tool_result(&raw, Some("bash")),
433 raw,
434 "foreign-tool output should still be compressed by the proxy"
435 );
436 }
437
438 #[test]
439 fn cited_research_output_is_preserved_verbatim() {
440 let cited = format!(
441 "Rust is a language.\n\n---\nSource: Rust — https://x.com/a\n\
442 Site: x.com · Retrieved: 2026-06-06T00:00:00Z\n{}",
443 "Extra body line that would otherwise be touched. ".repeat(20)
444 );
445 assert_eq!(compress_tool_result(&cited, Some("ctx_url_read")), cited);
446 }
447
448 #[test]
449 fn prose_is_squeezed_and_deduped() {
450 let para = "Rust is a multi-paradigm systems programming language that \
451 emphasizes performance, type safety, and fearless concurrency, \
452 achieving memory safety without a garbage collector at runtime.";
453 let input = format!("{}\n", [para; 8].join("\n\n"));
455 assert!(input.len() > 600);
456 let out = compress_tool_result(&input, Some("web_fetch"));
457 assert_eq!(out.matches("fearless concurrency").count(), 1);
458 assert!(out.contains("performance, type safety"));
459 }
460
461 #[test]
462 #[serial]
463 fn research_prose_cap_env_overrides_default() {
464 let _lock = crate::core::data_dir::test_env_lock();
465 crate::test_env::set_var(RESEARCH_PROSE_CAP_ENV, "1234");
466 assert_eq!(research_prose_cap(), 1234);
467 crate::test_env::remove_var(RESEARCH_PROSE_CAP_ENV);
468 }
469
470 #[test]
471 #[serial]
472 fn research_prose_cap_env_invalid_falls_back() {
473 let _lock = crate::core::data_dir::test_env_lock();
474 for value in ["", "not_a_number", "0"] {
475 crate::test_env::set_var(RESEARCH_PROSE_CAP_ENV, value);
476 assert_eq!(research_prose_cap(), RESEARCH_PROSE_CAP);
477 }
478 crate::test_env::remove_var(RESEARCH_PROSE_CAP_ENV);
479 }
480
481 #[test]
482 fn code_output_is_not_treated_as_prose() {
483 let code = "fn main() {\n let x = vec![1, 2, 3];\n \
484 for i in &x { println!(\"{}\", i); }\n}\n"
485 .repeat(20);
486 assert!(!looks_like_prose(&code));
487 }
488
489 #[test]
490 fn shell_log_is_not_treated_as_prose() {
491 let log = "$ cargo build\n Compiling foo v0.1.0\n Finished dev\n".repeat(20);
492 assert!(!looks_like_prose(&log));
493 }
494
495 #[test]
496 fn foreign_shell_build_failure_preserved_verbatim() {
497 let mut log = String::from("gcc -O2 -c src/versioncmp.c -o versioncmp.o\n");
501 log.push_str("src/versioncmp.c: In function 'version_cmp':\n");
502 log.push_str(
503 "src/versioncmp.c:142:17: error: invalid operands to binary < (have 'char *' and 'int')\n",
504 );
505 for i in 0..40 {
506 log.push_str(&format!(" note: expansion context line {i}\n"));
507 }
508 log.push_str("make: *** [Makefile:23: versioncmp.o] Error 1\n");
509
510 let out = compress_tool_result(&log, Some("shell"));
511 assert!(
512 out.contains("versioncmp.c:142:17: error:"),
513 "compiler error must survive the proxy"
514 );
515 assert!(
516 out.contains("make: ***"),
517 "make failure summary must survive"
518 );
519 }
520
521 #[test]
522 fn foreign_shell_test_failure_preserved_verbatim() {
523 let mut log = String::from("running 3 tests\n");
524 log.push_str("test version::tests::sorts_numeric ... FAILED\n");
525 for i in 0..40 {
526 log.push_str(&format!("note line {i} with some filler content here\n"));
527 }
528 log.push_str("test result: FAILED. 2 passed; 1 failed; 0 ignored\n");
529
530 let out = compress_tool_result(&log, Some("bash"));
531 assert!(
532 out.contains("test result: FAILED"),
533 "test summary must survive the proxy"
534 );
535 assert!(out.contains("sorts_numeric ... FAILED"));
536 }
537
538 #[test]
539 fn plain_shell_log_not_forced_verbatim() {
540 let log = "Listening on port 8080\nRequest received from 10.0.0.2\n".repeat(20);
541 assert!(!output_looks_like_test_run(&log));
542 assert!(!output_looks_like_build_failure(&log));
543 }
544
545 fn big_compressible_log() -> String {
546 (1..=400)
547 .map(|i| format!("[info] processed item {i:04} ok"))
548 .collect::<Vec<_>>()
549 .join("\n")
550 }
551
552 #[test]
553 fn live_compression_is_recoverable_via_ccr_handle() {
554 let _lock = crate::core::data_dir::test_env_lock();
555 let log = big_compressible_log();
556 let out = compress_tool_result(&log, Some("bash"));
557 assert!(
558 out.len() < log.len(),
559 "a large foreign log must be compressed"
560 );
561
562 let handle = ccr::persist(&log).expect("same content -> same handle");
566 assert!(out.contains(&handle), "CCR handle must be embedded: {out}");
567 let recovered = std::fs::read_to_string(&handle).expect("tee file readable");
568 assert!(
569 recovered.contains("processed item 0007 ok")
570 && recovered.contains("processed item 0400 ok"),
571 "verbatim original must be fully recoverable"
572 );
573 }
574
575 #[test]
576 fn live_compression_output_is_byte_stable_across_turns() {
577 let _lock = crate::core::data_dir::test_env_lock();
578 let log = big_compressible_log();
579 let a = compress_tool_result(&log, Some("bash"));
580 let b = compress_tool_result(&log, Some("bash"));
581 assert_eq!(
582 a, b,
583 "the CCR handle is content-addressed, so the rewritten result must be \
584 byte-identical across turns (provider cache prefix stays valid, #448)"
585 );
586 }
587
588 #[test]
594 fn gateway_stub_matches_litellm_marker_regex_and_retrieves() {
595 let _lock = crate::core::data_dir::test_env_lock();
596 let log = big_compressible_log();
597 let out = compress_tool_result_gateway(&log, Some("bash"));
598 assert!(out.len() < log.len(), "gateway funnel must still compress");
599
600 let litellm_regex = regex::Regex::new(r"hash=([a-f0-9]{24})").unwrap();
602 let captured = litellm_regex
603 .captures(&out)
604 .unwrap_or_else(|| panic!("gateway stub must carry a hash= marker: {out}"))
605 .get(1)
606 .unwrap()
607 .as_str();
608 assert_eq!(captured, ccr::litellm_hash(&log));
609
610 let recovered = ccr::retrieve_litellm(captured).expect("captured hash must resolve");
613 assert!(
614 recovered.contains("processed item 0007 ok")
615 && recovered.contains("processed item 0400 ok"),
616 "retrieve must return the verbatim original"
617 );
618
619 assert_eq!(out, compress_tool_result_gateway(&log, Some("bash")));
621
622 assert_eq!(
625 crate::core::protocol::strip_trailing_savings_footer(&out),
626 out
627 );
628 }
629
630 #[test]
631 fn gateway_stub_absent_for_passthrough_and_ctx_output() {
632 let _lock = crate::core::data_dir::test_env_lock();
633 assert!(!compress_tool_result_gateway("short output", Some("bash")).contains("hash="));
635 let raw = (1..=120)
638 .map(|i| format!("Line {i:04}: lorem ipsum dolor sit amet consectetur"))
639 .collect::<Vec<_>>()
640 .join("\n");
641 let out = compress_tool_result_gateway(&raw, Some("ctx_shell"));
642 assert_eq!(out, raw);
643 }
644
645 #[test]
646 fn small_or_passthrough_output_gets_no_ccr_handle() {
647 let _lock = crate::core::data_dir::test_env_lock();
648 let tiny = "ok\n".repeat(10);
650 assert!(!compress_tool_result(&tiny, Some("bash")).contains("full original at"));
651 let raw = (1..=120)
653 .map(|i| format!("Line {i:04}: lorem ipsum dolor sit amet consectetur"))
654 .collect::<Vec<_>>()
655 .join("\n");
656 let out = compress_tool_result(&raw, Some("ctx_shell"));
657 assert_eq!(out, raw, "lean-ctx tool result must stay verbatim (no CCR)");
658 }
659}