lean_ctx/core/terse/
mod.rs

1//! # Terse Compression Engine
2//!
3//! Unified, 4-layer token compression pipeline that replaces the legacy
4//! `compress_terse`/`compress_ultra` functions and the `TerseAgent` prompt system.
5//!
6//! ## Architecture
7//!
8//! ```text
9//! Tool Output → [Layer 1: Deterministic] → [Layer 2: Residual] → Compressed Output
10//! MCP Tools List → [Layer 4: Description Terse] → Compact Descriptions
11//! ```
12//!
13//! ## Layers
14//!
15//! - **Layer 1** (`engine.rs`): Deterministic output compression — surprisal scoring,
16//!   content/function word filtering, domain dictionaries, quality gate.
17//! - **Layer 2** (`residual.rs`): Pattern-aware post-terse — applies after pattern
18//!   compression, avoids double-compression, tracks attribution.
19//! - **Layer 4** (`mcp_compress.rs`): MCP description compression — shrinks tool
20//!   descriptions, lazy-load stubs, on-demand expansion.
21
22pub mod auto_dict;
23pub mod counter;
24pub mod dictionaries;
25pub mod engine;
26pub mod mcp_compress;
27pub mod pipeline;
28pub mod quality;
29pub mod residual;
30pub mod scoring;
31
32/// Tools whose textual output is read content the agent edits against and must
33/// therefore be returned byte-for-byte.
34const READ_FAMILY: &[&str] = &[
35    "ctx_read",
36    "ctx_multi_read",
37    "ctx_smart_read",
38    "ctx_compress",
39    "ctx_overview",
40];
41
42/// Whether a tool's output must be returned verbatim and so must never pass
43/// through the prose terse pipeline (#404).
44///
45/// Returns true for any read-family tool — those already apply their own
46/// mode-aware, structure-preserving compression, and a `full`/`lines:` read
47/// promises complete, edit-against-able content — and, as defense in depth, for
48/// any call whose `mode` is itself a verbatim mode (`full`, `raw`, `lines:N-M`).
49/// The mode arm protects a *future* read tool or caller by construction, even
50/// before it is added to `READ_FAMILY`. Shared by the MCP post-processor
51/// (`skip_terse`) and the CLI `read` command so both paths stay byte-exact.
52#[must_use]
53pub fn is_verbatim_read(name: &str, mode: Option<&str>) -> bool {
54    if READ_FAMILY.contains(&name) {
55        return true;
56    }
57    mode.is_some_and(|m| m == "full" || m == "raw" || m.starts_with("lines:"))
58}
59
60/// Result of a compression pipeline run with full attribution.
61#[derive(Debug, Clone)]
62pub struct TerseResult {
63    pub output: String,
64    pub tokens_before: u32,
65    pub tokens_after: u32,
66    pub savings_pct: f32,
67    pub layers_applied: Vec<&'static str>,
68    pub pattern_savings: u32,
69    pub terse_savings: u32,
70    pub quality_passed: bool,
71}
72
73impl TerseResult {
74    pub fn passthrough(text: String, tokens: u32) -> Self {
75        Self {
76            output: text,
77            tokens_before: tokens,
78            tokens_after: tokens,
79            savings_pct: 0.0,
80            layers_applied: Vec::new(),
81            pattern_savings: 0,
82            terse_savings: 0,
83            quality_passed: true,
84        }
85    }
86}
87
88#[cfg(test)]
89mod verbatim_read_tests {
90    use super::is_verbatim_read;
91
92    #[test]
93    fn read_family_is_always_verbatim() {
94        for name in [
95            "ctx_read",
96            "ctx_multi_read",
97            "ctx_smart_read",
98            "ctx_compress",
99            "ctx_overview",
100        ] {
101            // Even an intentionally-lossy mode like `signatures` is exempt: the
102            // read tool applies its own structure-preserving compression and the
103            // generic prose terse layer must never run on top of it (#404).
104            assert!(is_verbatim_read(name, Some("signatures")), "{name}");
105            assert!(is_verbatim_read(name, None), "{name}");
106        }
107    }
108
109    #[test]
110    fn verbatim_modes_protect_any_tool() {
111        // Defense-in-depth: a future read tool/caller using a verbatim mode is
112        // protected by construction, before it joins the name list.
113        for mode in ["full", "raw", "lines:1-40", "lines:10-10"] {
114            assert!(is_verbatim_read("ctx_future_reader", Some(mode)), "{mode}");
115        }
116    }
117
118    #[test]
119    fn non_read_lossy_modes_stay_eligible() {
120        for mode in ["map", "aggressive", "entropy", "signatures"] {
121            assert!(
122                !is_verbatim_read("ctx_search", Some(mode)),
123                "non-read {mode} must remain terse-eligible"
124            );
125        }
126        assert!(!is_verbatim_read("ctx_shell", None));
127    }
128}
lean_ctx/core/terse/mod.rs

lean_ctx/core/terse/
mod.rs
