lean_ctx/core/terse/
mod.rs

1//! # Terse Compression Engine
2//!
3//! Unified, 4-layer token compression pipeline that replaces the legacy
4//! `compress_terse`/`compress_ultra` functions and the `TerseAgent` prompt system.
5//!
6//! ## Architecture
7//!
8//! ```text
9//! Tool Output → [Layer 1: Deterministic] → [Layer 2: Residual] → Compressed Output
10//! MCP Tools List → [Layer 4: Description Terse] → Compact Descriptions
11//! ```
12//!
13//! ## Layers
14//!
15//! - **Layer 1** (`engine.rs`): Deterministic output compression — surprisal scoring,
16//!   content/function word filtering, domain dictionaries, quality gate.
17//! - **Layer 2** (`residual.rs`): Pattern-aware post-terse — applies after pattern
18//!   compression, avoids double-compression, tracks attribution.
19//! - **Layer 4** (`mcp_compress.rs`): MCP description compression — shrinks tool
20//!   descriptions, lazy-load stubs, on-demand expansion.
21
22pub mod counter;
23pub mod dictionaries;
24pub mod engine;
25pub mod mcp_compress;
26pub mod pipeline;
27pub mod quality;
28pub mod residual;
29pub mod scoring;
30
31/// Tools whose textual output is read content the agent edits against and must
32/// therefore be returned byte-for-byte.
33const READ_FAMILY: &[&str] = &[
34    "ctx_read",
35    "ctx_multi_read",
36    "ctx_smart_read",
37    "ctx_compress",
38    "ctx_overview",
39];
40
41/// Whether a tool's output must be returned verbatim and so must never pass
42/// through the prose terse pipeline (#404).
43///
44/// Returns true for any read-family tool — those already apply their own
45/// mode-aware, structure-preserving compression, and a `full`/`lines:` read
46/// promises complete, edit-against-able content — and, as defense in depth, for
47/// any call whose `mode` is itself a verbatim mode (`full`, `raw`, `lines:N-M`).
48/// The mode arm protects a *future* read tool or caller by construction, even
49/// before it is added to `READ_FAMILY`. Shared by the MCP post-processor
50/// (`skip_terse`) and the CLI `read` command so both paths stay byte-exact.
51#[must_use]
52pub fn is_verbatim_read(name: &str, mode: Option<&str>) -> bool {
53    if READ_FAMILY.contains(&name) {
54        return true;
55    }
56    mode.is_some_and(|m| m == "full" || m == "raw" || m.starts_with("lines:"))
57}
58
59/// Result of a compression pipeline run with full attribution.
60#[derive(Debug, Clone)]
61pub struct TerseResult {
62    pub output: String,
63    pub tokens_before: u32,
64    pub tokens_after: u32,
65    pub savings_pct: f32,
66    pub layers_applied: Vec<&'static str>,
67    pub pattern_savings: u32,
68    pub terse_savings: u32,
69    pub quality_passed: bool,
70}
71
72impl TerseResult {
73    pub fn passthrough(text: String, tokens: u32) -> Self {
74        Self {
75            output: text,
76            tokens_before: tokens,
77            tokens_after: tokens,
78            savings_pct: 0.0,
79            layers_applied: Vec::new(),
80            pattern_savings: 0,
81            terse_savings: 0,
82            quality_passed: true,
83        }
84    }
85}
86
87#[cfg(test)]
88mod verbatim_read_tests {
89    use super::is_verbatim_read;
90
91    #[test]
92    fn read_family_is_always_verbatim() {
93        for name in [
94            "ctx_read",
95            "ctx_multi_read",
96            "ctx_smart_read",
97            "ctx_compress",
98            "ctx_overview",
99        ] {
100            // Even an intentionally-lossy mode like `signatures` is exempt: the
101            // read tool applies its own structure-preserving compression and the
102            // generic prose terse layer must never run on top of it (#404).
103            assert!(is_verbatim_read(name, Some("signatures")), "{name}");
104            assert!(is_verbatim_read(name, None), "{name}");
105        }
106    }
107
108    #[test]
109    fn verbatim_modes_protect_any_tool() {
110        // Defense-in-depth: a future read tool/caller using a verbatim mode is
111        // protected by construction, before it joins the name list.
112        for mode in ["full", "raw", "lines:1-40", "lines:10-10"] {
113            assert!(is_verbatim_read("ctx_future_reader", Some(mode)), "{mode}");
114        }
115    }
116
117    #[test]
118    fn non_read_lossy_modes_stay_eligible() {
119        for mode in ["map", "aggressive", "entropy", "signatures"] {
120            assert!(
121                !is_verbatim_read("ctx_search", Some(mode)),
122                "non-read {mode} must remain terse-eligible"
123            );
124        }
125        assert!(!is_verbatim_read("ctx_shell", None));
126    }
127}
lean_ctx/core/terse/mod.rs

lean_ctx/core/terse/
mod.rs
