lean_ctx/core/terse/
mod.rs

1//! # Terse Compression Engine
2//!
3//! Unified, 4-layer token compression pipeline that replaces the legacy
4//! `compress_terse`/`compress_ultra` functions and the `TerseAgent` prompt system.
5//!
6//! ## Architecture
7//!
8//! ```text
9//! Tool Output → [Layer 1: Deterministic] → [Layer 2: Residual] → Compressed Output
10//! Model Prompt → [Layer 3: Agent Shaping] → Optimized Prompt
11//! MCP Tools List → [Layer 4: Description Terse] → Compact Descriptions
12//! ```
13//!
14//! ## Layers
15//!
16//! - **Layer 1** (`engine.rs`): Deterministic output compression — surprisal scoring,
17//!   content/function word filtering, domain dictionaries, quality gate.
18//! - **Layer 2** (`residual.rs`): Pattern-aware post-terse — applies after pattern
19//!   compression, avoids double-compression, tracks attribution.
20//! - **Layer 3** (`agent_prompts.rs`): Agent output shaping — scale-aware brevity
21//!   prompts, Telegraph-English-inspired format, adaptive levels.
22//! - **Layer 4** (`mcp_compress.rs`): MCP description compression — shrinks tool
23//!   descriptions, lazy-load stubs, on-demand expansion.
24
25pub mod agent_prompts;
26pub mod counter;
27pub mod dictionaries;
28pub mod engine;
29pub mod mcp_compress;
30pub mod pipeline;
31pub mod quality;
32pub mod residual;
33pub mod rules_inject;
34pub mod scoring;
35
36/// Result of a compression pipeline run with full attribution.
37#[derive(Debug, Clone)]
38pub struct TerseResult {
39    pub output: String,
40    pub tokens_before: u32,
41    pub tokens_after: u32,
42    pub savings_pct: f32,
43    pub layers_applied: Vec<&'static str>,
44    pub pattern_savings: u32,
45    pub terse_savings: u32,
46    pub quality_passed: bool,
47}
48
49impl TerseResult {
50    pub fn passthrough(text: String, tokens: u32) -> Self {
51        Self {
52            output: text,
53            tokens_before: tokens,
54            tokens_after: tokens,
55            savings_pct: 0.0,
56            layers_applied: Vec::new(),
57            pattern_savings: 0,
58            terse_savings: 0,
59            quality_passed: true,
60        }
61    }
62
63    pub fn format_footer(&self) -> String {
64        if self.savings_pct < 1.0 {
65            return String::new();
66        }
67        format!(
68            "[lean-ctx: {}→{} tok, pattern:-{} terse:-{}, -{:.0}%]",
69            self.tokens_before,
70            self.tokens_after,
71            self.pattern_savings,
72            self.terse_savings,
73            self.savings_pct,
74        )
75    }
76}
lean_ctx/core/terse/mod.rs

lean_ctx/core/terse/
mod.rs
