lean_ctx/tools/
ctx_response.rs

1use crate::core::tokens::count_tokens;
2use crate::tools::CrpMode;
3
4pub fn handle(response: &str, crp_mode: CrpMode) -> String {
5    let original_tokens = count_tokens(response);
6
7    if original_tokens <= 100 {
8        return response.to_string();
9    }
10
11    let compressed = if crp_mode.is_tdd() {
12        compress_tdd(response)
13    } else {
14        compress_standard(response)
15    };
16
17    let compressed_tokens = count_tokens(&compressed);
18    let savings = original_tokens.saturating_sub(compressed_tokens);
19    let pct = if original_tokens > 0 {
20        (savings as f64 / original_tokens as f64 * 100.0) as u32
21    } else {
22        0
23    };
24
25    if savings < 20 {
26        return response.to_string();
27    }
28
29    format!(
30        "{compressed}\n[response compressed: {original_tokens}→{compressed_tokens} tok, -{pct}%]"
31    )
32}
33
34fn compress_standard(text: &str) -> String {
35    let mut result = Vec::new();
36    let mut prev_empty = false;
37
38    for line in text.lines() {
39        let trimmed = line.trim();
40
41        if trimmed.is_empty() {
42            if !prev_empty {
43                result.push(String::new());
44                prev_empty = true;
45            }
46            continue;
47        }
48        prev_empty = false;
49
50        if is_filler_line(trimmed) {
51            continue;
52        }
53
54        result.push(line.to_string());
55    }
56
57    result.join("\n")
58}
59
60fn compress_tdd(text: &str) -> String {
61    let mut result = Vec::new();
62    let mut prev_empty = false;
63
64    for line in text.lines() {
65        let trimmed = line.trim();
66
67        if trimmed.is_empty() {
68            if !prev_empty {
69                prev_empty = true;
70            }
71            continue;
72        }
73        prev_empty = false;
74
75        if is_filler_line(trimmed) {
76            continue;
77        }
78
79        let compressed = apply_tdd_shortcuts(trimmed);
80        result.push(compressed);
81    }
82
83    result.join("\n")
84}
85
86fn is_filler_line(line: &str) -> bool {
87    let l = line.to_lowercase();
88
89    // Preserve lines with genuine information signals
90    if l.starts_with("note:")
91        || l.starts_with("hint:")
92        || l.starts_with("warning:")
93        || l.starts_with("error:")
94        || l.starts_with("however,")
95        || l.starts_with("but ")
96        || l.starts_with("caution:")
97        || l.starts_with("important:")
98    {
99        return false;
100    }
101
102    // H=0 patterns: carry zero task-relevant information
103    let prefix_fillers = [
104        // Narration / preamble
105        "here's what i",
106        "here is what i",
107        "let me explain",
108        "let me walk you",
109        "let me break",
110        "i'll now",
111        "i will now",
112        "i'm going to",
113        "first, let me",
114        "allow me to",
115        // Hedging
116        "i think",
117        "i believe",
118        "i would say",
119        "it seems like",
120        "it looks like",
121        "it appears that",
122        // Meta-commentary
123        "that's a great question",
124        "that's an interesting",
125        "good question",
126        "great question",
127        "sure thing",
128        "sure,",
129        "of course,",
130        "absolutely,",
131        // Transitions (zero-info)
132        "now, let's",
133        "now let's",
134        "next, i'll",
135        "moving on",
136        "going forward",
137        "with that said",
138        "with that in mind",
139        "having said that",
140        "that being said",
141        // Closings
142        "hope this helps",
143        "i hope this",
144        "let me know if",
145        "feel free to",
146        "don't hesitate",
147        "happy to help",
148        // Filler connectives
149        "as you can see",
150        "as we can see",
151        "this is because",
152        "the reason is",
153        "in this case",
154        "in other words",
155        "to summarize",
156        "to sum up",
157        "basically,",
158        "essentially,",
159        "it's worth noting",
160        "it should be noted",
161        "as mentioned",
162        "as i mentioned",
163        // Acknowledgments
164        "understood.",
165        "got it.",
166        "i understand.",
167        "i see.",
168        "right,",
169        "okay,",
170        "ok,",
171    ];
172
173    prefix_fillers.iter().any(|f| l.starts_with(f))
174}
175
176fn apply_tdd_shortcuts(line: &str) -> String {
177    let mut result = line.to_string();
178
179    let replacements = [
180        // Structural
181        ("function", "fn"),
182        ("configuration", "cfg"),
183        ("implementation", "impl"),
184        ("dependencies", "deps"),
185        ("dependency", "dep"),
186        ("request", "req"),
187        ("response", "res"),
188        ("context", "ctx"),
189        ("parameter", "param"),
190        ("argument", "arg"),
191        ("variable", "val"),
192        ("directory", "dir"),
193        ("repository", "repo"),
194        ("application", "app"),
195        ("environment", "env"),
196        ("description", "desc"),
197        ("information", "info"),
198        // Symbols (1 token each, replaces 5-10 tokens of prose)
199        ("returns ", "→ "),
200        ("therefore", "∴"),
201        ("approximately", "≈"),
202        ("successfully", "✓"),
203        ("completed", "✓"),
204        ("failed", "✗"),
205        ("warning", "⚠"),
206        // Operators
207        (" is not ", " ≠ "),
208        (" does not ", " ≠ "),
209        (" equals ", " = "),
210        (" and ", " & "),
211        ("error", "err"),
212        ("module", "mod"),
213        ("package", "pkg"),
214        ("initialize", "init"),
215    ];
216
217    for (from, to) in &replacements {
218        result = result.replace(from, to);
219    }
220
221    result
222}
223
224#[cfg(test)]
225mod tests {
226    use super::*;
227
228    #[test]
229    fn test_filler_detection_original() {
230        assert!(is_filler_line("Here's what I found"));
231        assert!(is_filler_line("Let me explain how this works"));
232        assert!(!is_filler_line("fn main() {}"));
233        assert!(!is_filler_line("Note: important detail"));
234    }
235
236    #[test]
237    fn test_filler_hedging_patterns() {
238        assert!(is_filler_line("I think the issue is here"));
239        assert!(is_filler_line("I believe this is correct"));
240        assert!(is_filler_line("It seems like the problem is"));
241        assert!(is_filler_line("It looks like we need to"));
242        assert!(is_filler_line("It appears that something broke"));
243    }
244
245    #[test]
246    fn test_filler_meta_commentary() {
247        assert!(is_filler_line("That's a great question!"));
248        assert!(is_filler_line("Good question, let me check"));
249        assert!(is_filler_line("Sure thing, I'll do that"));
250        assert!(is_filler_line("Of course, here's the code"));
251        assert!(is_filler_line("Absolutely, that makes sense"));
252    }
253
254    #[test]
255    fn test_filler_closings() {
256        assert!(is_filler_line("Hope this helps!"));
257        assert!(is_filler_line("Let me know if you need more"));
258        assert!(is_filler_line("Feel free to ask questions"));
259        assert!(is_filler_line("Don't hesitate to reach out"));
260        assert!(is_filler_line("Happy to help with anything"));
261    }
262
263    #[test]
264    fn test_filler_transitions() {
265        assert!(is_filler_line("Now, let's move on"));
266        assert!(is_filler_line("Moving on to the next part"));
267        assert!(is_filler_line("Going forward, we should"));
268        assert!(is_filler_line("With that said, here's what"));
269        assert!(is_filler_line("Having said that, let's"));
270    }
271
272    #[test]
273    fn test_filler_acknowledgments() {
274        assert!(is_filler_line("Understood."));
275        assert!(is_filler_line("Got it."));
276        assert!(is_filler_line("I understand."));
277        assert!(is_filler_line("I see."));
278    }
279
280    #[test]
281    fn test_filler_false_positive_protection() {
282        assert!(!is_filler_line("Note: this is critical"));
283        assert!(!is_filler_line("Warning: deprecated API"));
284        assert!(!is_filler_line("Error: connection refused"));
285        assert!(!is_filler_line("However, the edge case fails"));
286        assert!(!is_filler_line("But the second argument is wrong"));
287        assert!(!is_filler_line("Important: do not skip this step"));
288        assert!(!is_filler_line("Caution: this deletes all data"));
289        assert!(!is_filler_line("Hint: use --force flag"));
290        assert!(!is_filler_line("fn validate_token()"));
291        assert!(!is_filler_line("  let result = parse(input);"));
292        assert!(!is_filler_line("The token is expired after 24h"));
293    }
294
295    #[test]
296    fn test_tdd_shortcuts() {
297        let result = apply_tdd_shortcuts("the function returns successfully");
298        assert!(result.contains("fn"));
299        assert!(result.contains("→"));
300        assert!(result.contains("✓"));
301    }
302
303    #[test]
304    fn test_tdd_shortcuts_extended() {
305        let result = apply_tdd_shortcuts("the application environment failed");
306        assert!(result.contains("app"));
307        assert!(result.contains("env"));
308        assert!(result.contains("✗"));
309    }
310
311    #[test]
312    fn test_compress_integration() {
313        let response = "Let me explain how this works.\n\
314            I think this is correct.\n\
315            Hope this helps!\n\
316            \n\
317            The function returns an error when the token is expired.\n\
318            Note: always check the expiry first.";
319
320        let compressed = compress_standard(response);
321        assert!(!compressed.contains("Let me explain"));
322        assert!(!compressed.contains("I think"));
323        assert!(!compressed.contains("Hope this helps"));
324        assert!(compressed.contains("error when the token"));
325        assert!(compressed.contains("Note:"));
326    }
327}
lean_ctx/tools/ctx_response.rs

lean_ctx/tools/
ctx_response.rs
