lean_ctx/core/
output_echo.rs

1//! Output-echo detection (#501).
2//!
3//! lean-ctx aggressively optimizes the *input* side, but never looked at the
4//! agent's *output* — although output tokens cost 4-5x more. The most common
5//! waste pattern is the code echo: the agent re-quotes file content that is
6//! already in context. The `afterAgentResponse` hook delivers the reply text;
7//! this module measures which share of its code lines were echoed from
8//! recently read files (context radar tail) and feeds three consumers:
9//!
10//! 1. rolling stats (`~/.lean-ctx/output_echo.json`) for `ctx_metrics`,
11//!    `ctx_session output_stats` and the dashboard,
12//! 2. an automatic `LlmFeedbackEvent` so the adaptive mode policy finally
13//!    receives continuous data instead of voluntary `ctx_feedback` calls,
14//! 3. a stable, cache-friendly CEP nudge the MCP server appends to a tool
15//!    result when echo stays high (cooldown-limited).
16
17use std::collections::HashSet;
18use std::path::PathBuf;
19
20use serde::{Deserialize, Serialize};
21
22const STATS_FILE: &str = "output_echo.json";
23/// Rolling window of analyzed responses.
24const MAX_REPORTS: usize = 50;
25/// Echo lines shorter than this are noise (`}`, `);`, `end`).
26const MIN_LINE_CHARS: usize = 12;
27/// Nudge when the average echo ratio of the recent window exceeds this.
28const NUDGE_THRESHOLD: f64 = 0.30;
29/// Responses considered for the nudge decision.
30const NUDGE_WINDOW: usize = 5;
31/// Minimum analyzed responses between two nudges.
32const NUDGE_COOLDOWN: usize = 20;
33/// How much of the radar tail to scan for source content (bytes).
34const RADAR_TAIL_BYTES: u64 = 262_144;
35/// Cap on source documents compared against.
36const MAX_SOURCES: usize = 20;
37
38#[derive(Debug, Clone, Serialize, Deserialize, Default)]
39pub struct EchoReport {
40    pub response_lines: usize,
41    pub code_lines: usize,
42    pub echoed_lines: usize,
43    pub echo_ratio: f64,
44    pub recorded_unix: u64,
45}
46
47#[derive(Debug, Clone, Serialize, Deserialize, Default)]
48pub struct EchoStats {
49    pub reports: Vec<EchoReport>,
50    /// Index (count of analyzed responses) at the last nudge.
51    pub last_nudge_at: u64,
52    /// Total responses analyzed over the lifetime of the store.
53    pub total_analyzed: u64,
54}
55
56impl EchoStats {
57    pub fn avg_ratio(&self, window: usize) -> f64 {
58        let recent: Vec<&EchoReport> = self.reports.iter().rev().take(window).collect();
59        if recent.is_empty() {
60            return 0.0;
61        }
62        recent.iter().map(|r| r.echo_ratio).sum::<f64>() / recent.len() as f64
63    }
64
65    /// Per-day `(YYYY-MM-DD, avg_echo_ratio, samples)` over the last `days`
66    /// days, ascending by day — the dashboard's learning trend (#507).
67    /// Days are UTC, consistent with the ledger's day slices.
68    pub fn daily_trend(&self, days: u32) -> Vec<(String, f64, u64)> {
69        use std::collections::BTreeMap;
70        let cutoff = now_unix().saturating_sub(u64::from(days) * 86_400);
71        let mut by_day: BTreeMap<String, (f64, u64)> = BTreeMap::new();
72        for r in &self.reports {
73            if r.recorded_unix < cutoff {
74                continue;
75            }
76            let Some(dt) = chrono::DateTime::from_timestamp(r.recorded_unix as i64, 0) else {
77                continue;
78            };
79            let day = dt.format("%Y-%m-%d").to_string();
80            let entry = by_day.entry(day).or_default();
81            entry.0 += r.echo_ratio;
82            entry.1 += 1;
83        }
84        by_day
85            .into_iter()
86            .map(|(d, (sum, n))| (d, sum / n as f64, n))
87            .collect()
88    }
89}
90
91fn stats_path() -> PathBuf {
92    crate::core::data_dir::lean_ctx_data_dir()
93        .unwrap_or_else(|_| PathBuf::from("."))
94        .join(STATS_FILE)
95}
96
97fn now_unix() -> u64 {
98    std::time::SystemTime::now()
99        .duration_since(std::time::UNIX_EPOCH)
100        .map_or(0, |d| d.as_secs())
101}
102
103pub fn load_stats() -> EchoStats {
104    std::fs::read_to_string(stats_path())
105        .ok()
106        .and_then(|raw| serde_json::from_str(&raw).ok())
107        .unwrap_or_default()
108}
109
110fn save_stats(stats: &EchoStats) {
111    let path = stats_path();
112    if let Some(parent) = path.parent() {
113        let _ = std::fs::create_dir_all(parent);
114    }
115    if let Ok(json) = serde_json::to_string(stats) {
116        let tmp = path.with_extension("tmp");
117        if std::fs::write(&tmp, json).is_ok() {
118            let _ = std::fs::rename(&tmp, &path);
119        }
120    }
121}
122
123/// Normalize a line for comparison: trim + collapse internal whitespace.
124fn normalize_line(line: &str) -> String {
125    let mut out = String::with_capacity(line.len());
126    let mut last_space = false;
127    for c in line.trim().chars() {
128        if c.is_whitespace() {
129            if !last_space {
130                out.push(' ');
131            }
132            last_space = true;
133        } else {
134            out.push(c);
135            last_space = false;
136        }
137    }
138    out
139}
140
141/// Extract the code lines of a markdown response: fenced blocks plus
142/// 4-space-indented lines. Prose is never counted as echo.
143fn code_lines_of_response(response: &str) -> Vec<String> {
144    let mut lines = Vec::new();
145    let mut in_fence = false;
146    for raw in response.lines() {
147        let trimmed = raw.trim_start();
148        if trimmed.starts_with("```") {
149            in_fence = !in_fence;
150            continue;
151        }
152        if in_fence || raw.starts_with("    ") {
153            let norm = normalize_line(raw);
154            if norm.chars().count() >= MIN_LINE_CHARS {
155                lines.push(norm);
156            }
157        }
158    }
159    lines
160}
161
162/// Pure analysis: which share of the response's code lines appear verbatim
163/// in any source document (recently read file contents)?
164pub fn analyze(response: &str, sources: &[String]) -> EchoReport {
165    let code_lines = code_lines_of_response(response);
166    let response_lines = response.lines().count();
167
168    if code_lines.is_empty() {
169        return EchoReport {
170            response_lines,
171            code_lines: 0,
172            echoed_lines: 0,
173            echo_ratio: 0.0,
174            recorded_unix: now_unix(),
175        };
176    }
177
178    let mut source_set: HashSet<String> = HashSet::new();
179    for src in sources.iter().take(MAX_SOURCES) {
180        for line in src.lines() {
181            let norm = normalize_line(line);
182            if norm.chars().count() >= MIN_LINE_CHARS {
183                source_set.insert(norm);
184            }
185        }
186    }
187
188    let echoed = code_lines
189        .iter()
190        .filter(|l| source_set.contains(*l))
191        .count();
192    let ratio = echoed as f64 / code_lines.len() as f64;
193
194    EchoReport {
195        response_lines,
196        code_lines: code_lines.len(),
197        echoed_lines: echoed,
198        echo_ratio: ratio,
199        recorded_unix: now_unix(),
200    }
201}
202
203/// Read the tail of the context radar log and collect recently delivered
204/// content (file reads, MCP tool results, shell output) as echo sources,
205/// plus the token sum of events since the last user message (turn input
206/// approximation for the feedback event).
207fn radar_tail_sources() -> (Vec<String>, u64) {
208    let data_dir =
209        crate::core::data_dir::lean_ctx_data_dir().unwrap_or_else(|_| PathBuf::from("."));
210    let path = data_dir.join("context_radar.jsonl");
211    let Ok(file) = std::fs::File::open(&path) else {
212        return (Vec::new(), 0);
213    };
214    use std::io::{Read, Seek, SeekFrom};
215    let mut file = file;
216    let len = file.metadata().map_or(0, |m| m.len());
217    let start = len.saturating_sub(RADAR_TAIL_BYTES);
218    if file.seek(SeekFrom::Start(start)).is_err() {
219        return (Vec::new(), 0);
220    }
221    let mut raw = String::new();
222    if file.read_to_string(&mut raw).is_err() {
223        return (Vec::new(), 0);
224    }
225
226    let mut sources: Vec<String> = Vec::new();
227    let mut turn_tokens: u64 = 0;
228    // Skip the first (possibly truncated) line when we started mid-file.
229    let skip_first = start > 0;
230    for (i, line) in raw.lines().enumerate() {
231        if skip_first && i == 0 {
232            continue;
233        }
234        let Ok(v) = serde_json::from_str::<serde_json::Value>(line) else {
235            continue;
236        };
237        let event_type = v.get("event_type").and_then(|e| e.as_str()).unwrap_or("");
238        let tokens = v
239            .get("tokens")
240            .and_then(serde_json::Value::as_u64)
241            .unwrap_or(0);
242        match event_type {
243            "user_message" => turn_tokens = 0,
244            "agent_response" | "thinking" => {}
245            _ => turn_tokens = turn_tokens.saturating_add(tokens),
246        }
247        if matches!(event_type, "file_read" | "mcp_call" | "shell")
248            && let Some(content) = v.get("content").and_then(|c| c.as_str())
249            && !content.is_empty()
250        {
251            sources.push(content.to_string());
252        }
253    }
254    if sources.len() > MAX_SOURCES {
255        let excess = sources.len() - MAX_SOURCES;
256        sources.drain(..excess);
257    }
258    (sources, turn_tokens)
259}
260
261/// Hook entry point: analyze an agent response against the radar tail,
262/// persist rolling stats and emit the automatic feedback event.
263pub fn analyze_and_record(response: &str) {
264    let (sources, turn_input_tokens) = radar_tail_sources();
265    let report = analyze(response, &sources);
266
267    let mut stats = load_stats();
268    stats.total_analyzed = stats.total_analyzed.saturating_add(1);
269    stats.reports.push(report.clone());
270    if stats.reports.len() > MAX_REPORTS {
271        let excess = stats.reports.len() - MAX_REPORTS;
272        stats.reports.drain(..excess);
273    }
274    save_stats(&stats);
275
276    emit_feedback_event(response, turn_input_tokens);
277}
278
279/// Automatic `LlmFeedbackEvent` (#501): the adaptive mode policy used to
280/// depend on voluntary `ctx_feedback` calls that practically never happened.
281/// Mode attribution comes from the session's `files_touched.last_mode`
282/// aggregation — a session-window approximation, honest and available.
283fn emit_feedback_event(response: &str, turn_input_tokens: u64) {
284    let output_tokens = crate::core::tokens::count_tokens(response) as u64;
285    if output_tokens == 0 {
286        return;
287    }
288
289    let session = crate::core::session::SessionState::load_latest();
290    let modes: Option<std::collections::BTreeMap<String, u64>> = session.as_ref().map(|s| {
291        let mut m = std::collections::BTreeMap::new();
292        for f in &s.files_touched {
293            if !f.last_mode.is_empty() {
294                *m.entry(f.last_mode.clone()).or_insert(0) += u64::from(f.read_count.max(1));
295            }
296        }
297        m
298    });
299    let modes = modes.filter(|m| !m.is_empty());
300
301    let model = crate::hook_handlers::load_detected_model().map(|(name, _)| name);
302
303    let ev = crate::core::llm_feedback::LlmFeedbackEvent {
304        agent_id: "output_echo_auto".to_string(),
305        intent: session
306            .as_ref()
307            .and_then(|s| s.task.as_ref())
308            .and_then(|t| t.intent.clone()),
309        model,
310        llm_input_tokens: turn_input_tokens.max(1),
311        llm_output_tokens: output_tokens,
312        latency_ms: None,
313        note: None,
314        ctx_read_last_mode: None,
315        ctx_read_modes: modes,
316        timestamp: chrono::Utc::now().to_rfc3339(),
317    };
318
319    let mut policy = crate::core::adaptive_mode_policy::AdaptiveModePolicyStore::load();
320    policy.update_from_feedback(&ev);
321    let _ = policy.save();
322    let _ = crate::core::llm_feedback::LlmFeedbackStore::record(ev);
323}
324
325/// CEP nudge for the MCP server to append to a tool result. Stable text in
326/// 10%-steps (prompt-cache friendly, #498), cooldown-limited. Consuming the
327/// nudge advances the cooldown marker.
328pub fn take_pending_nudge() -> Option<String> {
329    let mut stats = load_stats();
330    if stats.reports.len() < NUDGE_WINDOW {
331        return None;
332    }
333    if stats.total_analyzed.saturating_sub(stats.last_nudge_at) < NUDGE_COOLDOWN as u64 {
334        return None;
335    }
336    let avg = stats.avg_ratio(NUDGE_WINDOW);
337    if avg < NUDGE_THRESHOLD {
338        return None;
339    }
340    let rounded = ((avg * 10.0).round() * 10.0) as u32;
341    stats.last_nudge_at = stats.total_analyzed;
342    save_stats(&stats);
343    Some(format!(
344        "\n[CEP: ~{rounded}% of your recent replies echoed file content already in context — reference lines (F1:42-58) instead of re-quoting]"
345    ))
346}
347
348/// Average echo ratio over the rolling window — CEP score input.
349pub fn current_avg_ratio() -> f64 {
350    load_stats().avg_ratio(MAX_REPORTS)
351}
352
353#[cfg(test)]
354mod tests {
355    use super::*;
356
357    fn file_content() -> String {
358        (0..30)
359            .map(|i| format!("pub fn compute_value_{i}(input: u32) -> u32 {{ input * {i} }}"))
360            .collect::<Vec<_>>()
361            .join("\n")
362    }
363
364    #[test]
365    fn echo_detected_for_quoted_file_content() {
366        let src = file_content();
367        let quoted: Vec<&str> = src.lines().take(10).collect();
368        let response = format!(
369            "Here is the relevant code:\n\n```rust\n{}\n```\n",
370            quoted.join("\n")
371        );
372        let report = analyze(&response, &[src]);
373        assert_eq!(report.code_lines, 10);
374        assert_eq!(report.echoed_lines, 10);
375        assert!(report.echo_ratio > 0.99);
376    }
377
378    #[test]
379    fn prose_response_has_zero_echo() {
380        let src = file_content();
381        let response = "The cache works by storing entries keyed by path. \
382                        No code needed here — the fix is a one-line change.";
383        let report = analyze(response, &[src]);
384        assert_eq!(report.code_lines, 0);
385        assert!(report.echo_ratio.abs() < f64::EPSILON);
386    }
387
388    #[test]
389    fn daily_trend_groups_by_utc_day_and_averages() {
390        let now = now_unix();
391        // Anchor both "today" samples to the start of the current UTC day. A naive
392        // `now` / `now - 60` pair straddles midnight when the suite runs within 60s
393        // of a UTC day rollover, yielding a spurious third day (flaky in CI).
394        let day = now - (now % 86_400);
395        let stats = EchoStats {
396            reports: vec![
397                // Two samples on the same UTC day (today): avg of 0.2 and 0.6 = 0.4.
398                EchoReport {
399                    echo_ratio: 0.2,
400                    recorded_unix: day + 100,
401                    ..Default::default()
402                },
403                EchoReport {
404                    echo_ratio: 0.6,
405                    recorded_unix: day + 200,
406                    ..Default::default()
407                },
408                // One sample two UTC days ago.
409                EchoReport {
410                    echo_ratio: 1.0,
411                    recorded_unix: day - 2 * 86_400 + 100,
412                    ..Default::default()
413                },
414                // Outside the 14-day window — must be excluded.
415                EchoReport {
416                    echo_ratio: 1.0,
417                    recorded_unix: day - 30 * 86_400,
418                    ..Default::default()
419                },
420            ],
421            ..Default::default()
422        };
423        let trend = stats.daily_trend(14);
424        assert_eq!(trend.len(), 2, "two distinct days inside the window");
425        // Ascending by day: the older day first.
426        assert_eq!(trend[0].2, 1, "older day has one sample");
427        assert!((trend[0].1 - 1.0).abs() < f64::EPSILON);
428        assert_eq!(trend[1].2, 2, "today has two samples");
429        assert!((trend[1].1 - 0.4).abs() < 1e-9);
430    }
431
432    #[test]
433    fn short_lines_are_ignored() {
434        let src = "}\n);\nend\nfn x() {}\n".to_string();
435        let response = "```rust\n}\n);\nend\n```\n";
436        let report = analyze(response, &[src]);
437        assert_eq!(report.code_lines, 0, "sub-12-char lines never count");
438    }
439
440    #[test]
441    fn novel_code_is_not_echo() {
442        let src = file_content();
443        let response = "```rust\npub fn completely_new_function(a: u64) -> u64 { a + 42 }\nlet result = completely_new_function(7);\n```";
444        let report = analyze(response, &[src]);
445        assert_eq!(report.echoed_lines, 0);
446        assert!(report.echo_ratio.abs() < f64::EPSILON);
447    }
448
449    #[test]
450    fn whitespace_differences_still_match() {
451        let src = "pub fn   spaced_out(value:    u32)   -> u32 { value }".to_string();
452        let response = "```rust\npub fn spaced_out(value: u32) -> u32 { value }\n```";
453        let report = analyze(response, &[src]);
454        assert_eq!(report.echoed_lines, 1);
455    }
456
457    #[test]
458    fn avg_ratio_windows_correctly() {
459        let mut stats = EchoStats::default();
460        for ratio in [0.0, 0.2, 0.4, 0.6, 0.8] {
461            stats.reports.push(EchoReport {
462                response_lines: 10,
463                code_lines: 10,
464                echoed_lines: (ratio * 10.0) as usize,
465                echo_ratio: ratio,
466                recorded_unix: 0,
467            });
468        }
469        assert!((stats.avg_ratio(5) - 0.4).abs() < 1e-9);
470        assert!((stats.avg_ratio(2) - 0.7).abs() < 1e-9);
471    }
472
473    #[test]
474    fn indented_code_outside_fences_counts() {
475        let src = "    let total = items.iter().map(|i| i.price).sum::<f64>();".to_string();
476        let response = "The sum is computed like this:\n\n    let total = items.iter().map(|i| i.price).sum::<f64>();\n";
477        let report = analyze(response, &[src]);
478        assert_eq!(report.code_lines, 1);
479        assert_eq!(report.echoed_lines, 1);
480    }
481}
lean_ctx/core/output_echo.rs

lean_ctx/core/
output_echo.rs
