lean_ctx/core/
output_echo.rs

1//! Output-echo detection (#501).
2//!
3//! lean-ctx aggressively optimizes the *input* side, but never looked at the
4//! agent's *output* — although output tokens cost 4-5x more. The most common
5//! waste pattern is the code echo: the agent re-quotes file content that is
6//! already in context. The `afterAgentResponse` hook delivers the reply text;
7//! this module measures which share of its code lines were echoed from
8//! recently read files (context radar tail) and feeds three consumers:
9//!
10//! 1. rolling stats (`~/.lean-ctx/output_echo.json`) for `ctx_metrics`,
11//!    `ctx_session output_stats` and the dashboard,
12//! 2. an automatic `LlmFeedbackEvent` so the adaptive mode policy finally
13//!    receives continuous data instead of voluntary `ctx_feedback` calls,
14//! 3. a stable, cache-friendly CEP nudge the MCP server appends to a tool
15//!    result when echo stays high (cooldown-limited).
16
17use std::collections::HashSet;
18use std::path::PathBuf;
19
20use serde::{Deserialize, Serialize};
21
22const STATS_FILE: &str = "output_echo.json";
23/// Rolling window of analyzed responses.
24const MAX_REPORTS: usize = 50;
25/// Echo lines shorter than this are noise (`}`, `);`, `end`).
26const MIN_LINE_CHARS: usize = 12;
27/// Nudge when the average echo ratio of the recent window exceeds this.
28const NUDGE_THRESHOLD: f64 = 0.30;
29/// Responses considered for the nudge decision.
30const NUDGE_WINDOW: usize = 5;
31/// Minimum analyzed responses between two nudges.
32const NUDGE_COOLDOWN: usize = 20;
33/// How much of the radar tail to scan for source content (bytes).
34const RADAR_TAIL_BYTES: u64 = 262_144;
35/// Cap on source documents compared against.
36const MAX_SOURCES: usize = 20;
37
38#[derive(Debug, Clone, Serialize, Deserialize, Default)]
39pub struct EchoReport {
40    pub response_lines: usize,
41    pub code_lines: usize,
42    pub echoed_lines: usize,
43    pub echo_ratio: f64,
44    pub recorded_unix: u64,
45}
46
47#[derive(Debug, Clone, Serialize, Deserialize, Default)]
48pub struct EchoStats {
49    pub reports: Vec<EchoReport>,
50    /// Index (count of analyzed responses) at the last nudge.
51    pub last_nudge_at: u64,
52    /// Total responses analyzed over the lifetime of the store.
53    pub total_analyzed: u64,
54}
55
56impl EchoStats {
57    pub fn avg_ratio(&self, window: usize) -> f64 {
58        let recent: Vec<&EchoReport> = self.reports.iter().rev().take(window).collect();
59        if recent.is_empty() {
60            return 0.0;
61        }
62        recent.iter().map(|r| r.echo_ratio).sum::<f64>() / recent.len() as f64
63    }
64
65    /// Per-day `(YYYY-MM-DD, avg_echo_ratio, samples)` over the last `days`
66    /// days, ascending by day — the dashboard's learning trend (#507).
67    /// Days are UTC, consistent with the ledger's day slices.
68    pub fn daily_trend(&self, days: u32) -> Vec<(String, f64, u64)> {
69        use std::collections::BTreeMap;
70        let cutoff = now_unix().saturating_sub(u64::from(days) * 86_400);
71        let mut by_day: BTreeMap<String, (f64, u64)> = BTreeMap::new();
72        for r in &self.reports {
73            if r.recorded_unix < cutoff {
74                continue;
75            }
76            let Some(dt) = chrono::DateTime::from_timestamp(r.recorded_unix as i64, 0) else {
77                continue;
78            };
79            let day = dt.format("%Y-%m-%d").to_string();
80            let entry = by_day.entry(day).or_default();
81            entry.0 += r.echo_ratio;
82            entry.1 += 1;
83        }
84        by_day
85            .into_iter()
86            .map(|(d, (sum, n))| (d, sum / n as f64, n))
87            .collect()
88    }
89}
90
91fn stats_path() -> PathBuf {
92    crate::core::data_dir::lean_ctx_data_dir()
93        .unwrap_or_else(|_| PathBuf::from("."))
94        .join(STATS_FILE)
95}
96
97fn now_unix() -> u64 {
98    std::time::SystemTime::now()
99        .duration_since(std::time::UNIX_EPOCH)
100        .map_or(0, |d| d.as_secs())
101}
102
103pub fn load_stats() -> EchoStats {
104    std::fs::read_to_string(stats_path())
105        .ok()
106        .and_then(|raw| serde_json::from_str(&raw).ok())
107        .unwrap_or_default()
108}
109
110fn save_stats(stats: &EchoStats) {
111    let path = stats_path();
112    if let Some(parent) = path.parent() {
113        let _ = std::fs::create_dir_all(parent);
114    }
115    if let Ok(json) = serde_json::to_string(stats) {
116        let tmp = path.with_extension("tmp");
117        if std::fs::write(&tmp, json).is_ok() {
118            let _ = std::fs::rename(&tmp, &path);
119        }
120    }
121}
122
123/// Normalize a line for comparison: trim + collapse internal whitespace.
124fn normalize_line(line: &str) -> String {
125    let mut out = String::with_capacity(line.len());
126    let mut last_space = false;
127    for c in line.trim().chars() {
128        if c.is_whitespace() {
129            if !last_space {
130                out.push(' ');
131            }
132            last_space = true;
133        } else {
134            out.push(c);
135            last_space = false;
136        }
137    }
138    out
139}
140
141/// Extract the code lines of a markdown response: fenced blocks plus
142/// 4-space-indented lines. Prose is never counted as echo.
143fn code_lines_of_response(response: &str) -> Vec<String> {
144    let mut lines = Vec::new();
145    let mut in_fence = false;
146    for raw in response.lines() {
147        let trimmed = raw.trim_start();
148        if trimmed.starts_with("```") {
149            in_fence = !in_fence;
150            continue;
151        }
152        if in_fence || raw.starts_with("    ") {
153            let norm = normalize_line(raw);
154            if norm.chars().count() >= MIN_LINE_CHARS {
155                lines.push(norm);
156            }
157        }
158    }
159    lines
160}
161
162/// Pure analysis: which share of the response's code lines appear verbatim
163/// in any source document (recently read file contents)?
164pub fn analyze(response: &str, sources: &[String]) -> EchoReport {
165    let code_lines = code_lines_of_response(response);
166    let response_lines = response.lines().count();
167
168    if code_lines.is_empty() {
169        return EchoReport {
170            response_lines,
171            code_lines: 0,
172            echoed_lines: 0,
173            echo_ratio: 0.0,
174            recorded_unix: now_unix(),
175        };
176    }
177
178    let mut source_set: HashSet<String> = HashSet::new();
179    for src in sources.iter().take(MAX_SOURCES) {
180        for line in src.lines() {
181            let norm = normalize_line(line);
182            if norm.chars().count() >= MIN_LINE_CHARS {
183                source_set.insert(norm);
184            }
185        }
186    }
187
188    let echoed = code_lines
189        .iter()
190        .filter(|l| source_set.contains(*l))
191        .count();
192    let ratio = echoed as f64 / code_lines.len() as f64;
193
194    EchoReport {
195        response_lines,
196        code_lines: code_lines.len(),
197        echoed_lines: echoed,
198        echo_ratio: ratio,
199        recorded_unix: now_unix(),
200    }
201}
202
203/// Read the tail of the context radar log and collect recently delivered
204/// content (file reads, MCP tool results, shell output) as echo sources,
205/// plus the token sum of events since the last user message (turn input
206/// approximation for the feedback event).
207fn radar_tail_sources() -> (Vec<String>, u64) {
208    let data_dir =
209        crate::core::data_dir::lean_ctx_data_dir().unwrap_or_else(|_| PathBuf::from("."));
210    let path = data_dir.join("context_radar.jsonl");
211    let Ok(file) = std::fs::File::open(&path) else {
212        return (Vec::new(), 0);
213    };
214    use std::io::{Read, Seek, SeekFrom};
215    let mut file = file;
216    let len = file.metadata().map_or(0, |m| m.len());
217    let start = len.saturating_sub(RADAR_TAIL_BYTES);
218    if file.seek(SeekFrom::Start(start)).is_err() {
219        return (Vec::new(), 0);
220    }
221    let mut raw = String::new();
222    if file.read_to_string(&mut raw).is_err() {
223        return (Vec::new(), 0);
224    }
225
226    let mut sources: Vec<String> = Vec::new();
227    let mut turn_tokens: u64 = 0;
228    // Skip the first (possibly truncated) line when we started mid-file.
229    let skip_first = start > 0;
230    for (i, line) in raw.lines().enumerate() {
231        if skip_first && i == 0 {
232            continue;
233        }
234        let Ok(v) = serde_json::from_str::<serde_json::Value>(line) else {
235            continue;
236        };
237        let event_type = v.get("event_type").and_then(|e| e.as_str()).unwrap_or("");
238        let tokens = v
239            .get("tokens")
240            .and_then(serde_json::Value::as_u64)
241            .unwrap_or(0);
242        match event_type {
243            "user_message" => turn_tokens = 0,
244            "agent_response" | "thinking" => {}
245            _ => turn_tokens = turn_tokens.saturating_add(tokens),
246        }
247        if matches!(event_type, "file_read" | "mcp_call" | "shell") {
248            if let Some(content) = v.get("content").and_then(|c| c.as_str()) {
249                if !content.is_empty() {
250                    sources.push(content.to_string());
251                }
252            }
253        }
254    }
255    if sources.len() > MAX_SOURCES {
256        let excess = sources.len() - MAX_SOURCES;
257        sources.drain(..excess);
258    }
259    (sources, turn_tokens)
260}
261
262/// Hook entry point: analyze an agent response against the radar tail,
263/// persist rolling stats and emit the automatic feedback event.
264pub fn analyze_and_record(response: &str) {
265    let (sources, turn_input_tokens) = radar_tail_sources();
266    let report = analyze(response, &sources);
267
268    let mut stats = load_stats();
269    stats.total_analyzed = stats.total_analyzed.saturating_add(1);
270    stats.reports.push(report.clone());
271    if stats.reports.len() > MAX_REPORTS {
272        let excess = stats.reports.len() - MAX_REPORTS;
273        stats.reports.drain(..excess);
274    }
275    save_stats(&stats);
276
277    emit_feedback_event(response, turn_input_tokens);
278}
279
280/// Automatic `LlmFeedbackEvent` (#501): the adaptive mode policy used to
281/// depend on voluntary `ctx_feedback` calls that practically never happened.
282/// Mode attribution comes from the session's `files_touched.last_mode`
283/// aggregation — a session-window approximation, honest and available.
284fn emit_feedback_event(response: &str, turn_input_tokens: u64) {
285    let output_tokens = crate::core::tokens::count_tokens(response) as u64;
286    if output_tokens == 0 {
287        return;
288    }
289
290    let session = crate::core::session::SessionState::load_latest();
291    let modes: Option<std::collections::BTreeMap<String, u64>> = session.as_ref().map(|s| {
292        let mut m = std::collections::BTreeMap::new();
293        for f in &s.files_touched {
294            if !f.last_mode.is_empty() {
295                *m.entry(f.last_mode.clone()).or_insert(0) += u64::from(f.read_count.max(1));
296            }
297        }
298        m
299    });
300    let modes = modes.filter(|m| !m.is_empty());
301
302    let model = crate::hook_handlers::load_detected_model().map(|(name, _)| name);
303
304    let ev = crate::core::llm_feedback::LlmFeedbackEvent {
305        agent_id: "output_echo_auto".to_string(),
306        intent: session
307            .as_ref()
308            .and_then(|s| s.task.as_ref())
309            .and_then(|t| t.intent.clone()),
310        model,
311        llm_input_tokens: turn_input_tokens.max(1),
312        llm_output_tokens: output_tokens,
313        latency_ms: None,
314        note: None,
315        ctx_read_last_mode: None,
316        ctx_read_modes: modes,
317        timestamp: chrono::Utc::now().to_rfc3339(),
318    };
319
320    let mut policy = crate::core::adaptive_mode_policy::AdaptiveModePolicyStore::load();
321    policy.update_from_feedback(&ev);
322    let _ = policy.save();
323    let _ = crate::core::llm_feedback::LlmFeedbackStore::record(ev);
324}
325
326/// CEP nudge for the MCP server to append to a tool result. Stable text in
327/// 10%-steps (prompt-cache friendly, #498), cooldown-limited. Consuming the
328/// nudge advances the cooldown marker.
329pub fn take_pending_nudge() -> Option<String> {
330    let mut stats = load_stats();
331    if stats.reports.len() < NUDGE_WINDOW {
332        return None;
333    }
334    if stats.total_analyzed.saturating_sub(stats.last_nudge_at) < NUDGE_COOLDOWN as u64 {
335        return None;
336    }
337    let avg = stats.avg_ratio(NUDGE_WINDOW);
338    if avg < NUDGE_THRESHOLD {
339        return None;
340    }
341    let rounded = ((avg * 10.0).round() * 10.0) as u32;
342    stats.last_nudge_at = stats.total_analyzed;
343    save_stats(&stats);
344    Some(format!(
345        "\n[CEP: ~{rounded}% of your recent replies echoed file content already in context — reference lines (F1:42-58) instead of re-quoting]"
346    ))
347}
348
349/// Average echo ratio over the rolling window — CEP score input.
350pub fn current_avg_ratio() -> f64 {
351    load_stats().avg_ratio(MAX_REPORTS)
352}
353
354#[cfg(test)]
355mod tests {
356    use super::*;
357
358    fn file_content() -> String {
359        (0..30)
360            .map(|i| format!("pub fn compute_value_{i}(input: u32) -> u32 {{ input * {i} }}"))
361            .collect::<Vec<_>>()
362            .join("\n")
363    }
364
365    #[test]
366    fn echo_detected_for_quoted_file_content() {
367        let src = file_content();
368        let quoted: Vec<&str> = src.lines().take(10).collect();
369        let response = format!(
370            "Here is the relevant code:\n\n```rust\n{}\n```\n",
371            quoted.join("\n")
372        );
373        let report = analyze(&response, &[src]);
374        assert_eq!(report.code_lines, 10);
375        assert_eq!(report.echoed_lines, 10);
376        assert!(report.echo_ratio > 0.99);
377    }
378
379    #[test]
380    fn prose_response_has_zero_echo() {
381        let src = file_content();
382        let response = "The cache works by storing entries keyed by path. \
383                        No code needed here — the fix is a one-line change.";
384        let report = analyze(response, &[src]);
385        assert_eq!(report.code_lines, 0);
386        assert!(report.echo_ratio.abs() < f64::EPSILON);
387    }
388
389    #[test]
390    fn daily_trend_groups_by_utc_day_and_averages() {
391        let now = now_unix();
392        // Anchor both "today" samples to the start of the current UTC day. A naive
393        // `now` / `now - 60` pair straddles midnight when the suite runs within 60s
394        // of a UTC day rollover, yielding a spurious third day (flaky in CI).
395        let day = now - (now % 86_400);
396        let stats = EchoStats {
397            reports: vec![
398                // Two samples on the same UTC day (today): avg of 0.2 and 0.6 = 0.4.
399                EchoReport {
400                    echo_ratio: 0.2,
401                    recorded_unix: day + 100,
402                    ..Default::default()
403                },
404                EchoReport {
405                    echo_ratio: 0.6,
406                    recorded_unix: day + 200,
407                    ..Default::default()
408                },
409                // One sample two UTC days ago.
410                EchoReport {
411                    echo_ratio: 1.0,
412                    recorded_unix: day - 2 * 86_400 + 100,
413                    ..Default::default()
414                },
415                // Outside the 14-day window — must be excluded.
416                EchoReport {
417                    echo_ratio: 1.0,
418                    recorded_unix: day - 30 * 86_400,
419                    ..Default::default()
420                },
421            ],
422            ..Default::default()
423        };
424        let trend = stats.daily_trend(14);
425        assert_eq!(trend.len(), 2, "two distinct days inside the window");
426        // Ascending by day: the older day first.
427        assert_eq!(trend[0].2, 1, "older day has one sample");
428        assert!((trend[0].1 - 1.0).abs() < f64::EPSILON);
429        assert_eq!(trend[1].2, 2, "today has two samples");
430        assert!((trend[1].1 - 0.4).abs() < 1e-9);
431    }
432
433    #[test]
434    fn short_lines_are_ignored() {
435        let src = "}\n);\nend\nfn x() {}\n".to_string();
436        let response = "```rust\n}\n);\nend\n```\n";
437        let report = analyze(response, &[src]);
438        assert_eq!(report.code_lines, 0, "sub-12-char lines never count");
439    }
440
441    #[test]
442    fn novel_code_is_not_echo() {
443        let src = file_content();
444        let response = "```rust\npub fn completely_new_function(a: u64) -> u64 { a + 42 }\nlet result = completely_new_function(7);\n```";
445        let report = analyze(response, &[src]);
446        assert_eq!(report.echoed_lines, 0);
447        assert!(report.echo_ratio.abs() < f64::EPSILON);
448    }
449
450    #[test]
451    fn whitespace_differences_still_match() {
452        let src = "pub fn   spaced_out(value:    u32)   -> u32 { value }".to_string();
453        let response = "```rust\npub fn spaced_out(value: u32) -> u32 { value }\n```";
454        let report = analyze(response, &[src]);
455        assert_eq!(report.echoed_lines, 1);
456    }
457
458    #[test]
459    fn avg_ratio_windows_correctly() {
460        let mut stats = EchoStats::default();
461        for ratio in [0.0, 0.2, 0.4, 0.6, 0.8] {
462            stats.reports.push(EchoReport {
463                response_lines: 10,
464                code_lines: 10,
465                echoed_lines: (ratio * 10.0) as usize,
466                echo_ratio: ratio,
467                recorded_unix: 0,
468            });
469        }
470        assert!((stats.avg_ratio(5) - 0.4).abs() < 1e-9);
471        assert!((stats.avg_ratio(2) - 0.7).abs() < 1e-9);
472    }
473
474    #[test]
475    fn indented_code_outside_fences_counts() {
476        let src = "    let total = items.iter().map(|i| i.price).sum::<f64>();".to_string();
477        let response = "The sum is computed like this:\n\n    let total = items.iter().map(|i| i.price).sum::<f64>();\n";
478        let report = analyze(response, &[src]);
479        assert_eq!(report.code_lines, 1);
480        assert_eq!(report.echoed_lines, 1);
481    }
482}
lean_ctx/core/output_echo.rs

lean_ctx/core/
output_echo.rs
