lean_ctx/core/
efficacy.rs

1//! Learning-efficacy evidence (#549, VIS-3).
2//!
3//! The learning layers (#538-#544) adapt continuously — this module proves
4//! whether the adaptation WORKS, with real telemetry instead of claims:
5//!
6//! - **Bounce-rate trend** straight from the savings ledger
7//!   (`daily_bounce_trend`, #507): week-over-week rate of compressed reads
8//!   that had to be re-read full. Learned thresholds (#538) must push this
9//!   down.
10//! - **LITM placement snapshots**: daily cumulative hit/miss counters from
11//!   the calibration store (#539) so hit-rate movement is visible over time.
12//! - **Playbook survival** (#541): share of entries that stayed net-helpful
13//!   past 10 turns — the ACE quality proxy for "facts worth keeping".
14//! - **Prevented duplicate work** (#540): lifetime count of rejected scent
15//!   claims.
16//!
17//! Snapshots live in `~/.lean-ctx/efficacy_snapshots.json`, bounded to a
18//! 30-day ring, captured lazily on every `ctx_metrics` call and on server
19//! shutdown — no timers, no daemons.
20
21use serde::{Deserialize, Serialize};
22
23/// Ring size: 30 calendar days of snapshots.
24const MAX_SNAPSHOTS: usize = 30;
25/// Playbook entries older than this many turns count toward survival stats.
26const SURVIVAL_AGE_TURNS: u32 = 10;
27
28#[derive(Debug, Clone, Serialize, Deserialize)]
29pub struct EfficacySnapshot {
30    /// Calendar day, `YYYY-MM-DD` (UTC).
31    pub day: String,
32    /// Cumulative LITM counters at capture time (#539).
33    pub litm_begin_hits: u32,
34    pub litm_begin_misses: u32,
35    pub litm_end_hits: u32,
36    pub litm_end_misses: u32,
37    /// Cumulative rejected scent claims at capture time (#540).
38    pub claims_rejected: u64,
39    /// Playbook size and net-helpful aged entries at capture time (#541).
40    pub playbook_entries: usize,
41    pub playbook_aged_helpful: usize,
42    pub playbook_aged_total: usize,
43}
44
45#[derive(Debug, Clone, Serialize, Deserialize, Default)]
46pub struct EfficacyStore {
47    pub snapshots: Vec<EfficacySnapshot>,
48    pub schema_version: u32,
49}
50
51fn store_path() -> std::path::PathBuf {
52    crate::core::data_dir::lean_ctx_data_dir()
53        .unwrap_or_else(|_| std::path::PathBuf::from("."))
54        .join("efficacy_snapshots.json")
55}
56
57impl EfficacyStore {
58    fn load() -> Self {
59        if let Ok(content) = std::fs::read_to_string(store_path()) {
60            if let Ok(s) = serde_json::from_str::<EfficacyStore>(&content) {
61                return s;
62            }
63        }
64        EfficacyStore {
65            schema_version: 1,
66            ..Default::default()
67        }
68    }
69
70    fn save(&self) {
71        let path = store_path();
72        if let Some(parent) = path.parent() {
73            let _ = std::fs::create_dir_all(parent);
74        }
75        if let Ok(json) = serde_json::to_string(self) {
76            let _ = std::fs::write(path, json);
77        }
78    }
79
80    /// Upsert today's snapshot (cumulative counters only move forward, so
81    /// re-capturing within the same day just refreshes the values) and trim
82    /// the ring.
83    pub fn upsert(&mut self, snap: EfficacySnapshot) {
84        match self.snapshots.iter_mut().find(|s| s.day == snap.day) {
85            Some(existing) => *existing = snap,
86            None => self.snapshots.push(snap),
87        }
88        self.snapshots.sort_by(|a, b| a.day.cmp(&b.day));
89        if self.snapshots.len() > MAX_SNAPSHOTS {
90            let excess = self.snapshots.len() - MAX_SNAPSHOTS;
91            self.snapshots.drain(0..excess);
92        }
93    }
94}
95
96/// Build today's snapshot from the live stores.
97fn current_snapshot() -> EfficacySnapshot {
98    let (bh, bm, eh, em) = crate::core::litm_calibration::totals();
99
100    let (entries, aged_helpful, aged_total) = crate::core::session::SessionState::load_latest()
101        .map_or((0, 0, 0), |s| {
102            let turn = s.stats.total_tool_calls;
103            let aged: Vec<_> = s
104                .playbook
105                .entries
106                .iter()
107                .filter(|e| turn.saturating_sub(e.created_turn) >= SURVIVAL_AGE_TURNS)
108                .collect();
109            let helpful = aged
110                .iter()
111                .filter(|e| e.helpful_votes >= e.harmful_votes)
112                .count();
113            (s.playbook.entries.len(), helpful, aged.len())
114        });
115
116    EfficacySnapshot {
117        day: chrono::Utc::now().format("%Y-%m-%d").to_string(),
118        litm_begin_hits: bh,
119        litm_begin_misses: bm,
120        litm_end_hits: eh,
121        litm_end_misses: em,
122        claims_rejected: crate::core::scent_field::claims_rejected_total(),
123        playbook_entries: entries,
124        playbook_aged_helpful: aged_helpful,
125        playbook_aged_total: aged_total,
126    }
127}
128
129/// Capture (upsert) today's snapshot. Called from ctx_metrics and shutdown.
130pub fn capture() {
131    let mut store = EfficacyStore::load();
132    store.upsert(current_snapshot());
133    store.save();
134}
135
136/// Week-over-week bounce rates from the ledger: `(previous, recent)` as
137/// `(rate, reads)` tuples over 7-day windows. `None` when a window has no
138/// compressed reads to be honest about.
139fn bounce_week_over_week() -> (Option<(f64, u64)>, Option<(f64, u64)>) {
140    let trend = crate::core::savings_ledger::daily_bounce_trend(14);
141    if trend.is_empty() {
142        return (None, None);
143    }
144    let today = chrono::Utc::now().date_naive();
145    let mut prev = (0u64, 0u64); // (bounces, reads) days 8-14
146    let mut recent = (0u64, 0u64); // days 0-7
147    for (day, bounces, reads) in &trend {
148        let Ok(d) = chrono::NaiveDate::parse_from_str(day, "%Y-%m-%d") else {
149            continue;
150        };
151        let age = (today - d).num_days();
152        if age < 7 {
153            recent.0 += bounces;
154            recent.1 += reads;
155        } else {
156            prev.0 += bounces;
157            prev.1 += reads;
158        }
159    }
160    let rate = |(b, r): (u64, u64)| {
161        if r == 0 {
162            None
163        } else {
164            Some((b as f64 / r as f64, r))
165        }
166    };
167    (rate(prev), rate(recent))
168}
169
170fn fmt_pct(x: f64) -> String {
171    format!("{:.1}%", x * 100.0)
172}
173
174/// Human-readable efficacy section for ctx_metrics.
175pub fn report() -> Vec<String> {
176    let mut out = Vec::new();
177
178    match bounce_week_over_week() {
179        (Some((prev, prev_n)), Some((rec, rec_n))) => {
180            let arrow = if rec < prev {
181                "improving"
182            } else if rec > prev {
183                "regressing"
184            } else {
185                "flat"
186            };
187            out.push(format!(
188                "bounce rate: {} (prev 7d, n={prev_n}) -> {} (last 7d, n={rec_n}) [{arrow}]",
189                fmt_pct(prev),
190                fmt_pct(rec)
191            ));
192        }
193        (None, Some((rec, rec_n))) => {
194            out.push(format!(
195                "bounce rate: {} (last 7d, n={rec_n}) — no prior week yet",
196                fmt_pct(rec)
197            ));
198        }
199        _ => {}
200    }
201
202    let store = EfficacyStore::load();
203    if let (Some(first), Some(last)) = (store.snapshots.first(), store.snapshots.last()) {
204        if first.day != last.day {
205            let hit_rate = |s: &EfficacySnapshot| {
206                let hits = u64::from(s.litm_begin_hits) + u64::from(s.litm_end_hits);
207                let total = hits + u64::from(s.litm_begin_misses) + u64::from(s.litm_end_misses);
208                if total == 0 {
209                    None
210                } else {
211                    Some(hits as f64 / total as f64)
212                }
213            };
214            if let (Some(a), Some(b)) = (hit_rate(first), hit_rate(last)) {
215                out.push(format!(
216                    "litm placement hits: {} ({}) -> {} ({})",
217                    fmt_pct(a),
218                    first.day,
219                    fmt_pct(b),
220                    last.day
221                ));
222            }
223            let delta = last.claims_rejected.saturating_sub(first.claims_rejected);
224            if delta > 0 {
225                out.push(format!(
226                    "duplicate work prevented: {delta} rejected claim(s) since {}",
227                    first.day
228                ));
229            }
230        }
231    }
232    if let Some(last) = store.snapshots.last() {
233        if last.playbook_aged_total > 0 {
234            out.push(format!(
235                "playbook survival: {}/{} aged entries net-helpful ({})",
236                last.playbook_aged_helpful,
237                last.playbook_aged_total,
238                fmt_pct(last.playbook_aged_helpful as f64 / last.playbook_aged_total as f64)
239            ));
240        }
241    }
242
243    out
244}
245
246/// Machine-readable efficacy for the dashboard (#548).
247pub fn report_json() -> serde_json::Value {
248    let (prev, recent) = bounce_week_over_week();
249    let store = EfficacyStore::load();
250    serde_json::json!({
251        "bounce": {
252            "prev_week": prev.map(|(r, n)| serde_json::json!({"rate": r, "reads": n})),
253            "last_week": recent.map(|(r, n)| serde_json::json!({"rate": r, "reads": n})),
254            "daily": crate::core::savings_ledger::daily_bounce_trend(14)
255                .into_iter()
256                .map(|(d, b, r)| serde_json::json!({"day": d, "bounces": b, "reads": r}))
257                .collect::<Vec<_>>(),
258        },
259        "snapshots": store.snapshots,
260        "claims_rejected_total": crate::core::scent_field::claims_rejected_total(),
261    })
262}
263
264#[cfg(test)]
265mod tests {
266    use super::*;
267
268    fn snap(day: &str, claims: u64) -> EfficacySnapshot {
269        EfficacySnapshot {
270            day: day.to_string(),
271            litm_begin_hits: 10,
272            litm_begin_misses: 2,
273            litm_end_hits: 5,
274            litm_end_misses: 3,
275            claims_rejected: claims,
276            playbook_entries: 4,
277            playbook_aged_helpful: 3,
278            playbook_aged_total: 4,
279        }
280    }
281
282    #[test]
283    fn upsert_replaces_same_day_and_appends_new() {
284        let mut s = EfficacyStore::default();
285        s.upsert(snap("2026-06-10", 1));
286        s.upsert(snap("2026-06-10", 2));
287        assert_eq!(s.snapshots.len(), 1);
288        assert_eq!(s.snapshots[0].claims_rejected, 2);
289        s.upsert(snap("2026-06-11", 3));
290        assert_eq!(s.snapshots.len(), 2);
291    }
292
293    #[test]
294    fn ring_is_bounded_to_30_days() {
295        let mut s = EfficacyStore::default();
296        for i in 0..40 {
297            s.upsert(snap(&format!("2026-05-{:02}", i % 31 + 1), i));
298        }
299        // 31 distinct days collapse into <=30 after trimming, oldest dropped.
300        assert!(s.snapshots.len() <= MAX_SNAPSHOTS);
301        assert!(s.snapshots.first().unwrap().day.as_str() > "2026-05-01");
302    }
303
304    #[test]
305    fn snapshots_stay_sorted_by_day() {
306        let mut s = EfficacyStore::default();
307        s.upsert(snap("2026-06-11", 1));
308        s.upsert(snap("2026-06-09", 1));
309        s.upsert(snap("2026-06-10", 1));
310        let days: Vec<&str> = s.snapshots.iter().map(|x| x.day.as_str()).collect();
311        assert_eq!(days, vec!["2026-06-09", "2026-06-10", "2026-06-11"]);
312    }
313
314    #[test]
315    fn current_snapshot_has_today() {
316        let snap = current_snapshot();
317        assert_eq!(snap.day, chrono::Utc::now().format("%Y-%m-%d").to_string());
318    }
319}
lean_ctx/core/efficacy.rs

lean_ctx/core/
efficacy.rs
