lean_ctx/core/
efficacy.rs

1//! Learning-efficacy evidence (#549, VIS-3).
2//!
3//! The learning layers (#538-#544) adapt continuously — this module proves
4//! whether the adaptation WORKS, with real telemetry instead of claims:
5//!
6//! - **Bounce-rate trend** straight from the savings ledger
7//!   (`daily_bounce_trend`, #507): week-over-week rate of compressed reads
8//!   that had to be re-read full. Learned thresholds (#538) must push this
9//!   down.
10//! - **Playbook survival** (#541): share of entries that stayed net-helpful
11//!   past 10 turns — the ACE quality proxy for "facts worth keeping".
12//! - **Prevented duplicate work** (#540): lifetime count of rejected scent
13//!   claims.
14//!
15//! Snapshots live in `~/.lean-ctx/efficacy_snapshots.json`, bounded to a
16//! 30-day ring, captured lazily on every `ctx_metrics` call and on server
17//! shutdown — no timers, no daemons.
18
19use serde::{Deserialize, Serialize};
20
21/// Ring size: 30 calendar days of snapshots.
22const MAX_SNAPSHOTS: usize = 30;
23/// Playbook entries older than this many turns count toward survival stats.
24const SURVIVAL_AGE_TURNS: u32 = 10;
25
26#[derive(Debug, Clone, Serialize, Deserialize)]
27pub(crate) struct EfficacySnapshot {
28    /// Calendar day, `YYYY-MM-DD` (UTC).
29    pub day: String,
30    /// Cumulative rejected scent claims at capture time (#540).
31    pub claims_rejected: u64,
32    /// Playbook size and net-helpful aged entries at capture time (#541).
33    pub playbook_entries: usize,
34    pub playbook_aged_helpful: usize,
35    pub playbook_aged_total: usize,
36}
37
38#[derive(Debug, Clone, Serialize, Deserialize, Default)]
39pub(crate) struct EfficacyStore {
40    pub snapshots: Vec<EfficacySnapshot>,
41    pub schema_version: u32,
42}
43
44fn store_path() -> std::path::PathBuf {
45    crate::core::paths::cache_dir()
46        .unwrap_or_else(|_| std::path::PathBuf::from("."))
47        .join("efficacy_snapshots.json")
48}
49
50impl EfficacyStore {
51    pub(crate) fn load() -> Self {
52        if let Ok(content) = std::fs::read_to_string(store_path())
53            && let Ok(s) = serde_json::from_str::<EfficacyStore>(&content)
54        {
55            return s;
56        }
57        EfficacyStore {
58            schema_version: 1,
59            ..Default::default()
60        }
61    }
62
63    fn save(&self) {
64        let path = store_path();
65        if let Some(parent) = path.parent() {
66            let _ = std::fs::create_dir_all(parent);
67        }
68        if let Ok(json) = serde_json::to_string(self) {
69            let _ = std::fs::write(path, json);
70        }
71    }
72
73    /// Upsert today's snapshot (cumulative counters only move forward, so
74    /// re-capturing within the same day just refreshes the values) and trim
75    /// the ring.
76    pub(crate) fn upsert(&mut self, snap: EfficacySnapshot) {
77        match self.snapshots.iter_mut().find(|s| s.day == snap.day) {
78            Some(existing) => *existing = snap,
79            None => self.snapshots.push(snap),
80        }
81        self.snapshots.sort_by(|a, b| a.day.cmp(&b.day));
82        if self.snapshots.len() > MAX_SNAPSHOTS {
83            let excess = self.snapshots.len() - MAX_SNAPSHOTS;
84            self.snapshots.drain(0..excess);
85        }
86    }
87}
88
89/// Build today's snapshot from the live stores.
90fn current_snapshot() -> EfficacySnapshot {
91    let (entries, aged_helpful, aged_total) = crate::core::session::SessionState::load_latest()
92        .map_or((0, 0, 0), |s| {
93            let turn = s.stats.total_tool_calls;
94            let aged: Vec<_> = s
95                .playbook
96                .entries
97                .iter()
98                .filter(|e| turn.saturating_sub(e.created_turn) >= SURVIVAL_AGE_TURNS)
99                .collect();
100            let helpful = aged
101                .iter()
102                .filter(|e| e.helpful_votes >= e.harmful_votes)
103                .count();
104            (s.playbook.entries.len(), helpful, aged.len())
105        });
106
107    EfficacySnapshot {
108        day: chrono::Utc::now().format("%Y-%m-%d").to_string(),
109        claims_rejected: crate::core::scent_field::claims_rejected_total(),
110        playbook_entries: entries,
111        playbook_aged_helpful: aged_helpful,
112        playbook_aged_total: aged_total,
113    }
114}
115
116/// Capture (upsert) today's snapshot. Called from ctx_metrics and shutdown.
117pub fn capture() {
118    let mut store = EfficacyStore::load();
119    store.upsert(current_snapshot());
120    store.save();
121}
122
123/// Week-over-week bounce rates from the ledger: `(previous, recent)` as
124/// `(rate, reads)` tuples over 7-day windows. `None` when a window has no
125/// compressed reads to be honest about.
126fn bounce_week_over_week() -> (Option<(f64, u64)>, Option<(f64, u64)>) {
127    let trend = crate::core::savings_ledger::daily_bounce_trend(14);
128    if trend.is_empty() {
129        return (None, None);
130    }
131    let today = chrono::Utc::now().date_naive();
132    let mut prev = (0u64, 0u64); // (bounces, reads) days 8-14
133    let mut recent = (0u64, 0u64); // days 0-7
134    for (day, bounces, reads) in &trend {
135        let Ok(d) = chrono::NaiveDate::parse_from_str(day, "%Y-%m-%d") else {
136            continue;
137        };
138        let age = (today - d).num_days();
139        if age < 7 {
140            recent.0 += bounces;
141            recent.1 += reads;
142        } else {
143            prev.0 += bounces;
144            prev.1 += reads;
145        }
146    }
147    let rate = |(b, r): (u64, u64)| {
148        if r == 0 {
149            None
150        } else {
151            Some((b as f64 / r as f64, r))
152        }
153    };
154    (rate(prev), rate(recent))
155}
156
157fn fmt_pct(x: f64) -> String {
158    format!("{:.1}%", x * 100.0)
159}
160
161/// Human-readable efficacy section for ctx_metrics.
162pub fn report() -> Vec<String> {
163    let mut out = Vec::new();
164
165    match bounce_week_over_week() {
166        (Some((prev, prev_n)), Some((rec, rec_n))) => {
167            let arrow = if rec < prev {
168                "improving"
169            } else if rec > prev {
170                "regressing"
171            } else {
172                "flat"
173            };
174            out.push(format!(
175                "bounce rate: {} (prev 7d, n={prev_n}) -> {} (last 7d, n={rec_n}) [{arrow}]",
176                fmt_pct(prev),
177                fmt_pct(rec)
178            ));
179        }
180        (None, Some((rec, rec_n))) => {
181            out.push(format!(
182                "bounce rate: {} (last 7d, n={rec_n}) — no prior week yet",
183                fmt_pct(rec)
184            ));
185        }
186        _ => {}
187    }
188
189    let store = EfficacyStore::load();
190    if let (Some(first), Some(last)) = (store.snapshots.first(), store.snapshots.last())
191        && first.day != last.day
192    {
193        let delta = last.claims_rejected.saturating_sub(first.claims_rejected);
194        if delta > 0 {
195            out.push(format!(
196                "duplicate work prevented: {delta} rejected claim(s) since {}",
197                first.day
198            ));
199        }
200    }
201    if let Some(last) = store.snapshots.last()
202        && last.playbook_aged_total > 0
203    {
204        out.push(format!(
205            "playbook survival: {}/{} aged entries net-helpful ({})",
206            last.playbook_aged_helpful,
207            last.playbook_aged_total,
208            fmt_pct(last.playbook_aged_helpful as f64 / last.playbook_aged_total as f64)
209        ));
210    }
211
212    out
213}
214
215/// Machine-readable efficacy for the dashboard (#548).
216pub fn report_json() -> serde_json::Value {
217    let (prev, recent) = bounce_week_over_week();
218    let store = EfficacyStore::load();
219    serde_json::json!({
220        "bounce": {
221            "prev_week": prev.map(|(r, n)| serde_json::json!({"rate": r, "reads": n})),
222            "last_week": recent.map(|(r, n)| serde_json::json!({"rate": r, "reads": n})),
223            "daily": crate::core::savings_ledger::daily_bounce_trend(14)
224                .into_iter()
225                .map(|(d, b, r)| serde_json::json!({"day": d, "bounces": b, "reads": r}))
226                .collect::<Vec<_>>(),
227        },
228        "snapshots": store.snapshots,
229        "claims_rejected_total": crate::core::scent_field::claims_rejected_total(),
230    })
231}
232
233#[cfg(test)]
234mod tests {
235    use super::*;
236
237    fn snap(day: &str, claims: u64) -> EfficacySnapshot {
238        EfficacySnapshot {
239            day: day.to_string(),
240            claims_rejected: claims,
241            playbook_entries: 4,
242            playbook_aged_helpful: 3,
243            playbook_aged_total: 4,
244        }
245    }
246
247    #[test]
248    fn upsert_replaces_same_day_and_appends_new() {
249        let mut s = EfficacyStore::default();
250        s.upsert(snap("2026-06-10", 1));
251        s.upsert(snap("2026-06-10", 2));
252        assert_eq!(s.snapshots.len(), 1);
253        assert_eq!(s.snapshots[0].claims_rejected, 2);
254        s.upsert(snap("2026-06-11", 3));
255        assert_eq!(s.snapshots.len(), 2);
256    }
257
258    #[test]
259    fn ring_is_bounded_to_30_days() {
260        let mut s = EfficacyStore::default();
261        for i in 0..40 {
262            s.upsert(snap(&format!("2026-05-{:02}", i % 31 + 1), i));
263        }
264        // 31 distinct days collapse into <=30 after trimming, oldest dropped.
265        assert!(s.snapshots.len() <= MAX_SNAPSHOTS);
266        assert!(s.snapshots.first().unwrap().day.as_str() > "2026-05-01");
267    }
268
269    #[test]
270    fn snapshots_stay_sorted_by_day() {
271        let mut s = EfficacyStore::default();
272        s.upsert(snap("2026-06-11", 1));
273        s.upsert(snap("2026-06-09", 1));
274        s.upsert(snap("2026-06-10", 1));
275        let days: Vec<&str> = s.snapshots.iter().map(|x| x.day.as_str()).collect();
276        assert_eq!(days, vec!["2026-06-09", "2026-06-10", "2026-06-11"]);
277    }
278
279    #[test]
280    fn a_store_with_retired_litm_counters_still_loads() {
281        let json = r#"{"schema_version":1,"snapshots":[{"day":"2026-06-10",
282            "litm_begin_hits":10,"litm_begin_misses":2,"litm_end_hits":5,"litm_end_misses":3,
283            "claims_rejected":4,"playbook_entries":1,"playbook_aged_helpful":1,
284            "playbook_aged_total":1}]}"#;
285        let store: EfficacyStore = serde_json::from_str(json).expect("old store loads");
286        assert_eq!(store.snapshots.len(), 1);
287        assert_eq!(store.snapshots[0].claims_rejected, 4);
288        let saved = serde_json::to_string(&store).expect("serialize");
289        assert!(!saved.contains("litm"), "{saved}");
290    }
291
292    #[test]
293    fn current_snapshot_has_today() {
294        let snap = current_snapshot();
295        assert_eq!(snap.day, chrono::Utc::now().format("%Y-%m-%d").to_string());
296    }
297}
lean_ctx/core/efficacy.rs

lean_ctx/core/
efficacy.rs
