lean_ctx/core/
edit_metering.rs

1//! Edit-efficiency metering (#1008, honest-metering philosophy #361).
2//!
3//! Anchored editing (`ctx_patch`) saves *output* tokens: the model references a
4//! `(line, hash)` anchor instead of reproducing the replaced span byte-for-byte
5//! the way a str_replace `old_string` requires. This module measures that claim
6//! with real per-edit numbers instead of a marketing multiplier:
7//!
8//! * **avoided output tokens** — per successful anchored op:
9//!   `tokens(replaced span) − tokens(anchor args)`, floored at 0. The replaced
10//!   span is exactly what a str_replace edit would have re-emitted as
11//!   `old_string`; the anchor args are what the model actually sent instead.
12//! * **conflict round-trips** — stale-anchor `CONFLICT` responses (each one is
13//!   an extra turn the anchored loop needed).
14//! * **str_replace baseline** — successful `ctx_edit` calls with the
15//!   `old_string` tokens they really paid, plus `old_string`-miss round-trips.
16//!
17//! This is a **separate metric channel**: values are never folded into the
18//! read-gain ledger and never appear in tool output bodies (#498 determinism).
19//! Consumers are `ctx_metrics`, the dashboard (`/api/stats` →
20//! `edit_efficiency`) and the A/B eval harness.
21//!
22//! Storage: `~/.lean-ctx/edit_metering.json`, atomic write (tmp+rename),
23//! loaded once per process, flushed every few records like `edit_quality`.
24
25use std::sync::atomic::{AtomicUsize, Ordering};
26use std::sync::{Mutex, OnceLock};
27
28use serde::{Deserialize, Serialize};
29
30#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default)]
31#[allow(dead_code)]
32pub struct EditMetrics {
33    pub lines_added: u64,
34    pub lines_removed: u64,
35    pub net_loc_delta: i64,
36    pub edits_count: u32,
37    pub files_touched: u32,
38}
39
40const STORE_FILE: &str = "edit_metering.json";
41const FLUSH_EVERY: usize = 5;
42
43static STORE: OnceLock<Mutex<EditMeteringStore>> = OnceLock::new();
44static RECORD_CALLS: AtomicUsize = AtomicUsize::new(0);
45
46/// All-time counters for both edit paths. Small, append-only aggregates —
47/// per-file/per-op detail intentionally lives in `edit_quality`, not here.
48/// `serde(default)` keeps older/partial store files loadable field-by-field
49/// instead of silently resetting all counters via `unwrap_or_default`.
50#[derive(Debug, Clone, Serialize, Deserialize, Default)]
51#[serde(default)]
52pub(crate) struct EditMeteringStore {
53    /// Successful `ctx_patch` calls (a batch counts once).
54    pub anchored_calls: u64,
55    /// Anchored ops applied across those calls.
56    pub anchored_ops: u64,
57    /// Σ `max(0, tokens(replaced span) − tokens(anchor args))` over applied ops.
58    pub anchored_avoided_output_tokens: u64,
59    /// Stale-anchor `CONFLICT` responses (self-heal retry round-trips).
60    pub anchored_conflicts: u64,
61    /// Successful `ctx_edit` (str_replace) calls.
62    pub str_replace_calls: u64,
63    /// Σ `tokens(old_string)` those calls actually paid in output.
64    pub str_replace_old_string_tokens: u64,
65    /// `old_string`-not-found responses (blind retry round-trips).
66    pub str_replace_misses: u64,
67    /// Lines added across all edit operations.
68    pub loc_added: u64,
69    /// Lines removed across all edit operations.
70    pub loc_removed: u64,
71    #[serde(skip)]
72    dirty: bool,
73}
74
75impl EditMeteringStore {
76    fn load_from_disk() -> Self {
77        let Ok(raw) = std::fs::read_to_string(store_path()) else {
78            return Self::default();
79        };
80        serde_json::from_str(&raw).unwrap_or_default()
81    }
82
83    pub(crate) fn save(&self) -> std::io::Result<()> {
84        let path = store_path();
85        if let Some(parent) = path.parent() {
86            std::fs::create_dir_all(parent)?;
87        }
88        let json = serde_json::to_string(self)?;
89        let tmp = path.with_extension("tmp");
90        std::fs::write(&tmp, json)?;
91        std::fs::rename(&tmp, &path)
92    }
93}
94
95fn store_path() -> std::path::PathBuf {
96    crate::core::data_dir::lean_ctx_data_dir()
97        .unwrap_or_else(|_| std::path::PathBuf::from("."))
98        .join(STORE_FILE)
99}
100
101fn global() -> &'static Mutex<EditMeteringStore> {
102    STORE.get_or_init(|| Mutex::new(EditMeteringStore::load_from_disk()))
103}
104
105fn with_store(f: impl FnOnce(&mut EditMeteringStore)) {
106    let Ok(mut store) = global().lock() else {
107        return;
108    };
109    f(&mut store);
110    store.dirty = true;
111    let n = RECORD_CALLS.fetch_add(1, Ordering::Relaxed) + 1;
112    if n.is_multiple_of(FLUSH_EVERY) && store.save().is_ok() {
113        store.dirty = false;
114    }
115}
116
117/// A successful anchored patch: `ops` applied, `avoided_tokens` output tokens
118/// the model did not have to reproduce (already anchor-overhead-adjusted).
119pub fn record_anchored_success(ops: u64, avoided_tokens: u64) {
120    with_store(|s| {
121        s.anchored_calls = s.anchored_calls.saturating_add(1);
122        s.anchored_ops = s.anchored_ops.saturating_add(ops);
123        s.anchored_avoided_output_tokens = s
124            .anchored_avoided_output_tokens
125            .saturating_add(avoided_tokens);
126    });
127}
128
129/// A stale-anchor `CONFLICT` response (one extra self-heal round-trip).
130pub fn record_anchored_conflict() {
131    with_store(|s| s.anchored_conflicts = s.anchored_conflicts.saturating_add(1));
132}
133
134/// A successful str_replace edit and the `old_string` tokens it paid.
135pub fn record_str_replace_success(old_string_tokens: u64) {
136    with_store(|s| {
137        s.str_replace_calls = s.str_replace_calls.saturating_add(1);
138        s.str_replace_old_string_tokens = s
139            .str_replace_old_string_tokens
140            .saturating_add(old_string_tokens);
141    });
142}
143
144/// An `old_string`-not-found miss (one blind retry round-trip).
145pub fn record_str_replace_miss() {
146    with_store(|s| s.str_replace_misses = s.str_replace_misses.saturating_add(1));
147}
148
149#[allow(dead_code)]
150pub fn record_loc_change(path: Option<&str>, added: u64, removed: u64) {
151    with_store(|s| {
152        s.loc_added = s.loc_added.saturating_add(added);
153        s.loc_removed = s.loc_removed.saturating_add(removed);
154    });
155    crate::core::solution_tracker::record_loc_change(added, removed);
156    if let Some(path) = path {
157        crate::core::savings_ledger::record_edit_event(path, added, removed);
158    }
159}
160
161/// Snapshot for `ctx_metrics` and the dashboard `/api/stats` payload.
162pub fn metrics_snapshot() -> serde_json::Value {
163    let Ok(store) = global().lock() else {
164        return serde_json::json!({});
165    };
166    serde_json::json!({
167        "anchored_calls": store.anchored_calls,
168        "anchored_ops": store.anchored_ops,
169        "anchored_avoided_output_tokens": store.anchored_avoided_output_tokens,
170        "anchored_conflicts": store.anchored_conflicts,
171        "str_replace_calls": store.str_replace_calls,
172        "str_replace_old_string_tokens": store.str_replace_old_string_tokens,
173        "str_replace_misses": store.str_replace_misses,
174        "loc_added": store.loc_added,
175        "loc_removed": store.loc_removed,
176    })
177}
178
179pub(crate) fn flush() {
180    if let Ok(store) = global().lock()
181        && store.dirty
182    {
183        let _ = store.save();
184    }
185}
186
187#[cfg(test)]
188mod tests {
189    use super::*;
190
191    #[test]
192    fn counters_accumulate_and_saturate() {
193        let mut s = EditMeteringStore::default();
194        s.anchored_calls = u64::MAX;
195        s.anchored_calls = s.anchored_calls.saturating_add(1);
196        assert_eq!(s.anchored_calls, u64::MAX, "saturating, never wraps");
197    }
198
199    #[test]
200    fn roundtrip_serialization() {
201        let s = EditMeteringStore {
202            anchored_calls: 3,
203            anchored_ops: 7,
204            anchored_avoided_output_tokens: 412,
205            anchored_conflicts: 1,
206            str_replace_calls: 2,
207            str_replace_old_string_tokens: 260,
208            str_replace_misses: 4,
209            loc_added: 15,
210            loc_removed: 9,
211            dirty: false,
212        };
213        let json = serde_json::to_string(&s).unwrap();
214        let back: EditMeteringStore = serde_json::from_str(&json).unwrap();
215        assert_eq!(back.anchored_ops, 7);
216        assert_eq!(back.anchored_avoided_output_tokens, 412);
217        assert_eq!(back.str_replace_misses, 4);
218        assert_eq!(back.loc_added, 15);
219        assert_eq!(back.loc_removed, 9);
220    }
221
222    #[test]
223    fn legacy_or_empty_store_deserializes() {
224        let s: EditMeteringStore = serde_json::from_str("{}").unwrap();
225        assert_eq!(s.anchored_calls, 0);
226        assert_eq!(s.str_replace_old_string_tokens, 0);
227        assert_eq!(s.loc_added, 0);
228        assert_eq!(s.loc_removed, 0);
229    }
230}
lean_ctx/core/edit_metering.rs

lean_ctx/core/
edit_metering.rs
