lean_ctx/core/
llm_feedback.rs

1use std::collections::{BTreeMap, VecDeque};
2use std::fs::{File, OpenOptions};
3use std::io::{BufRead, BufReader, Write};
4use std::path::{Path, PathBuf};
5
6use serde::{Deserialize, Serialize};
7
8const LLM_FEEDBACK_FILE: &str = "llm_feedback.jsonl";
9const LLM_FEEDBACK_MAX_EVENTS: usize = 5_000;
10const LLM_FEEDBACK_MAX_BYTES: u64 = 8 * 1024 * 1024;
11
12#[derive(Debug, Clone, Serialize, Deserialize)]
13pub struct LlmFeedbackEvent {
14    pub agent_id: String,
15    pub intent: Option<String>,
16    pub model: Option<String>,
17    pub llm_input_tokens: u64,
18    pub llm_output_tokens: u64,
19    pub latency_ms: Option<u64>,
20    pub note: Option<String>,
21    pub ctx_read_last_mode: Option<String>,
22    pub ctx_read_modes: Option<BTreeMap<String, u64>>,
23    pub timestamp: String,
24}
25
26#[derive(Debug, Clone, Serialize, Deserialize, Default)]
27pub struct LlmFeedbackSummary {
28    pub total_events: usize,
29    pub avg_output_ratio: f64,
30    pub avg_latency_ms: Option<f64>,
31    pub max_output_tokens: u64,
32    pub max_output_ratio: f64,
33    pub by_model: BTreeMap<String, ModelSummary>,
34}
35
36#[derive(Debug, Clone, Serialize, Deserialize, Default)]
37pub struct ModelSummary {
38    pub events: usize,
39    pub avg_output_ratio: f64,
40    pub avg_latency_ms: Option<f64>,
41    pub max_output_tokens: u64,
42}
43
44pub struct LlmFeedbackStore;
45
46impl LlmFeedbackStore {
47    pub fn record(mut event: LlmFeedbackEvent) -> Result<(), String> {
48        if event.agent_id.trim().is_empty() {
49            return Err("agent_id is required".to_string());
50        }
51        if event.llm_input_tokens == 0 {
52            return Err("llm_input_tokens must be > 0".to_string());
53        }
54        if event.llm_output_tokens == 0 {
55            return Err("llm_output_tokens must be > 0".to_string());
56        }
57        if let Some(n) = event.note.as_ref() {
58            if n.len() > 2000 {
59                event.note = Some(n.chars().take(2000).collect());
60            }
61        }
62
63        let path = feedback_path();
64        ensure_parent_dir(&path)?;
65
66        let mut f = OpenOptions::new()
67            .create(true)
68            .append(true)
69            .open(&path)
70            .map_err(|e| format!("open {}: {e}", path.display()))?;
71
72        let line = serde_json::to_string(&event).map_err(|e| format!("serialize: {e}"))?;
73        f.write_all(line.as_bytes())
74            .and_then(|_| f.write_all(b"\n"))
75            .map_err(|e| format!("write {}: {e}", path.display()))?;
76
77        maybe_compact(&path)?;
78        Ok(())
79    }
80
81    pub fn status() -> LlmFeedbackStatus {
82        let path = feedback_path();
83        let bytes = std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
84        LlmFeedbackStatus {
85            path,
86            bytes,
87            max_events: LLM_FEEDBACK_MAX_EVENTS,
88            max_bytes: LLM_FEEDBACK_MAX_BYTES,
89        }
90    }
91
92    pub fn reset() -> Result<(), String> {
93        let path = feedback_path();
94        if path.exists() {
95            std::fs::remove_file(&path).map_err(|e| format!("remove {}: {e}", path.display()))?;
96        }
97        Ok(())
98    }
99
100    pub fn recent(limit: usize) -> Vec<LlmFeedbackEvent> {
101        let path = feedback_path();
102        let mut out: VecDeque<LlmFeedbackEvent> = VecDeque::with_capacity(limit.max(1));
103        let Ok(f) = File::open(&path) else {
104            return Vec::new();
105        };
106        let reader = BufReader::new(f);
107        for line in reader.lines().map_while(Result::ok) {
108            if line.trim().is_empty() {
109                continue;
110            }
111            if let Ok(ev) = serde_json::from_str::<LlmFeedbackEvent>(&line) {
112                out.push_back(ev);
113                while out.len() > limit {
114                    out.pop_front();
115                }
116            }
117        }
118        out.into_iter().collect()
119    }
120
121    pub fn summarize(limit: usize) -> LlmFeedbackSummary {
122        let events = Self::recent(limit);
123        summarize_events(&events)
124    }
125}
126
127#[derive(Debug, Clone, Serialize, Deserialize)]
128pub struct LlmFeedbackStatus {
129    pub path: PathBuf,
130    pub bytes: u64,
131    pub max_events: usize,
132    pub max_bytes: u64,
133}
134
135fn summarize_events(events: &[LlmFeedbackEvent]) -> LlmFeedbackSummary {
136    if events.is_empty() {
137        return LlmFeedbackSummary::default();
138    }
139
140    let mut by_model: BTreeMap<String, Vec<&LlmFeedbackEvent>> = BTreeMap::new();
141    let mut ratio_sum = 0.0;
142    let mut ratio_max: f64 = 0.0;
143    let mut max_out = 0u64;
144    let mut latency_sum = 0u64;
145    let mut latency_n = 0u64;
146
147    for ev in events {
148        let ratio = ev.llm_output_tokens as f64 / ev.llm_input_tokens.max(1) as f64;
149        ratio_sum += ratio;
150        ratio_max = ratio_max.max(ratio);
151        max_out = max_out.max(ev.llm_output_tokens);
152        if let Some(ms) = ev.latency_ms {
153            latency_sum = latency_sum.saturating_add(ms);
154            latency_n += 1;
155        }
156        by_model
157            .entry(ev.model.clone().unwrap_or_else(|| "unknown".to_string()))
158            .or_default()
159            .push(ev);
160    }
161
162    let avg_latency_ms = if latency_n > 0 {
163        Some(latency_sum as f64 / latency_n as f64)
164    } else {
165        None
166    };
167
168    let mut model_summaries = BTreeMap::new();
169    for (model, evs) in by_model {
170        let mut r_sum = 0.0;
171        let mut max_out = 0u64;
172        let mut l_sum = 0u64;
173        let mut l_n = 0u64;
174        let n = evs.len();
175        for ev in &evs {
176            r_sum += ev.llm_output_tokens as f64 / ev.llm_input_tokens.max(1) as f64;
177            max_out = max_out.max(ev.llm_output_tokens);
178            if let Some(ms) = ev.latency_ms {
179                l_sum = l_sum.saturating_add(ms);
180                l_n += 1;
181            }
182        }
183        model_summaries.insert(
184            model,
185            ModelSummary {
186                events: n,
187                avg_output_ratio: r_sum / n.max(1) as f64,
188                avg_latency_ms: if l_n > 0 {
189                    Some(l_sum as f64 / l_n as f64)
190                } else {
191                    None
192                },
193                max_output_tokens: max_out,
194            },
195        );
196    }
197
198    LlmFeedbackSummary {
199        total_events: events.len(),
200        avg_output_ratio: ratio_sum / events.len().max(1) as f64,
201        avg_latency_ms,
202        max_output_tokens: max_out,
203        max_output_ratio: ratio_max,
204        by_model: model_summaries,
205    }
206}
207
208fn feedback_path() -> PathBuf {
209    crate::core::data_dir::lean_ctx_data_dir()
210        .unwrap_or_else(|_| PathBuf::from("."))
211        .join("feedback")
212        .join(LLM_FEEDBACK_FILE)
213}
214
215fn ensure_parent_dir(path: &Path) -> Result<(), String> {
216    let Some(parent) = path.parent() else {
217        return Ok(());
218    };
219    std::fs::create_dir_all(parent).map_err(|e| format!("create_dir_all {}: {e}", parent.display()))
220}
221
222fn maybe_compact(path: &Path) -> Result<(), String> {
223    let Ok(meta) = std::fs::metadata(path) else {
224        return Ok(());
225    };
226    if meta.len() <= LLM_FEEDBACK_MAX_BYTES {
227        return Ok(());
228    }
229
230    let f = File::open(path).map_err(|e| format!("open {}: {e}", path.display()))?;
231    let reader = BufReader::new(f);
232
233    let mut keep: VecDeque<String> = VecDeque::with_capacity(LLM_FEEDBACK_MAX_EVENTS);
234    for line in reader.lines().map_while(Result::ok) {
235        if line.trim().is_empty() {
236            continue;
237        }
238        keep.push_back(line);
239        while keep.len() > LLM_FEEDBACK_MAX_EVENTS {
240            keep.pop_front();
241        }
242    }
243
244    let dir = path.parent().unwrap_or_else(|| Path::new("."));
245    let tmp = dir.join(".llm_feedback.compact.tmp");
246    {
247        let mut out = File::create(&tmp).map_err(|e| format!("create {}: {e}", tmp.display()))?;
248        for line in keep {
249            out.write_all(line.as_bytes())
250                .and_then(|_| out.write_all(b"\n"))
251                .map_err(|e| format!("write {}: {e}", tmp.display()))?;
252        }
253        out.flush()
254            .map_err(|e| format!("flush {}: {e}", tmp.display()))?;
255    }
256
257    std::fs::rename(&tmp, path)
258        .map_err(|e| format!("rename {} -> {}: {e}", tmp.display(), path.display()))?;
259    Ok(())
260}
261
262#[cfg(test)]
263mod tests {
264    use super::*;
265
266    #[test]
267    fn summarize_empty_is_default() {
268        let s = summarize_events(&[]);
269        assert_eq!(s.total_events, 0);
270        assert!(s.by_model.is_empty());
271    }
272}
lean_ctx/core/llm_feedback.rs

lean_ctx/core/
llm_feedback.rs
