zagens_topic_memory/
metrics.rs

1//! Lightweight evaluation metrics persisted beside the graph (B2.5).
2
3use std::fs;
4use std::path::{Path, PathBuf};
5
6use serde::{Deserialize, Serialize};
7
8/// Sidecar metrics file: `metrics.json` in the same directory as the graph file.
9#[derive(Debug, Clone, Default, Serialize, Deserialize)]
10pub struct TopicMemoryMetrics {
11    pub turn_updates: u64,
12    pub inject_count: u64,
13    #[serde(skip_serializing_if = "Option::is_none")]
14    pub last_inject_at: Option<String>,
15    pub clarification_rounds: u64,
16    pub repeat_topic_turns: u64,
17    #[serde(skip_serializing_if = "Option::is_none")]
18    pub last_user_topics: Option<Vec<String>>,
19}
20
21#[must_use]
22pub fn metrics_path_for_graph(graph_path: &Path) -> PathBuf {
23    graph_path
24        .parent()
25        .map(|p| p.join("metrics.json"))
26        .unwrap_or_else(|| PathBuf::from("metrics.json"))
27}
28
29#[must_use]
30pub fn load_metrics(path: &Path) -> TopicMemoryMetrics {
31    fs::read_to_string(path)
32        .ok()
33        .and_then(|s| serde_json::from_str(&s).ok())
34        .unwrap_or_default()
35}
36
37/// Persist metrics atomically (best-effort).
38pub fn save_metrics(path: &Path, metrics: &TopicMemoryMetrics) -> std::io::Result<()> {
39    if let Some(parent) = path.parent() {
40        fs::create_dir_all(parent)?;
41    }
42    let json = serde_json::to_string_pretty(metrics)?;
43    let tmp = path.with_extension("json.tmp");
44    fs::write(&tmp, json)?;
45    fs::rename(tmp, path)?;
46    Ok(())
47}
48
49/// Record a turn update; bumps clarification counters when topics repeat across turns.
50pub fn record_turn_update(metrics: &mut TopicMemoryMetrics, user_topics: &[String]) {
51    metrics.turn_updates = metrics.turn_updates.saturating_add(1);
52
53    if let Some(prev) = metrics.last_user_topics.as_ref() {
54        let overlap = user_topics.iter().any(|t| {
55            prev.iter()
56                .any(|p| p == t || p.contains(t) || t.contains(p))
57        });
58        if overlap && !user_topics.is_empty() {
59            metrics.repeat_topic_turns = metrics.repeat_topic_turns.saturating_add(1);
60            metrics.clarification_rounds = metrics.clarification_rounds.saturating_add(1);
61        }
62    }
63
64    if !user_topics.is_empty() {
65        metrics.last_user_topics = Some(user_topics.to_vec());
66    }
67}
68
69pub fn record_inject(metrics: &mut TopicMemoryMetrics, today: &str) {
70    metrics.inject_count = metrics.inject_count.saturating_add(1);
71    metrics.last_inject_at = Some(today.to_string());
72}
73
74/// B2.5 — derived rates for evaluation scripts and HTTP `/v1/topic-memory`.
75#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
76pub struct TopicMemoryEvalReport {
77    pub turn_updates: u64,
78    pub inject_count: u64,
79    pub clarification_rounds: u64,
80    pub repeat_topic_turns: u64,
81    /// `clarification_rounds / turn_updates` (0 when no turns).
82    pub clarification_rate: f64,
83    /// `repeat_topic_turns / turn_updates`.
84    pub repeat_topic_rate: f64,
85    /// Proxy for long-session usefulness: injects per 10 turns.
86    pub injects_per_10_turns: f64,
87    #[serde(skip_serializing_if = "Option::is_none")]
88    pub last_inject_at: Option<String>,
89}
90
91#[must_use]
92pub fn eval_report(metrics: &TopicMemoryMetrics) -> TopicMemoryEvalReport {
93    let turns = metrics.turn_updates.max(1) as f64;
94    TopicMemoryEvalReport {
95        turn_updates: metrics.turn_updates,
96        inject_count: metrics.inject_count,
97        clarification_rounds: metrics.clarification_rounds,
98        repeat_topic_turns: metrics.repeat_topic_turns,
99        clarification_rate: metrics.clarification_rounds as f64 / turns,
100        repeat_topic_rate: metrics.repeat_topic_turns as f64 / turns,
101        injects_per_10_turns: metrics.inject_count as f64 / turns * 10.0,
102        last_inject_at: metrics.last_inject_at.clone(),
103    }
104}
105
106/// Compare current metrics to a baseline file; used by `scripts/topic-memory-eval.ps1`.
107#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
108pub struct TopicMemoryEvalComparison {
109    pub current: TopicMemoryEvalReport,
110    pub baseline: TopicMemoryEvalReport,
111    pub clarification_rate_delta: f64,
112    pub repeat_topic_rate_delta: f64,
113    /// True when clarification rate worsened by more than `max_regression` (absolute).
114    pub regression: bool,
115}
116
117#[must_use]
118pub fn compare_eval(
119    current: &TopicMemoryMetrics,
120    baseline: &TopicMemoryMetrics,
121    max_regression: f64,
122) -> TopicMemoryEvalComparison {
123    let current = eval_report(current);
124    let baseline = eval_report(baseline);
125    let clarification_rate_delta = current.clarification_rate - baseline.clarification_rate;
126    let repeat_topic_rate_delta = current.repeat_topic_rate - baseline.repeat_topic_rate;
127    let regression = clarification_rate_delta > max_regression;
128    TopicMemoryEvalComparison {
129        current,
130        baseline,
131        clarification_rate_delta,
132        repeat_topic_rate_delta,
133        regression,
134    }
135}
136
137#[cfg(test)]
138mod tests {
139    use super::*;
140
141    #[test]
142    fn eval_report_rates() {
143        let m = TopicMemoryMetrics {
144            turn_updates: 10,
145            inject_count: 2,
146            clarification_rounds: 3,
147            repeat_topic_turns: 3,
148            ..Default::default()
149        };
150        let r = eval_report(&m);
151        assert!((r.clarification_rate - 0.3).abs() < f64::EPSILON);
152        assert!((r.injects_per_10_turns - 2.0).abs() < f64::EPSILON);
153    }
154
155    #[test]
156    fn compare_detects_regression() {
157        let baseline = TopicMemoryMetrics {
158            turn_updates: 20,
159            clarification_rounds: 2,
160            ..Default::default()
161        };
162        let current = TopicMemoryMetrics {
163            turn_updates: 20,
164            clarification_rounds: 8,
165            ..Default::default()
166        };
167        let cmp = compare_eval(&current, &baseline, 0.1);
168        assert!(cmp.regression);
169    }
170}
zagens_topic_memory/metrics.rs

zagens_topic_memory/
metrics.rs
