1use std::fmt::{self, Display};
2
3use serde::Serialize;
4
5pub use crate::eval::governance::RateMetric as InjectionRateMetric;
6
7pub(super) const CORPUS_NAME: &str = "builtin-context-injection-v2";
8
9#[derive(Debug, Clone, Copy, Default)]
10pub struct InjectionEvalOptions {
11 pub keep_data_dir: bool,
12}
13
14#[derive(Debug, Serialize)]
15pub struct InjectionEvalReport {
16 pub metadata: InjectionEvalMetadata,
17 pub metrics: InjectionMetricSummary,
18 pub churn: InjectionChurnReport,
19 pub rank_signal_ab: InjectionRankSignalAbReport,
20 pub cases: Vec<InjectionCaseReport>,
21 pub failing_examples: Vec<String>,
22}
23
24#[derive(Debug, Serialize)]
25pub struct InjectionEvalMetadata {
26 pub corpus: String,
27 pub boundary: String,
28 pub storage: String,
29 pub data_dir: String,
30 pub data_dir_kept: bool,
31 pub real_db_touched: bool,
32 pub project: String,
33 pub host: String,
34 pub branch: String,
35 pub render_contract_version: u32,
36 pub output_chars: usize,
37 pub memories_loaded: usize,
38 pub core_count: usize,
39 pub index_count: usize,
40 pub lesson_count: usize,
41 pub preference_count: usize,
42 pub session_count: usize,
43 pub workstream_count: usize,
44 pub truncated: bool,
45}
46
47#[derive(Debug, Serialize)]
48pub struct InjectionMetricSummary {
49 pub expected_memory_recall: InjectionRateMetric,
50 pub forbidden_memory_exclusion: InjectionRateMetric,
51 pub abstention_false_positive_bound: InjectionRateMetric,
52 pub stale_anchor_labeling: InjectionRateMetric,
53 pub user_prompt_submit_memory_recall: InjectionRateMetric,
54 pub user_prompt_submit_abstention_false_positive_bound: InjectionRateMetric,
55 pub block_churn_unchanged: InjectionRateMetric,
56 pub block_churn_one_added_prefix_preserved: InjectionRateMetric,
57 pub all_checks_passed: bool,
58}
59
60#[derive(Debug, Serialize)]
61pub struct InjectionChurnReport {
62 pub unchanged_changed_bytes: usize,
63 pub one_added_changed_bytes: usize,
64 pub one_added_first_affected_section: Option<String>,
65 pub one_added_prefix_preserved: bool,
66}
67
68#[derive(Debug, Serialize)]
69pub struct InjectionRankSignalAbReport {
70 pub query: String,
71 pub expected_memory_id: i64,
72 pub baseline: InjectionRankSignalArm,
73 pub candidate: InjectionRankSignalArm,
74 pub passed: bool,
75}
76
77#[derive(Debug, Serialize)]
78pub struct InjectionRankSignalArm {
79 pub algorithm: String,
80 pub retrieved_ids: Vec<i64>,
81 pub mrr_at_10: f64,
82 pub ndcg_at_10: f64,
83}
84
85#[derive(Debug, Serialize)]
86pub struct InjectionCaseReport {
87 pub id: String,
88 pub expectation: String,
89 pub title: String,
90 pub topic_key: String,
91 pub matched: bool,
92}
93
94impl Display for InjectionEvalReport {
95 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
96 writeln!(f, "=== remem eval-injection ({}) ===", self.metadata.corpus)?;
97 writeln!(f, "boundary: {}", self.metadata.boundary)?;
98 writeln!(
99 f,
100 "storage: {}; real_db_touched={}",
101 self.metadata.storage, self.metadata.real_db_touched
102 )?;
103 writeln!(
104 f,
105 "expected_memory_recall: {}/{} ({:.1}%)",
106 self.metrics.expected_memory_recall.passed,
107 self.metrics.expected_memory_recall.total,
108 self.metrics.expected_memory_recall.rate * 100.0
109 )?;
110 writeln!(
111 f,
112 "forbidden_memory_exclusion: {}/{} ({:.1}%)",
113 self.metrics.forbidden_memory_exclusion.passed,
114 self.metrics.forbidden_memory_exclusion.total,
115 self.metrics.forbidden_memory_exclusion.rate * 100.0
116 )?;
117 writeln!(
118 f,
119 "abstention_false_positive_bound: {}/{} ({:.1}%)",
120 self.metrics.abstention_false_positive_bound.passed,
121 self.metrics.abstention_false_positive_bound.total,
122 self.metrics.abstention_false_positive_bound.rate * 100.0
123 )?;
124 writeln!(
125 f,
126 "stale_anchor_labeling: {}/{} ({:.1}%)",
127 self.metrics.stale_anchor_labeling.passed,
128 self.metrics.stale_anchor_labeling.total,
129 self.metrics.stale_anchor_labeling.rate * 100.0
130 )?;
131 writeln!(
132 f,
133 "user_prompt_submit_memory_recall: {}/{} ({:.1}%)",
134 self.metrics.user_prompt_submit_memory_recall.passed,
135 self.metrics.user_prompt_submit_memory_recall.total,
136 self.metrics.user_prompt_submit_memory_recall.rate * 100.0
137 )?;
138 writeln!(
139 f,
140 "user_prompt_submit_abstention_false_positive_bound: {}/{} ({:.1}%)",
141 self.metrics
142 .user_prompt_submit_abstention_false_positive_bound
143 .passed,
144 self.metrics
145 .user_prompt_submit_abstention_false_positive_bound
146 .total,
147 self.metrics
148 .user_prompt_submit_abstention_false_positive_bound
149 .rate
150 * 100.0
151 )?;
152 writeln!(
153 f,
154 "block_churn_unchanged: {}/{} ({:.1}%)",
155 self.metrics.block_churn_unchanged.passed,
156 self.metrics.block_churn_unchanged.total,
157 self.metrics.block_churn_unchanged.rate * 100.0
158 )?;
159 writeln!(
160 f,
161 "block_churn_one_added_prefix_preserved: {}/{} ({:.1}%)",
162 self.metrics.block_churn_one_added_prefix_preserved.passed,
163 self.metrics.block_churn_one_added_prefix_preserved.total,
164 self.metrics.block_churn_one_added_prefix_preserved.rate * 100.0
165 )?;
166 writeln!(
167 f,
168 "rendered: render_contract_version={} memories_loaded={} core={} index={} chars={} truncated={}",
169 self.metadata.render_contract_version,
170 self.metadata.memories_loaded,
171 self.metadata.core_count,
172 self.metadata.index_count,
173 self.metadata.output_chars,
174 self.metadata.truncated
175 )?;
176 writeln!(
177 f,
178 "churn: unchanged_changed_bytes={} one_added_changed_bytes={} one_added_prefix_preserved={}",
179 self.churn.unchanged_changed_bytes,
180 self.churn.one_added_changed_bytes,
181 self.churn.one_added_prefix_preserved
182 )?;
183 writeln!(
184 f,
185 "rank_signal_ab: baseline_mrr_at_10={:.6} candidate_mrr_at_10={:.6} baseline_ndcg_at_10={:.6} candidate_ndcg_at_10={:.6} passed={}",
186 self.rank_signal_ab.baseline.mrr_at_10,
187 self.rank_signal_ab.candidate.mrr_at_10,
188 self.rank_signal_ab.baseline.ndcg_at_10,
189 self.rank_signal_ab.candidate.ndcg_at_10,
190 self.rank_signal_ab.passed
191 )?;
192 writeln!(f, "all_checks_passed: {}", self.metrics.all_checks_passed)?;
193 if self.failing_examples.is_empty() {
194 return Ok(());
195 }
196 writeln!(f, "failures:")?;
197 for failure in &self.failing_examples {
198 writeln!(f, "- {failure}")?;
199 }
200 Ok(())
201 }
202}