1use std::fmt::{self, Display};
2
3use serde::Serialize;
4
5pub use crate::eval::governance::RateMetric as InjectionRateMetric;
6
7pub(super) const CORPUS_NAME: &str = "builtin-context-injection-v2";
8
9#[derive(Debug, Clone, Copy, Default)]
10pub struct InjectionEvalOptions {
11 pub keep_data_dir: bool,
12}
13
14#[derive(Debug, Serialize)]
15pub struct InjectionEvalReport {
16 pub metadata: InjectionEvalMetadata,
17 pub metrics: InjectionMetricSummary,
18 pub churn: InjectionChurnReport,
19 pub cases: Vec<InjectionCaseReport>,
20 pub failing_examples: Vec<String>,
21}
22
23#[derive(Debug, Serialize)]
24pub struct InjectionEvalMetadata {
25 pub corpus: String,
26 pub boundary: String,
27 pub storage: String,
28 pub data_dir: String,
29 pub data_dir_kept: bool,
30 pub real_db_touched: bool,
31 pub project: String,
32 pub host: String,
33 pub branch: String,
34 pub render_contract_version: u32,
35 pub output_chars: usize,
36 pub memories_loaded: usize,
37 pub core_count: usize,
38 pub index_count: usize,
39 pub lesson_count: usize,
40 pub preference_count: usize,
41 pub session_count: usize,
42 pub workstream_count: usize,
43 pub truncated: bool,
44}
45
46#[derive(Debug, Serialize)]
47pub struct InjectionMetricSummary {
48 pub expected_memory_recall: InjectionRateMetric,
49 pub forbidden_memory_exclusion: InjectionRateMetric,
50 pub abstention_false_positive_bound: InjectionRateMetric,
51 pub stale_anchor_labeling: InjectionRateMetric,
52 pub user_prompt_submit_memory_recall: InjectionRateMetric,
53 pub user_prompt_submit_abstention_false_positive_bound: InjectionRateMetric,
54 pub block_churn_unchanged: InjectionRateMetric,
55 pub block_churn_one_added_prefix_preserved: InjectionRateMetric,
56 pub all_checks_passed: bool,
57}
58
59#[derive(Debug, Serialize)]
60pub struct InjectionChurnReport {
61 pub unchanged_changed_bytes: usize,
62 pub one_added_changed_bytes: usize,
63 pub one_added_first_affected_section: Option<String>,
64 pub one_added_prefix_preserved: bool,
65}
66
67#[derive(Debug, Serialize)]
68pub struct InjectionCaseReport {
69 pub id: String,
70 pub expectation: String,
71 pub title: String,
72 pub topic_key: String,
73 pub matched: bool,
74}
75
76impl Display for InjectionEvalReport {
77 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
78 writeln!(f, "=== remem eval-injection ({}) ===", self.metadata.corpus)?;
79 writeln!(f, "boundary: {}", self.metadata.boundary)?;
80 writeln!(
81 f,
82 "storage: {}; real_db_touched={}",
83 self.metadata.storage, self.metadata.real_db_touched
84 )?;
85 writeln!(
86 f,
87 "expected_memory_recall: {}/{} ({:.1}%)",
88 self.metrics.expected_memory_recall.passed,
89 self.metrics.expected_memory_recall.total,
90 self.metrics.expected_memory_recall.rate * 100.0
91 )?;
92 writeln!(
93 f,
94 "forbidden_memory_exclusion: {}/{} ({:.1}%)",
95 self.metrics.forbidden_memory_exclusion.passed,
96 self.metrics.forbidden_memory_exclusion.total,
97 self.metrics.forbidden_memory_exclusion.rate * 100.0
98 )?;
99 writeln!(
100 f,
101 "abstention_false_positive_bound: {}/{} ({:.1}%)",
102 self.metrics.abstention_false_positive_bound.passed,
103 self.metrics.abstention_false_positive_bound.total,
104 self.metrics.abstention_false_positive_bound.rate * 100.0
105 )?;
106 writeln!(
107 f,
108 "stale_anchor_labeling: {}/{} ({:.1}%)",
109 self.metrics.stale_anchor_labeling.passed,
110 self.metrics.stale_anchor_labeling.total,
111 self.metrics.stale_anchor_labeling.rate * 100.0
112 )?;
113 writeln!(
114 f,
115 "user_prompt_submit_memory_recall: {}/{} ({:.1}%)",
116 self.metrics.user_prompt_submit_memory_recall.passed,
117 self.metrics.user_prompt_submit_memory_recall.total,
118 self.metrics.user_prompt_submit_memory_recall.rate * 100.0
119 )?;
120 writeln!(
121 f,
122 "user_prompt_submit_abstention_false_positive_bound: {}/{} ({:.1}%)",
123 self.metrics
124 .user_prompt_submit_abstention_false_positive_bound
125 .passed,
126 self.metrics
127 .user_prompt_submit_abstention_false_positive_bound
128 .total,
129 self.metrics
130 .user_prompt_submit_abstention_false_positive_bound
131 .rate
132 * 100.0
133 )?;
134 writeln!(
135 f,
136 "block_churn_unchanged: {}/{} ({:.1}%)",
137 self.metrics.block_churn_unchanged.passed,
138 self.metrics.block_churn_unchanged.total,
139 self.metrics.block_churn_unchanged.rate * 100.0
140 )?;
141 writeln!(
142 f,
143 "block_churn_one_added_prefix_preserved: {}/{} ({:.1}%)",
144 self.metrics.block_churn_one_added_prefix_preserved.passed,
145 self.metrics.block_churn_one_added_prefix_preserved.total,
146 self.metrics.block_churn_one_added_prefix_preserved.rate * 100.0
147 )?;
148 writeln!(
149 f,
150 "rendered: render_contract_version={} memories_loaded={} core={} index={} chars={} truncated={}",
151 self.metadata.render_contract_version,
152 self.metadata.memories_loaded,
153 self.metadata.core_count,
154 self.metadata.index_count,
155 self.metadata.output_chars,
156 self.metadata.truncated
157 )?;
158 writeln!(
159 f,
160 "churn: unchanged_changed_bytes={} one_added_changed_bytes={} one_added_prefix_preserved={}",
161 self.churn.unchanged_changed_bytes,
162 self.churn.one_added_changed_bytes,
163 self.churn.one_added_prefix_preserved
164 )?;
165 writeln!(f, "all_checks_passed: {}", self.metrics.all_checks_passed)?;
166 if self.failing_examples.is_empty() {
167 return Ok(());
168 }
169 writeln!(f, "failures:")?;
170 for failure in &self.failing_examples {
171 writeln!(f, "- {failure}")?;
172 }
173 Ok(())
174 }
175}