remem/eval/golden/
types.rs

1use std::collections::BTreeMap;
2
3use crate::memory::Memory;
4
5#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
6pub struct GoldenDataset {
7    pub version: Option<String>,
8    pub description: Option<String>,
9    #[serde(default)]
10    pub corpus: Vec<GoldenMemory>,
11    #[serde(default)]
12    pub queries: Vec<GoldenQuery>,
13}
14
15impl GoldenDataset {
16    pub fn has_fixture_corpus(&self) -> bool {
17        !self.corpus.is_empty()
18    }
19}
20
21#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
22pub struct GoldenMemory {
23    pub project: String,
24    #[serde(default)]
25    pub topic_key: Option<String>,
26    pub title: String,
27    #[serde(alias = "text")]
28    pub content: String,
29    pub memory_type: String,
30    #[serde(default)]
31    pub branch: Option<String>,
32    #[serde(default = "default_scope")]
33    pub scope: String,
34    #[serde(default = "default_status")]
35    pub status: String,
36    #[serde(default)]
37    pub files: Option<String>,
38    #[serde(default)]
39    pub created_at_epoch: Option<i64>,
40    #[serde(default)]
41    pub access_count: Option<i64>,
42    #[serde(default)]
43    pub last_accessed_epoch: Option<i64>,
44    /// Eval-fixture-only enrichment installed through the production
45    /// security/composition path. Proves FTS/vector channel wiring only; it
46    /// never marks the row ready and never enters the paraphrase quality gate.
47    #[serde(default, skip_serializing_if = "Option::is_none")]
48    pub search_context: Option<GoldenSearchContext>,
49}
50
51#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
52pub struct GoldenSearchContext {
53    pub context: String,
54    #[serde(default)]
55    pub keywords: Vec<String>,
56}
57
58fn default_scope() -> String {
59    "project".to_string()
60}
61
62fn default_status() -> String {
63    "active".to_string()
64}
65
66#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
67pub struct GoldenQuery {
68    pub id: String,
69    pub query: String,
70    pub category: String,
71    #[serde(default)]
72    pub slice: Option<String>,
73    #[serde(default)]
74    pub hop_path: Option<GoldenHopPath>,
75    pub project: Option<String>,
76    #[serde(default)]
77    pub branch: Option<String>,
78    #[serde(default)]
79    pub memory_type: Option<String>,
80    #[serde(default)]
81    pub relevant_ids: Vec<i64>,
82    #[serde(default, alias = "expected_refs")]
83    pub evidence_refs: Vec<EvidenceRef>,
84    #[serde(default)]
85    pub expect_abstain: bool,
86    #[serde(default)]
87    pub false_premise: bool,
88    pub notes: Option<String>,
89}
90
91impl GoldenQuery {
92    pub fn expects_abstention(&self) -> bool {
93        self.expect_abstain || self.false_premise
94    }
95
96    pub fn slice_label(&self) -> &str {
97        self.slice
98            .as_deref()
99            .filter(|slice| !slice.trim().is_empty())
100            .unwrap_or(&self.category)
101    }
102
103    pub fn expected_refs(&self) -> Vec<EvidenceRef> {
104        let mut refs = self.evidence_refs.clone();
105        refs.extend(self.relevant_ids.iter().map(|id| EvidenceRef {
106            memory_id: Some(*id),
107            ..EvidenceRef::default()
108        }));
109        refs
110    }
111}
112
113#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
114pub struct GoldenHopPath {
115    pub source: String,
116    pub entity_type: String,
117    pub entity: String,
118    pub target: String,
119}
120
121#[derive(Debug, Clone, Default, serde::Deserialize, serde::Serialize)]
122pub struct EvidenceRef {
123    pub memory_id: Option<i64>,
124    pub topic_key: Option<String>,
125    pub project: Option<String>,
126    pub branch: Option<String>,
127    pub memory_type: Option<String>,
128    pub scope: Option<String>,
129    pub title_contains: Option<String>,
130    pub text_contains: Option<String>,
131}
132
133impl EvidenceRef {
134    pub fn has_match_criteria(&self) -> bool {
135        self.memory_id.is_some()
136            || self.topic_key.is_some()
137            || self.project.is_some()
138            || self.branch.is_some()
139            || self.memory_type.is_some()
140            || self.scope.is_some()
141            || self.title_contains.is_some()
142            || self.text_contains.is_some()
143    }
144
145    pub fn matches(&self, memory: &Memory) -> bool {
146        if !self.has_match_criteria() {
147            return false;
148        }
149        if let Some(memory_id) = self.memory_id {
150            if memory.id != memory_id {
151                return false;
152            }
153        }
154        if let Some(project) = self.project.as_deref() {
155            if !crate::project_id::project_matches(Some(&memory.project), project) {
156                return false;
157            }
158        }
159        if let Some(branch) = self.branch.as_deref() {
160            if memory.branch.as_deref() != Some(branch) {
161                return false;
162            }
163        }
164        if let Some(topic_key) = self.topic_key.as_deref() {
165            if memory.topic_key.as_deref() != Some(topic_key) {
166                return false;
167            }
168        }
169        if let Some(memory_type) = self.memory_type.as_deref() {
170            if memory.memory_type != memory_type {
171                return false;
172            }
173        }
174        if let Some(scope) = self.scope.as_deref() {
175            if memory.scope != scope {
176                return false;
177            }
178        }
179        if let Some(needle) = self.title_contains.as_deref() {
180            if !contains_case_insensitive(&memory.title, needle) {
181                return false;
182            }
183        }
184        if let Some(needle) = self.text_contains.as_deref() {
185            if !contains_case_insensitive(&memory.text, needle) {
186                return false;
187            }
188        }
189        true
190    }
191}
192
193pub(super) fn contains_case_insensitive(haystack: &str, needle: &str) -> bool {
194    haystack.to_lowercase().contains(&needle.to_lowercase())
195}
196
197#[derive(Debug, Clone, serde::Serialize)]
198pub struct GoldenEvalReport {
199    pub evaluation_layers: EvaluationLayers,
200    pub version: Option<String>,
201    pub description: Option<String>,
202    pub k: usize,
203    pub rank_k: usize,
204    pub total_queries: usize,
205    pub scored_queries: usize,
206    pub skipped_queries: usize,
207    pub abstention_queries: usize,
208    pub abstention_passed: usize,
209    pub overall: Option<MetricAverages>,
210    pub by_slice: BTreeMap<String, CategoryEvaluation>,
211    pub by_category: BTreeMap<String, CategoryEvaluation>,
212    pub queries: Vec<QueryEvaluation>,
213}
214
215#[derive(Debug, Clone, serde::Serialize)]
216pub struct CategoryEvaluation {
217    pub total_queries: usize,
218    pub scored_queries: usize,
219    pub abstention_queries: usize,
220    pub abstention_passed: usize,
221    pub query_tokens_per_query: f64,
222    pub retrieval_latency_p50_ms: f64,
223    pub retrieval_latency_p95_ms: f64,
224    pub metrics: Option<MetricAverages>,
225}
226
227#[derive(Debug, Clone, serde::Serialize)]
228pub struct QueryEvaluation {
229    pub id: String,
230    pub query: String,
231    pub category: String,
232    pub slice: String,
233    pub status: QueryStatus,
234    pub result_count: usize,
235    pub retrieved_ids: Vec<i64>,
236    pub expected_relevant_ids: Vec<i64>,
237    pub missing_relevant_ids: Vec<i64>,
238    pub missing_evidence_refs: Vec<EvidenceRef>,
239    pub matched_refs: usize,
240    pub expected_refs: usize,
241    pub query_tokens: usize,
242    pub retrieval_latency_ms: f64,
243    pub metrics: Option<QueryMetrics>,
244}
245
246#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
247pub enum QueryStatus {
248    #[serde(rename = "HIT")]
249    Hit,
250    #[serde(rename = "MISS")]
251    Miss,
252    #[serde(rename = "PASS")]
253    Pass,
254    #[serde(rename = "FAIL")]
255    Fail,
256    #[serde(rename = "SKIP")]
257    Skip,
258}
259
260impl QueryStatus {
261    pub fn label(self) -> &'static str {
262        match self {
263            QueryStatus::Hit => "HIT",
264            QueryStatus::Miss => "MISS",
265            QueryStatus::Pass => "PASS",
266            QueryStatus::Fail => "FAIL",
267            QueryStatus::Skip => "SKIP",
268        }
269    }
270}
271
272#[derive(Debug, Clone, serde::Serialize)]
273pub struct EvaluationLayers {
274    pub retrieval: LayerStatus,
275    pub answer_generation: LayerStatus,
276    pub llm_judge: LayerStatus,
277}
278
279impl EvaluationLayers {
280    pub fn deterministic_retrieval_only() -> Self {
281        Self {
282            retrieval: LayerStatus {
283                status: "deterministic",
284                description:
285                    "fixed golden retrieval metrics: Hit@K, Recall@K, MRR, nDCG, evidence recall",
286            },
287            answer_generation: LayerStatus {
288                status: "not_run",
289                description:
290                    "answer generation is intentionally excluded from golden retrieval eval",
291            },
292            llm_judge: LayerStatus {
293                status: "not_run",
294                description: "LLM judging is intentionally excluded from deterministic golden eval",
295            },
296        }
297    }
298}
299
300#[derive(Debug, Clone, serde::Serialize)]
301pub struct LayerStatus {
302    pub status: &'static str,
303    pub description: &'static str,
304}
305
306#[derive(Debug, Clone, Default, serde::Serialize)]
307pub struct QueryMetrics {
308    pub hit_at_k: f64,
309    pub mrr_at_10: f64,
310    pub precision_at_k: f64,
311    pub recall_at_k: f64,
312    pub ndcg_at_10: f64,
313    pub evidence_recall_at_k: f64,
314}
315
316#[derive(Debug, Clone, Default, serde::Serialize)]
317pub struct MetricAverages {
318    pub count: usize,
319    pub hit_at_k: f64,
320    pub mrr_at_10: f64,
321    pub precision_at_k: f64,
322    pub recall_at_k: f64,
323    pub ndcg_at_10: f64,
324    pub evidence_recall_at_k: f64,
325}
326
327#[derive(Debug, Default)]
328pub(super) struct MetricSums {
329    count: usize,
330    hit_at_k: f64,
331    mrr_at_10: f64,
332    precision_at_k: f64,
333    recall_at_k: f64,
334    ndcg_at_10: f64,
335    evidence_recall_at_k: f64,
336}
337
338impl MetricSums {
339    pub(super) fn add(&mut self, metrics: &QueryMetrics) {
340        self.count += 1;
341        self.hit_at_k += metrics.hit_at_k;
342        self.mrr_at_10 += metrics.mrr_at_10;
343        self.precision_at_k += metrics.precision_at_k;
344        self.recall_at_k += metrics.recall_at_k;
345        self.ndcg_at_10 += metrics.ndcg_at_10;
346        self.evidence_recall_at_k += metrics.evidence_recall_at_k;
347    }
348
349    pub(super) fn averages(&self) -> Option<MetricAverages> {
350        if self.count == 0 {
351            return None;
352        }
353        let n = self.count as f64;
354        Some(MetricAverages {
355            count: self.count,
356            hit_at_k: self.hit_at_k / n,
357            mrr_at_10: self.mrr_at_10 / n,
358            precision_at_k: self.precision_at_k / n,
359            recall_at_k: self.recall_at_k / n,
360            ndcg_at_10: self.ndcg_at_10 / n,
361            evidence_recall_at_k: self.evidence_recall_at_k / n,
362        })
363    }
364}
remem/eval/golden/types.rs

remem/eval/golden/
types.rs
