1use std::collections::BTreeMap;
2
3use crate::memory::Memory;
4
5#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
6pub struct GoldenDataset {
7 pub version: Option<String>,
8 pub description: Option<String>,
9 #[serde(default)]
10 pub corpus: Vec<GoldenMemory>,
11 #[serde(default)]
12 pub queries: Vec<GoldenQuery>,
13}
14
15impl GoldenDataset {
16 pub fn has_fixture_corpus(&self) -> bool {
17 !self.corpus.is_empty()
18 }
19}
20
21#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
22pub struct GoldenMemory {
23 pub project: String,
24 #[serde(default)]
25 pub topic_key: Option<String>,
26 pub title: String,
27 #[serde(alias = "text")]
28 pub content: String,
29 pub memory_type: String,
30 #[serde(default)]
31 pub branch: Option<String>,
32 #[serde(default = "default_scope")]
33 pub scope: String,
34 #[serde(default = "default_status")]
35 pub status: String,
36 #[serde(default)]
37 pub files: Option<String>,
38 #[serde(default)]
39 pub created_at_epoch: Option<i64>,
40 #[serde(default)]
41 pub access_count: Option<i64>,
42 #[serde(default)]
43 pub last_accessed_epoch: Option<i64>,
44}
45
46fn default_scope() -> String {
47 "project".to_string()
48}
49
50fn default_status() -> String {
51 "active".to_string()
52}
53
54#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
55pub struct GoldenQuery {
56 pub id: String,
57 pub query: String,
58 pub category: String,
59 #[serde(default)]
60 pub slice: Option<String>,
61 #[serde(default)]
62 pub hop_path: Option<GoldenHopPath>,
63 pub project: Option<String>,
64 #[serde(default)]
65 pub branch: Option<String>,
66 #[serde(default)]
67 pub memory_type: Option<String>,
68 #[serde(default)]
69 pub relevant_ids: Vec<i64>,
70 #[serde(default, alias = "expected_refs")]
71 pub evidence_refs: Vec<EvidenceRef>,
72 #[serde(default)]
73 pub expect_abstain: bool,
74 #[serde(default)]
75 pub false_premise: bool,
76 pub notes: Option<String>,
77}
78
79impl GoldenQuery {
80 pub fn expects_abstention(&self) -> bool {
81 self.expect_abstain || self.false_premise
82 }
83
84 pub fn slice_label(&self) -> &str {
85 self.slice
86 .as_deref()
87 .filter(|slice| !slice.trim().is_empty())
88 .unwrap_or(&self.category)
89 }
90
91 pub fn expected_refs(&self) -> Vec<EvidenceRef> {
92 let mut refs = self.evidence_refs.clone();
93 refs.extend(self.relevant_ids.iter().map(|id| EvidenceRef {
94 memory_id: Some(*id),
95 ..EvidenceRef::default()
96 }));
97 refs
98 }
99}
100
101#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
102pub struct GoldenHopPath {
103 pub source: String,
104 pub entity_type: String,
105 pub entity: String,
106 pub target: String,
107}
108
109#[derive(Debug, Clone, Default, serde::Deserialize, serde::Serialize)]
110pub struct EvidenceRef {
111 pub memory_id: Option<i64>,
112 pub topic_key: Option<String>,
113 pub project: Option<String>,
114 pub branch: Option<String>,
115 pub memory_type: Option<String>,
116 pub scope: Option<String>,
117 pub title_contains: Option<String>,
118 pub text_contains: Option<String>,
119}
120
121impl EvidenceRef {
122 pub fn has_match_criteria(&self) -> bool {
123 self.memory_id.is_some()
124 || self.topic_key.is_some()
125 || self.project.is_some()
126 || self.branch.is_some()
127 || self.memory_type.is_some()
128 || self.scope.is_some()
129 || self.title_contains.is_some()
130 || self.text_contains.is_some()
131 }
132
133 pub fn matches(&self, memory: &Memory) -> bool {
134 if !self.has_match_criteria() {
135 return false;
136 }
137 if let Some(memory_id) = self.memory_id {
138 if memory.id != memory_id {
139 return false;
140 }
141 }
142 if let Some(project) = self.project.as_deref() {
143 if !crate::project_id::project_matches(Some(&memory.project), project) {
144 return false;
145 }
146 }
147 if let Some(branch) = self.branch.as_deref() {
148 if memory.branch.as_deref() != Some(branch) {
149 return false;
150 }
151 }
152 if let Some(topic_key) = self.topic_key.as_deref() {
153 if memory.topic_key.as_deref() != Some(topic_key) {
154 return false;
155 }
156 }
157 if let Some(memory_type) = self.memory_type.as_deref() {
158 if memory.memory_type != memory_type {
159 return false;
160 }
161 }
162 if let Some(scope) = self.scope.as_deref() {
163 if memory.scope != scope {
164 return false;
165 }
166 }
167 if let Some(needle) = self.title_contains.as_deref() {
168 if !contains_case_insensitive(&memory.title, needle) {
169 return false;
170 }
171 }
172 if let Some(needle) = self.text_contains.as_deref() {
173 if !contains_case_insensitive(&memory.text, needle) {
174 return false;
175 }
176 }
177 true
178 }
179}
180
181pub(super) fn contains_case_insensitive(haystack: &str, needle: &str) -> bool {
182 haystack.to_lowercase().contains(&needle.to_lowercase())
183}
184
185#[derive(Debug, Clone, serde::Serialize)]
186pub struct GoldenEvalReport {
187 pub evaluation_layers: EvaluationLayers,
188 pub version: Option<String>,
189 pub description: Option<String>,
190 pub k: usize,
191 pub rank_k: usize,
192 pub total_queries: usize,
193 pub scored_queries: usize,
194 pub skipped_queries: usize,
195 pub abstention_queries: usize,
196 pub abstention_passed: usize,
197 pub overall: Option<MetricAverages>,
198 pub by_slice: BTreeMap<String, CategoryEvaluation>,
199 pub by_category: BTreeMap<String, CategoryEvaluation>,
200 pub queries: Vec<QueryEvaluation>,
201}
202
203#[derive(Debug, Clone, serde::Serialize)]
204pub struct CategoryEvaluation {
205 pub total_queries: usize,
206 pub scored_queries: usize,
207 pub abstention_queries: usize,
208 pub abstention_passed: usize,
209 pub query_tokens_per_query: f64,
210 pub retrieval_latency_p50_ms: f64,
211 pub retrieval_latency_p95_ms: f64,
212 pub metrics: Option<MetricAverages>,
213}
214
215#[derive(Debug, Clone, serde::Serialize)]
216pub struct QueryEvaluation {
217 pub id: String,
218 pub query: String,
219 pub category: String,
220 pub slice: String,
221 pub status: QueryStatus,
222 pub result_count: usize,
223 pub retrieved_ids: Vec<i64>,
224 pub expected_relevant_ids: Vec<i64>,
225 pub missing_relevant_ids: Vec<i64>,
226 pub missing_evidence_refs: Vec<EvidenceRef>,
227 pub matched_refs: usize,
228 pub expected_refs: usize,
229 pub query_tokens: usize,
230 pub retrieval_latency_ms: f64,
231 pub metrics: Option<QueryMetrics>,
232}
233
234#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
235pub enum QueryStatus {
236 #[serde(rename = "HIT")]
237 Hit,
238 #[serde(rename = "MISS")]
239 Miss,
240 #[serde(rename = "PASS")]
241 Pass,
242 #[serde(rename = "FAIL")]
243 Fail,
244 #[serde(rename = "SKIP")]
245 Skip,
246}
247
248impl QueryStatus {
249 pub fn label(self) -> &'static str {
250 match self {
251 QueryStatus::Hit => "HIT",
252 QueryStatus::Miss => "MISS",
253 QueryStatus::Pass => "PASS",
254 QueryStatus::Fail => "FAIL",
255 QueryStatus::Skip => "SKIP",
256 }
257 }
258}
259
260#[derive(Debug, Clone, serde::Serialize)]
261pub struct EvaluationLayers {
262 pub retrieval: LayerStatus,
263 pub answer_generation: LayerStatus,
264 pub llm_judge: LayerStatus,
265}
266
267impl EvaluationLayers {
268 pub fn deterministic_retrieval_only() -> Self {
269 Self {
270 retrieval: LayerStatus {
271 status: "deterministic",
272 description:
273 "fixed golden retrieval metrics: Hit@K, Recall@K, MRR, nDCG, evidence recall",
274 },
275 answer_generation: LayerStatus {
276 status: "not_run",
277 description:
278 "answer generation is intentionally excluded from golden retrieval eval",
279 },
280 llm_judge: LayerStatus {
281 status: "not_run",
282 description: "LLM judging is intentionally excluded from deterministic golden eval",
283 },
284 }
285 }
286}
287
288#[derive(Debug, Clone, serde::Serialize)]
289pub struct LayerStatus {
290 pub status: &'static str,
291 pub description: &'static str,
292}
293
294#[derive(Debug, Clone, Default, serde::Serialize)]
295pub struct QueryMetrics {
296 pub hit_at_k: f64,
297 pub mrr_at_10: f64,
298 pub precision_at_k: f64,
299 pub recall_at_k: f64,
300 pub ndcg_at_10: f64,
301 pub evidence_recall_at_k: f64,
302}
303
304#[derive(Debug, Clone, Default, serde::Serialize)]
305pub struct MetricAverages {
306 pub count: usize,
307 pub hit_at_k: f64,
308 pub mrr_at_10: f64,
309 pub precision_at_k: f64,
310 pub recall_at_k: f64,
311 pub ndcg_at_10: f64,
312 pub evidence_recall_at_k: f64,
313}
314
315#[derive(Debug, Default)]
316pub(super) struct MetricSums {
317 count: usize,
318 hit_at_k: f64,
319 mrr_at_10: f64,
320 precision_at_k: f64,
321 recall_at_k: f64,
322 ndcg_at_10: f64,
323 evidence_recall_at_k: f64,
324}
325
326impl MetricSums {
327 pub(super) fn add(&mut self, metrics: &QueryMetrics) {
328 self.count += 1;
329 self.hit_at_k += metrics.hit_at_k;
330 self.mrr_at_10 += metrics.mrr_at_10;
331 self.precision_at_k += metrics.precision_at_k;
332 self.recall_at_k += metrics.recall_at_k;
333 self.ndcg_at_10 += metrics.ndcg_at_10;
334 self.evidence_recall_at_k += metrics.evidence_recall_at_k;
335 }
336
337 pub(super) fn averages(&self) -> Option<MetricAverages> {
338 if self.count == 0 {
339 return None;
340 }
341 let n = self.count as f64;
342 Some(MetricAverages {
343 count: self.count,
344 hit_at_k: self.hit_at_k / n,
345 mrr_at_10: self.mrr_at_10 / n,
346 precision_at_k: self.precision_at_k / n,
347 recall_at_k: self.recall_at_k / n,
348 ndcg_at_10: self.ndcg_at_10 / n,
349 evidence_recall_at_k: self.evidence_recall_at_k / n,
350 })
351 }
352}