1use crate::core::config::SearchConfig;
8use crate::core::index::Connection;
9use crate::core::search::{Hit, QueryEmbedder, SearchPipeline, SharedEmbedder};
10use serde::{Deserialize, Serialize};
11use std::collections::hash_map::DefaultHasher;
12use std::collections::HashMap;
13use std::fs;
14use std::hash::{Hash, Hasher};
15use std::path::{Path, PathBuf};
16use tempfile::TempDir;
17use thiserror::Error;
18
19#[derive(Debug, Error)]
21pub enum EvalError {
22 #[error("I/O error: {0}")]
23 Io(#[from] std::io::Error),
24 #[error("JSON parse error: {0}")]
25 Json(#[from] serde_json::Error),
26 #[error("Search error: {0}")]
27 Search(#[from] anyhow::Error),
28 #[error("Database error: {0}")]
29 Database(#[from] rusqlite::Error),
30 #[error("Citation resolution failed: {0}")]
31 Citation(String),
32 #[error("Invalid golden set: {0}")]
33 InvalidGoldenSet(String),
34}
35
36#[derive(Debug, Clone, Serialize, Deserialize)]
38pub struct GoldenQuery {
39 pub id: String,
41 pub text: String,
43 pub qrels: HashMap<i64, u8>,
45 pub answerable: bool,
47 #[serde(default)]
49 pub expected_snippet: Option<String>,
50}
51
52#[derive(Debug, Clone, Serialize, Deserialize)]
54pub struct GoldenDocument {
55 pub id: i64,
57 pub path: String,
59 pub content: String,
61 #[serde(default)]
63 pub page: i64,
64}
65
66#[derive(Debug, Clone, Serialize, Deserialize)]
68pub struct GoldenSet {
69 pub queries: Vec<GoldenQuery>,
71 pub documents: Vec<GoldenDocument>,
73}
74
75#[derive(Debug, Clone, Serialize, Deserialize)]
77pub struct QueryMetrics {
78 pub query_id: String,
79 pub recall_at_10: f32,
80 pub recall_at_5: f32,
81 pub mrr: f32,
82 pub ndcg_at_10: f32,
83 pub citation_accuracy: f32,
84 pub abstention_correct: bool,
85 pub retrieved_count: usize,
86 pub relevant_count: usize,
87}
88
89#[derive(Debug, Clone, Serialize, Deserialize)]
91pub struct EvalReport {
92 pub per_query: Vec<QueryMetrics>,
93 pub macro_recall_at_10: f32,
94 pub macro_recall_at_5: f32,
95 pub macro_mrr: f32,
96 pub macro_ndcg_at_10: f32,
97 pub macro_citation_accuracy: f32,
98 pub macro_abstention_correctness: f32,
99 pub total_queries: usize,
100 pub answerable_queries: usize,
101 pub unanswerable_queries: usize,
102}
103
104pub fn load_golden_set<P: AsRef<Path>>(path: P) -> Result<GoldenSet, EvalError> {
106 let content = fs::read_to_string(path)?;
107 let set: GoldenSet = serde_json::from_str(&content)?;
108 set.validate()?;
109 Ok(set)
110}
111
112pub fn build_golden_index(
115 golden_set: &GoldenSet,
116 embedder: SharedEmbedder,
117) -> Result<(Connection, HashMap<usize, Vec<i64>>), EvalError> {
118 use crate::core::index::{init_db, Indexer};
119
120 let temp_dir = TempDir::new()?;
126 let db_path = temp_dir.path().join("eval.db");
127 let docs_dir = temp_dir.path().join("docs");
128 fs::create_dir_all(&docs_dir)?;
129
130 for doc in &golden_set.documents {
132 let doc_path = docs_dir.join(&doc.path);
133 fs::create_dir_all(doc_path.parent().unwrap())?;
134 fs::write(&doc_path, &doc.content)?;
135 }
136
137 {
139 let conn = Connection::open(&db_path)?;
140 init_db(&conn)?;
141 let indexer = Indexer::new(conn, Some(embedder))?;
142 indexer.index_roots(&[docs_dir])?;
143 }
144 let conn = Connection::open(&db_path)?;
146
147 let mut doc_to_chunks: HashMap<usize, Vec<i64>> = HashMap::new();
150
151 let path_to_doc_idx: HashMap<String, usize> = golden_set
153 .documents
154 .iter()
155 .enumerate()
156 .map(|(i, d)| (d.path.clone(), i))
157 .collect();
158
159 {
160 let mut stmt = conn.inner().prepare("SELECT id, source_path FROM chunks")?;
161 let rows = stmt.query_map([], |row| {
162 let id: i64 = row.get(0)?;
163 let source_path: String = row.get(1)?;
164 Ok((id, source_path))
165 })?;
166
167 for row in rows {
168 let (chunk_id, source_path) = row?;
169 let filename = std::path::Path::new(&source_path)
171 .file_name()
172 .and_then(|n| n.to_str())
173 .unwrap_or(&source_path)
174 .to_string();
175 if let Some(&doc_idx) = path_to_doc_idx.get(&filename) {
176 doc_to_chunks.entry(doc_idx).or_default().push(chunk_id);
177 }
178 }
179 } Ok((conn, doc_to_chunks))
182}
183
184pub fn run_eval(
186 pipeline: &SearchPipeline,
187 golden_set: &GoldenSet,
188 doc_to_chunks: &HashMap<usize, Vec<i64>>,
189) -> Result<EvalReport, EvalError> {
190 let mut per_query = Vec::with_capacity(golden_set.queries.len());
191
192 for query in &golden_set.queries {
193 let metrics = evaluate_query(pipeline, query, &golden_set.documents, doc_to_chunks)?;
194 per_query.push(metrics);
195 }
196
197 let total = per_query.len() as f32;
198 let answerable = per_query.iter().filter(|q| q.relevant_count > 0).count() as f32;
199 let unanswerable = total - answerable;
200
201 let macro_recall_at_10 = per_query.iter().map(|q| q.recall_at_10).sum::<f32>() / total;
202 let macro_recall_at_5 = per_query.iter().map(|q| q.recall_at_5).sum::<f32>() / total;
203 let macro_mrr = per_query.iter().map(|q| q.mrr).sum::<f32>() / total;
204 let macro_ndcg_at_10 = per_query.iter().map(|q| q.ndcg_at_10).sum::<f32>() / total;
205 let macro_citation_accuracy =
206 per_query.iter().map(|q| q.citation_accuracy).sum::<f32>() / total;
207 let macro_abstention_correctness = per_query
208 .iter()
209 .map(|q| q.abstention_correct as u8 as f32)
210 .sum::<f32>()
211 / total;
212
213 Ok(EvalReport {
214 per_query,
215 macro_recall_at_10,
216 macro_recall_at_5,
217 macro_mrr,
218 macro_ndcg_at_10,
219 macro_citation_accuracy,
220 macro_abstention_correctness,
221 total_queries: total as usize,
222 answerable_queries: answerable as usize,
223 unanswerable_queries: unanswerable as usize,
224 })
225}
226
227fn evaluate_query(
229 pipeline: &SearchPipeline,
230 query: &GoldenQuery,
231 documents: &[GoldenDocument],
232 doc_to_chunks: &HashMap<usize, Vec<i64>>,
233) -> Result<QueryMetrics, EvalError> {
234 let roots: Vec<PathBuf> = vec![]; let hits = pipeline.search(&query.text, &roots)?;
237
238 let mut translated_qrels: HashMap<i64, u8> = HashMap::new();
240 for (&doc_idx, &rel) in &query.qrels {
241 if let Some(chunk_ids) = doc_to_chunks.get(&(doc_idx as usize)) {
242 for &chunk_id in chunk_ids {
243 translated_qrels.insert(chunk_id, rel);
244 }
245 }
246 }
247
248 let relevant_ids: Vec<i64> = translated_qrels
250 .iter()
251 .filter(|(_, &rel)| rel > 0)
252 .map(|(&id, _)| id)
253 .collect();
254
255 let retrieved_ids: Vec<i64> = hits.iter().map(|h| h.source_id).collect();
256
257 let recall_at_10 = recall_at_k(&retrieved_ids, &relevant_ids, 10);
259 let recall_at_5 = recall_at_k(&retrieved_ids, &relevant_ids, 5);
260
261 let mrr = compute_mrr(&retrieved_ids, &relevant_ids);
263
264 let ndcg_at_10 = compute_ndcg_at_k(&retrieved_ids, &translated_qrels, 10);
266
267 let citation_accuracy = compute_citation_accuracy(&hits, query, documents, doc_to_chunks)?;
271
272 let abstention_correct = compute_abstention_correctness(&hits, query.answerable);
274
275 Ok(QueryMetrics {
276 query_id: query.id.clone(),
277 recall_at_10,
278 recall_at_5,
279 mrr,
280 ndcg_at_10,
281 citation_accuracy,
282 abstention_correct,
283 retrieved_count: hits.len(),
284 relevant_count: relevant_ids.len(),
285 })
286}
287
288fn recall_at_k(retrieved: &[i64], relevant: &[i64], k: usize) -> f32 {
290 if relevant.is_empty() {
291 return 1.0; }
293 let top_k: std::collections::HashSet<_> = retrieved.iter().take(k).collect();
294 let found = relevant.iter().filter(|id| top_k.contains(id)).count();
295 found as f32 / relevant.len() as f32
296}
297
298fn compute_mrr(retrieved: &[i64], relevant: &[i64]) -> f32 {
300 let relevant_set: std::collections::HashSet<_> = relevant.iter().collect();
301 for (rank, id) in retrieved.iter().enumerate() {
302 if relevant_set.contains(id) {
303 return 1.0 / (rank + 1) as f32;
304 }
305 }
306 0.0
307}
308
309fn compute_ndcg_at_k(retrieved: &[i64], qrels: &HashMap<i64, u8>, k: usize) -> f32 {
312 let mut dcg = 0.0f32;
313 for (i, &id) in retrieved.iter().take(k).enumerate() {
314 let rel = qrels.get(&id).copied().unwrap_or(0) as f32;
315 let gain = 2.0f32.powf(rel) - 1.0; dcg += gain / ((i + 2) as f32).log2();
317 }
318
319 let mut ideal_gains: Vec<f32> = qrels
321 .values()
322 .filter(|&&r| r > 0)
323 .map(|&r| 2.0f32.powf(r as f32) - 1.0)
324 .collect();
325 ideal_gains.sort_by(|a, b| b.partial_cmp(a).unwrap());
326
327 let mut idcg = 0.0f32;
328 for (i, gain) in ideal_gains.iter().take(k).enumerate() {
329 idcg += gain / ((i + 2) as f32).log2();
330 }
331
332 if idcg == 0.0 {
333 1.0 } else {
335 dcg / idcg
336 }
337}
338
339fn compute_citation_accuracy(
343 hits: &[Hit],
344 query: &GoldenQuery,
345 documents: &[GoldenDocument],
346 doc_to_chunks: &HashMap<usize, Vec<i64>>,
347) -> Result<f32, EvalError> {
348 let supported_hits: Vec<&Hit> = hits
350 .iter()
351 .filter(|h| !matches!(h.confidence, crate::core::search::Confidence::Low))
352 .collect();
353
354 if supported_hits.is_empty() {
355 return Ok(if query.answerable { 0.0 } else { 1.0 });
357 }
358
359 let top_hit = supported_hits[0];
363
364 let mut chunk_to_doc: HashMap<i64, usize> = HashMap::new();
366 for (doc_idx, chunk_ids) in doc_to_chunks {
367 for &chunk_id in chunk_ids {
368 chunk_to_doc.insert(chunk_id, *doc_idx);
369 }
370 }
371
372 if let Some(&doc_idx) = chunk_to_doc.get(&top_hit.source_id) {
373 if let Some(doc) = documents.get(doc_idx) {
374 let lines: Vec<&str> = doc.content.lines().collect();
376 let start = (top_hit.line_start - 1).max(0) as usize;
377 let end = (top_hit.line_end as usize).min(lines.len());
378
379 if start < lines.len() && start <= end {
380 let cited_text: String = lines[start..end].join("\n");
381
382 let is_correct = if let Some(expected) = &query.expected_snippet {
384 cited_text.to_lowercase().contains(&expected.to_lowercase())
385 } else {
386 let query_lower = query.text.to_lowercase();
388 let query_terms: Vec<String> = query_lower
389 .split_whitespace()
390 .filter(|t| t.len() >= 3)
391 .map(|s| s.to_string())
392 .collect();
393 let cited_lower = cited_text.to_lowercase();
394 if query_terms.is_empty() {
395 false
396 } else {
397 let matches = query_terms
398 .iter()
399 .filter(|t| cited_lower.contains(*t))
400 .count();
401 matches as f32 / query_terms.len() as f32 > 0.3
402 }
403 };
404
405 return Ok(if is_correct { 1.0 } else { 0.0 });
406 }
407 }
408 }
409
410 Ok(0.0)
411}
412
413fn compute_abstention_correctness(hits: &[Hit], answerable: bool) -> bool {
418 let has_high_or_medium = hits
419 .iter()
420 .any(|h| !matches!(h.confidence, crate::core::search::Confidence::Low));
421
422 if answerable {
423 has_high_or_medium
425 } else {
426 !has_high_or_medium
428 }
429}
430
431impl GoldenSet {
432 fn validate(&self) -> Result<(), EvalError> {
434 if self.queries.is_empty() {
435 return Err(EvalError::InvalidGoldenSet(
436 "No queries in golden set".into(),
437 ));
438 }
439 if self.documents.is_empty() {
440 return Err(EvalError::InvalidGoldenSet(
441 "No documents in golden set".into(),
442 ));
443 }
444
445 let doc_ids: std::collections::HashSet<i64> = self
447 .documents
448 .iter()
449 .enumerate()
450 .map(|(i, _)| i as i64)
451 .collect();
452
453 for query in &self.queries {
454 for &chunk_id in query.qrels.keys() {
455 if !doc_ids.contains(&chunk_id) {
456 return Err(EvalError::InvalidGoldenSet(format!(
457 "Query {} references unknown chunk_id {}",
458 query.id, chunk_id
459 )));
460 }
461 }
462 }
463
464 Ok(())
465 }
466}
467
468impl EvalReport {
469 pub fn save_json<P: AsRef<Path>>(&self, path: P) -> Result<(), EvalError> {
471 let json = serde_json::to_string_pretty(self)?;
472 fs::write(path, json)?;
473 Ok(())
474 }
475
476 pub fn check_ci_gates(&self) -> (bool, Vec<String>) {
479 let mut reasons = Vec::new();
480
481 if self.macro_recall_at_10 < 0.75 {
482 reasons.push(format!("recall@10 {:.3} < 0.75", self.macro_recall_at_10));
483 }
484 if self.macro_citation_accuracy < 0.95 {
485 reasons.push(format!(
486 "citation_accuracy {:.3} < 0.95",
487 self.macro_citation_accuracy
488 ));
489 }
490
491 (reasons.is_empty(), reasons)
492 }
493}
494
495pub fn make_eval_pipeline(conn: Connection) -> SearchPipeline {
497 use crate::core::search::SharedEmbedder;
498 use std::sync::Arc;
499
500 struct HashEmbedder;
501 impl QueryEmbedder for HashEmbedder {
502 fn embed(&self, text: &str) -> anyhow::Result<Vec<f32>> {
503 let mut v = vec![0.0f32; crate::core::index::EMBEDDING_DIMS];
504 for token in text
505 .to_lowercase()
506 .split(|c: char| !c.is_alphanumeric())
507 .filter(|t| t.len() > 2)
508 {
509 let mut h = DefaultHasher::new();
510 token.hash(&mut h);
511 let idx = (h.finish() as usize) % v.len();
512 v[idx] += 1.0;
513 }
514 let norm = v.iter().map(|x| x * x).sum::<f32>().sqrt();
515 if norm > 0.0 {
516 for x in &mut v {
517 *x /= norm;
518 }
519 }
520 Ok(v)
521 }
522
523 fn model_id(&self) -> &str {
524 "test-hash-embedder"
525 }
526 }
527
528 let embedder: SharedEmbedder = Arc::new(HashEmbedder);
529 SearchPipeline::new(conn, Some(embedder), None, SearchConfig::default())
530}
531
532#[cfg(test)]
533mod tests {
534 use super::*;
535 use crate::core::search::Confidence;
536
537 #[test]
538 fn test_recall_at_k() {
539 let retrieved = vec![1, 2, 3, 4, 5];
540 let relevant = vec![2, 4, 6];
541 assert!((recall_at_k(&retrieved, &relevant, 5) - 2.0 / 3.0).abs() < 0.001);
542 assert!((recall_at_k(&retrieved, &relevant, 3) - 1.0 / 3.0).abs() < 0.001);
543 assert_eq!(recall_at_k(&retrieved, &[], 5), 1.0);
544 }
545
546 #[test]
547 fn test_mrr() {
548 let retrieved = vec![1, 2, 3, 4, 5];
549 let relevant = vec![3, 5];
550 assert!((compute_mrr(&retrieved, &relevant) - 1.0 / 3.0).abs() < 0.001);
551
552 let relevant = vec![6, 7];
553 assert_eq!(compute_mrr(&retrieved, &relevant), 0.0);
554 }
555
556 #[test]
557 fn test_ndcg_at_k() {
558 let retrieved = vec![1, 2, 3];
559 let mut qrels = HashMap::new();
560 qrels.insert(1, 2); qrels.insert(2, 1); qrels.insert(3, 0); let ndcg = compute_ndcg_at_k(&retrieved, &qrels, 3);
565 assert!((ndcg - 1.0).abs() < 0.01);
569
570 let retrieved = vec![2, 1, 3];
572 let ndcg = compute_ndcg_at_k(&retrieved, &qrels, 3);
573 assert!((ndcg - 0.796).abs() < 0.01);
576 }
577
578 #[test]
579 fn test_abstention_correctness() {
580 let hits = vec![Hit {
582 source_id: 1,
583 path: "test.txt".into(),
584 source_path: "test.txt".into(),
585 page: 0,
586 line_start: 1,
587 line_end: 1,
588 anchor: None,
589 section: None,
590 snippet: "test".into(),
591 content: "test".into(),
592 score: 1.0,
593 why_matched: "test".into(),
594 confidence: Confidence::High,
595 engine: crate::core::search::Engine::Local,
596 }];
597 assert!(compute_abstention_correctness(&hits, true));
598
599 let hits = vec![Hit {
601 source_id: 1,
602 path: "test.txt".into(),
603 source_path: "test.txt".into(),
604 page: 0,
605 line_start: 1,
606 line_end: 1,
607 anchor: None,
608 section: None,
609 snippet: "test".into(),
610 content: "test".into(),
611 score: 0.1,
612 why_matched: "test".into(),
613 confidence: Confidence::Low,
614 engine: crate::core::search::Engine::Local,
615 }];
616 assert!(!compute_abstention_correctness(&hits, true));
617
618 let hits = vec![];
620 assert!(compute_abstention_correctness(&hits, false));
621
622 let hits = vec![Hit {
624 source_id: 1,
625 path: "test.txt".into(),
626 source_path: "test.txt".into(),
627 page: 0,
628 line_start: 1,
629 line_end: 1,
630 anchor: None,
631 section: None,
632 snippet: "test".into(),
633 content: "test".into(),
634 score: 0.1,
635 why_matched: "test".into(),
636 confidence: Confidence::Low,
637 engine: crate::core::search::Engine::Local,
638 }];
639 assert!(compute_abstention_correctness(&hits, false));
640
641 let hits = vec![Hit {
643 source_id: 1,
644 path: "test.txt".into(),
645 source_path: "test.txt".into(),
646 page: 0,
647 line_start: 1,
648 line_end: 1,
649 anchor: None,
650 section: None,
651 snippet: "test".into(),
652 content: "test".into(),
653 score: 0.5,
654 why_matched: "test".into(),
655 confidence: Confidence::Medium,
656 engine: crate::core::search::Engine::Local,
657 }];
658 assert!(!compute_abstention_correctness(&hits, false));
659 }
660}