1use crate::core::config::SearchConfig;
8use crate::core::index::Connection;
9use crate::core::search::{Hit, QueryEmbedder, SearchPipeline, SharedEmbedder};
10use serde::{Deserialize, Serialize};
11use std::collections::hash_map::DefaultHasher;
12use std::collections::HashMap;
13use std::fs;
14use std::hash::{Hash, Hasher};
15use std::path::{Path, PathBuf};
16use thiserror::Error;
17
18#[derive(Debug, Error)]
20pub enum EvalError {
21 #[error("I/O error: {0}")]
22 Io(#[from] std::io::Error),
23 #[error("JSON parse error: {0}")]
24 Json(#[from] serde_json::Error),
25 #[error("Search error: {0}")]
26 Search(#[from] anyhow::Error),
27 #[error("Database error: {0}")]
28 Database(#[from] rusqlite::Error),
29 #[error("Citation resolution failed: {0}")]
30 Citation(String),
31 #[error("Invalid golden set: {0}")]
32 InvalidGoldenSet(String),
33}
34
35#[derive(Debug, Clone, Serialize, Deserialize)]
37pub struct GoldenQuery {
38 pub id: String,
40 pub text: String,
42 pub qrels: HashMap<i64, u8>,
44 pub answerable: bool,
46 #[serde(default)]
48 pub expected_snippet: Option<String>,
49}
50
51#[derive(Debug, Clone, Serialize, Deserialize)]
53pub struct GoldenDocument {
54 pub id: i64,
56 pub path: String,
58 pub content: String,
60 #[serde(default)]
62 pub page: i64,
63}
64
65#[derive(Debug, Clone, Serialize, Deserialize)]
67pub struct GoldenSet {
68 pub queries: Vec<GoldenQuery>,
70 pub documents: Vec<GoldenDocument>,
72}
73
74#[derive(Debug, Clone, Serialize, Deserialize)]
76pub struct QueryMetrics {
77 pub query_id: String,
78 pub recall_at_10: f32,
79 pub recall_at_5: f32,
80 pub mrr: f32,
81 pub ndcg_at_10: f32,
82 pub citation_accuracy: f32,
83 pub abstention_correct: bool,
84 pub retrieved_count: usize,
85 pub relevant_count: usize,
86}
87
88#[derive(Debug, Clone, Serialize, Deserialize)]
90pub struct EvalReport {
91 pub per_query: Vec<QueryMetrics>,
92 pub macro_recall_at_10: f32,
93 pub macro_recall_at_5: f32,
94 pub macro_mrr: f32,
95 pub macro_ndcg_at_10: f32,
96 pub macro_citation_accuracy: f32,
97 pub macro_abstention_correctness: f32,
98 pub total_queries: usize,
99 pub answerable_queries: usize,
100 pub unanswerable_queries: usize,
101}
102
103pub fn load_golden_set<P: AsRef<Path>>(path: P) -> Result<GoldenSet, EvalError> {
105 let content = fs::read_to_string(path)?;
106 let set: GoldenSet = serde_json::from_str(&content)?;
107 set.validate()?;
108 Ok(set)
109}
110
111pub fn build_golden_index(
114 golden_set: &GoldenSet,
115 embedder: SharedEmbedder,
116) -> Result<(Connection, HashMap<usize, Vec<i64>>), EvalError> {
117 use crate::core::index::{init_db, Indexer};
118 use std::env;
119
120 let temp_dir = env::temp_dir().join(format!("paper_eval_{}", std::process::id()));
122 let db_path = temp_dir.join("eval.db");
123 let docs_dir = temp_dir.join("docs");
124 fs::create_dir_all(&docs_dir)?;
125
126 for doc in &golden_set.documents {
128 let doc_path = docs_dir.join(&doc.path);
129 fs::create_dir_all(doc_path.parent().unwrap())?;
130 fs::write(&doc_path, &doc.content)?;
131 }
132
133 {
135 let conn = Connection::open(&db_path)?;
136 init_db(&conn)?;
137 let indexer = Indexer::new(conn, Some(embedder))?;
138 indexer.index_roots(&[docs_dir])?;
139 }
140 let conn = Connection::open(&db_path)?;
142
143 let mut doc_to_chunks: HashMap<usize, Vec<i64>> = HashMap::new();
146
147 let path_to_doc_idx: HashMap<String, usize> = golden_set
149 .documents
150 .iter()
151 .enumerate()
152 .map(|(i, d)| (d.path.clone(), i))
153 .collect();
154
155 {
156 let mut stmt = conn.inner().prepare("SELECT id, source_path FROM chunks")?;
157 let rows = stmt.query_map([], |row| {
158 let id: i64 = row.get(0)?;
159 let source_path: String = row.get(1)?;
160 Ok((id, source_path))
161 })?;
162
163 for row in rows {
164 let (chunk_id, source_path) = row?;
165 let filename = std::path::Path::new(&source_path)
167 .file_name()
168 .and_then(|n| n.to_str())
169 .unwrap_or(&source_path)
170 .to_string();
171 if let Some(&doc_idx) = path_to_doc_idx.get(&filename) {
172 doc_to_chunks.entry(doc_idx).or_default().push(chunk_id);
173 }
174 }
175 } Ok((conn, doc_to_chunks))
178}
179
180pub fn run_eval(
182 pipeline: &SearchPipeline,
183 golden_set: &GoldenSet,
184 doc_to_chunks: &HashMap<usize, Vec<i64>>,
185) -> Result<EvalReport, EvalError> {
186 let mut per_query = Vec::with_capacity(golden_set.queries.len());
187
188 for query in &golden_set.queries {
189 let metrics = evaluate_query(pipeline, query, &golden_set.documents, doc_to_chunks)?;
190 per_query.push(metrics);
191 }
192
193 let total = per_query.len() as f32;
194 let answerable = per_query.iter().filter(|q| q.relevant_count > 0).count() as f32;
195 let unanswerable = total - answerable;
196
197 let macro_recall_at_10 = per_query.iter().map(|q| q.recall_at_10).sum::<f32>() / total;
198 let macro_recall_at_5 = per_query.iter().map(|q| q.recall_at_5).sum::<f32>() / total;
199 let macro_mrr = per_query.iter().map(|q| q.mrr).sum::<f32>() / total;
200 let macro_ndcg_at_10 = per_query.iter().map(|q| q.ndcg_at_10).sum::<f32>() / total;
201 let macro_citation_accuracy =
202 per_query.iter().map(|q| q.citation_accuracy).sum::<f32>() / total;
203 let macro_abstention_correctness = per_query
204 .iter()
205 .map(|q| q.abstention_correct as u8 as f32)
206 .sum::<f32>()
207 / total;
208
209 Ok(EvalReport {
210 per_query,
211 macro_recall_at_10,
212 macro_recall_at_5,
213 macro_mrr,
214 macro_ndcg_at_10,
215 macro_citation_accuracy,
216 macro_abstention_correctness,
217 total_queries: total as usize,
218 answerable_queries: answerable as usize,
219 unanswerable_queries: unanswerable as usize,
220 })
221}
222
223fn evaluate_query(
225 pipeline: &SearchPipeline,
226 query: &GoldenQuery,
227 documents: &[GoldenDocument],
228 doc_to_chunks: &HashMap<usize, Vec<i64>>,
229) -> Result<QueryMetrics, EvalError> {
230 let roots: Vec<PathBuf> = vec![]; let hits = pipeline.search(&query.text, &roots)?;
233
234 let mut translated_qrels: HashMap<i64, u8> = HashMap::new();
236 for (&doc_idx, &rel) in &query.qrels {
237 if let Some(chunk_ids) = doc_to_chunks.get(&(doc_idx as usize)) {
238 for &chunk_id in chunk_ids {
239 translated_qrels.insert(chunk_id, rel);
240 }
241 }
242 }
243
244 let relevant_ids: Vec<i64> = translated_qrels
246 .iter()
247 .filter(|(_, &rel)| rel > 0)
248 .map(|(&id, _)| id)
249 .collect();
250
251 let retrieved_ids: Vec<i64> = hits.iter().map(|h| h.source_id).collect();
252
253 let recall_at_10 = recall_at_k(&retrieved_ids, &relevant_ids, 10);
255 let recall_at_5 = recall_at_k(&retrieved_ids, &relevant_ids, 5);
256
257 let mrr = compute_mrr(&retrieved_ids, &relevant_ids);
259
260 let ndcg_at_10 = compute_ndcg_at_k(&retrieved_ids, &translated_qrels, 10);
262
263 let citation_accuracy = compute_citation_accuracy(&hits, query, documents, doc_to_chunks)?;
267
268 let abstention_correct = compute_abstention_correctness(&hits, query.answerable);
270
271 Ok(QueryMetrics {
272 query_id: query.id.clone(),
273 recall_at_10,
274 recall_at_5,
275 mrr,
276 ndcg_at_10,
277 citation_accuracy,
278 abstention_correct,
279 retrieved_count: hits.len(),
280 relevant_count: relevant_ids.len(),
281 })
282}
283
284fn recall_at_k(retrieved: &[i64], relevant: &[i64], k: usize) -> f32 {
286 if relevant.is_empty() {
287 return 1.0; }
289 let top_k: std::collections::HashSet<_> = retrieved.iter().take(k).collect();
290 let found = relevant.iter().filter(|id| top_k.contains(id)).count();
291 found as f32 / relevant.len() as f32
292}
293
294fn compute_mrr(retrieved: &[i64], relevant: &[i64]) -> f32 {
296 let relevant_set: std::collections::HashSet<_> = relevant.iter().collect();
297 for (rank, id) in retrieved.iter().enumerate() {
298 if relevant_set.contains(id) {
299 return 1.0 / (rank + 1) as f32;
300 }
301 }
302 0.0
303}
304
305fn compute_ndcg_at_k(retrieved: &[i64], qrels: &HashMap<i64, u8>, k: usize) -> f32 {
308 let mut dcg = 0.0f32;
309 for (i, &id) in retrieved.iter().take(k).enumerate() {
310 let rel = qrels.get(&id).copied().unwrap_or(0) as f32;
311 let gain = 2.0f32.powf(rel) - 1.0; dcg += gain / ((i + 2) as f32).log2();
313 }
314
315 let mut ideal_gains: Vec<f32> = qrels
317 .values()
318 .filter(|&&r| r > 0)
319 .map(|&r| 2.0f32.powf(r as f32) - 1.0)
320 .collect();
321 ideal_gains.sort_by(|a, b| b.partial_cmp(a).unwrap());
322
323 let mut idcg = 0.0f32;
324 for (i, gain) in ideal_gains.iter().take(k).enumerate() {
325 idcg += gain / ((i + 2) as f32).log2();
326 }
327
328 if idcg == 0.0 {
329 1.0 } else {
331 dcg / idcg
332 }
333}
334
335fn compute_citation_accuracy(
339 hits: &[Hit],
340 query: &GoldenQuery,
341 documents: &[GoldenDocument],
342 doc_to_chunks: &HashMap<usize, Vec<i64>>,
343) -> Result<f32, EvalError> {
344 let supported_hits: Vec<&Hit> = hits
346 .iter()
347 .filter(|h| !matches!(h.confidence, crate::core::search::Confidence::Low))
348 .collect();
349
350 if supported_hits.is_empty() {
351 return Ok(if query.answerable { 0.0 } else { 1.0 });
353 }
354
355 let top_hit = supported_hits[0];
359
360 let mut chunk_to_doc: HashMap<i64, usize> = HashMap::new();
362 for (doc_idx, chunk_ids) in doc_to_chunks {
363 for &chunk_id in chunk_ids {
364 chunk_to_doc.insert(chunk_id, *doc_idx);
365 }
366 }
367
368 if let Some(&doc_idx) = chunk_to_doc.get(&top_hit.source_id) {
369 if let Some(doc) = documents.get(doc_idx) {
370 let lines: Vec<&str> = doc.content.lines().collect();
372 let start = (top_hit.line_start - 1).max(0) as usize;
373 let end = (top_hit.line_end as usize).min(lines.len());
374
375 if start < lines.len() && start <= end {
376 let cited_text: String = lines[start..end].join("\n");
377
378 let is_correct = if let Some(expected) = &query.expected_snippet {
380 cited_text.to_lowercase().contains(&expected.to_lowercase())
381 } else {
382 let query_lower = query.text.to_lowercase();
384 let query_terms: Vec<String> = query_lower
385 .split_whitespace()
386 .filter(|t| t.len() >= 3)
387 .map(|s| s.to_string())
388 .collect();
389 let cited_lower = cited_text.to_lowercase();
390 if query_terms.is_empty() {
391 false
392 } else {
393 let matches = query_terms
394 .iter()
395 .filter(|t| cited_lower.contains(*t))
396 .count();
397 matches as f32 / query_terms.len() as f32 > 0.3
398 }
399 };
400
401 return Ok(if is_correct { 1.0 } else { 0.0 });
402 }
403 }
404 }
405
406 Ok(0.0)
407}
408
409fn compute_abstention_correctness(hits: &[Hit], answerable: bool) -> bool {
414 let has_high_or_medium = hits
415 .iter()
416 .any(|h| !matches!(h.confidence, crate::core::search::Confidence::Low));
417
418 if answerable {
419 has_high_or_medium
421 } else {
422 !has_high_or_medium
424 }
425}
426
427impl GoldenSet {
428 fn validate(&self) -> Result<(), EvalError> {
430 if self.queries.is_empty() {
431 return Err(EvalError::InvalidGoldenSet(
432 "No queries in golden set".into(),
433 ));
434 }
435 if self.documents.is_empty() {
436 return Err(EvalError::InvalidGoldenSet(
437 "No documents in golden set".into(),
438 ));
439 }
440
441 let doc_ids: std::collections::HashSet<i64> = self
443 .documents
444 .iter()
445 .enumerate()
446 .map(|(i, _)| i as i64)
447 .collect();
448
449 for query in &self.queries {
450 for &chunk_id in query.qrels.keys() {
451 if !doc_ids.contains(&chunk_id) {
452 return Err(EvalError::InvalidGoldenSet(format!(
453 "Query {} references unknown chunk_id {}",
454 query.id, chunk_id
455 )));
456 }
457 }
458 }
459
460 Ok(())
461 }
462}
463
464impl EvalReport {
465 pub fn save_json<P: AsRef<Path>>(&self, path: P) -> Result<(), EvalError> {
467 let json = serde_json::to_string_pretty(self)?;
468 fs::write(path, json)?;
469 Ok(())
470 }
471
472 pub fn check_ci_gates(&self) -> (bool, Vec<String>) {
475 let mut reasons = Vec::new();
476
477 if self.macro_recall_at_10 < 0.75 {
478 reasons.push(format!("recall@10 {:.3} < 0.75", self.macro_recall_at_10));
479 }
480 if self.macro_citation_accuracy < 0.95 {
481 reasons.push(format!(
482 "citation_accuracy {:.3} < 0.95",
483 self.macro_citation_accuracy
484 ));
485 }
486
487 (reasons.is_empty(), reasons)
488 }
489}
490
491pub fn make_eval_pipeline(conn: Connection) -> SearchPipeline {
493 use crate::core::search::SharedEmbedder;
494 use std::sync::Arc;
495
496 struct HashEmbedder;
497 impl QueryEmbedder for HashEmbedder {
498 fn embed(&self, text: &str) -> anyhow::Result<Vec<f32>> {
499 let mut v = vec![0.0f32; crate::core::index::EMBEDDING_DIMS];
500 for token in text
501 .to_lowercase()
502 .split(|c: char| !c.is_alphanumeric())
503 .filter(|t| t.len() > 2)
504 {
505 let mut h = DefaultHasher::new();
506 token.hash(&mut h);
507 let idx = (h.finish() as usize) % v.len();
508 v[idx] += 1.0;
509 }
510 let norm = v.iter().map(|x| x * x).sum::<f32>().sqrt();
511 if norm > 0.0 {
512 for x in &mut v {
513 *x /= norm;
514 }
515 }
516 Ok(v)
517 }
518
519 fn model_id(&self) -> &str {
520 "test-hash-embedder"
521 }
522 }
523
524 let embedder: SharedEmbedder = Arc::new(HashEmbedder);
525 SearchPipeline::new(conn, Some(embedder), None, SearchConfig::default())
526}
527
528#[cfg(test)]
529mod tests {
530 use super::*;
531 use crate::core::search::Confidence;
532
533 #[test]
534 fn test_recall_at_k() {
535 let retrieved = vec![1, 2, 3, 4, 5];
536 let relevant = vec![2, 4, 6];
537 assert!((recall_at_k(&retrieved, &relevant, 5) - 2.0 / 3.0).abs() < 0.001);
538 assert!((recall_at_k(&retrieved, &relevant, 3) - 1.0 / 3.0).abs() < 0.001);
539 assert_eq!(recall_at_k(&retrieved, &[], 5), 1.0);
540 }
541
542 #[test]
543 fn test_mrr() {
544 let retrieved = vec![1, 2, 3, 4, 5];
545 let relevant = vec![3, 5];
546 assert!((compute_mrr(&retrieved, &relevant) - 1.0 / 3.0).abs() < 0.001);
547
548 let relevant = vec![6, 7];
549 assert_eq!(compute_mrr(&retrieved, &relevant), 0.0);
550 }
551
552 #[test]
553 fn test_ndcg_at_k() {
554 let retrieved = vec![1, 2, 3];
555 let mut qrels = HashMap::new();
556 qrels.insert(1, 2); qrels.insert(2, 1); qrels.insert(3, 0); let ndcg = compute_ndcg_at_k(&retrieved, &qrels, 3);
561 assert!((ndcg - 1.0).abs() < 0.01);
565
566 let retrieved = vec![2, 1, 3];
568 let ndcg = compute_ndcg_at_k(&retrieved, &qrels, 3);
569 assert!((ndcg - 0.796).abs() < 0.01);
572 }
573
574 #[test]
575 fn test_abstention_correctness() {
576 let hits = vec![Hit {
578 source_id: 1,
579 path: "test.txt".into(),
580 source_path: "test.txt".into(),
581 page: 0,
582 line_start: 1,
583 line_end: 1,
584 anchor: None,
585 section: None,
586 snippet: "test".into(),
587 content: "test".into(),
588 score: 1.0,
589 why_matched: "test".into(),
590 confidence: Confidence::High,
591 engine: crate::core::search::Engine::Local,
592 }];
593 assert!(compute_abstention_correctness(&hits, true));
594
595 let hits = vec![Hit {
597 source_id: 1,
598 path: "test.txt".into(),
599 source_path: "test.txt".into(),
600 page: 0,
601 line_start: 1,
602 line_end: 1,
603 anchor: None,
604 section: None,
605 snippet: "test".into(),
606 content: "test".into(),
607 score: 0.1,
608 why_matched: "test".into(),
609 confidence: Confidence::Low,
610 engine: crate::core::search::Engine::Local,
611 }];
612 assert!(!compute_abstention_correctness(&hits, true));
613
614 let hits = vec![];
616 assert!(compute_abstention_correctness(&hits, false));
617
618 let hits = vec![Hit {
620 source_id: 1,
621 path: "test.txt".into(),
622 source_path: "test.txt".into(),
623 page: 0,
624 line_start: 1,
625 line_end: 1,
626 anchor: None,
627 section: None,
628 snippet: "test".into(),
629 content: "test".into(),
630 score: 0.1,
631 why_matched: "test".into(),
632 confidence: Confidence::Low,
633 engine: crate::core::search::Engine::Local,
634 }];
635 assert!(compute_abstention_correctness(&hits, false));
636
637 let hits = vec![Hit {
639 source_id: 1,
640 path: "test.txt".into(),
641 source_path: "test.txt".into(),
642 page: 0,
643 line_start: 1,
644 line_end: 1,
645 anchor: None,
646 section: None,
647 snippet: "test".into(),
648 content: "test".into(),
649 score: 0.5,
650 why_matched: "test".into(),
651 confidence: Confidence::Medium,
652 engine: crate::core::search::Engine::Local,
653 }];
654 assert!(!compute_abstention_correctness(&hits, false));
655 }
656}