Skip to main content

paper/core/
eval.rs

1//! Offline evaluation harness for retrieval quality.
2//!
3//! Uses a deterministic `HashEmbedder` (from `tests/common/mod.rs`) so CI never
4//! downloads models. Metrics: recall@k, MRR, nDCG@10, citation accuracy,
5//! abstention correctness.
6
7use crate::core::config::SearchConfig;
8use crate::core::index::Connection;
9use crate::core::search::{Hit, QueryEmbedder, SearchPipeline, SharedEmbedder};
10use serde::{Deserialize, Serialize};
11use std::collections::hash_map::DefaultHasher;
12use std::collections::HashMap;
13use std::fs;
14use std::hash::{Hash, Hasher};
15use std::path::{Path, PathBuf};
16use thiserror::Error;
17
18/// Errors that can occur during evaluation.
19#[derive(Debug, Error)]
20pub enum EvalError {
21    #[error("I/O error: {0}")]
22    Io(#[from] std::io::Error),
23    #[error("JSON parse error: {0}")]
24    Json(#[from] serde_json::Error),
25    #[error("Search error: {0}")]
26    Search(#[from] anyhow::Error),
27    #[error("Database error: {0}")]
28    Database(#[from] rusqlite::Error),
29    #[error("Citation resolution failed: {0}")]
30    Citation(String),
31    #[error("Invalid golden set: {0}")]
32    InvalidGoldenSet(String),
33}
34
35/// A single query in the golden set.
36#[derive(Debug, Clone, Serialize, Deserialize)]
37pub struct GoldenQuery {
38    /// Unique query identifier.
39    pub id: String,
40    /// The query text.
41    pub text: String,
42    /// Relevance judgments: chunk_id -> relevance (0=not relevant, 1=relevant, 2=highly relevant).
43    pub qrels: HashMap<i64, u8>,
44    /// Whether this query should have an answer (false = unanswerable, tests abstention).
45    pub answerable: bool,
46    /// Expected answer text snippet (for citation accuracy verification).
47    #[serde(default)]
48    pub expected_snippet: Option<String>,
49}
50
51/// A document in the golden set corpus.
52#[derive(Debug, Clone, Serialize, Deserialize)]
53pub struct GoldenDocument {
54    /// Document identifier (used as chunk_id in the index).
55    pub id: i64,
56    /// Path to the document file.
57    pub path: String,
58    /// Full text content of the document.
59    pub content: String,
60    /// Optional page number (for paged formats).
61    #[serde(default)]
62    pub page: i64,
63}
64
65/// The complete golden evaluation set.
66#[derive(Debug, Clone, Serialize, Deserialize)]
67pub struct GoldenSet {
68    /// All queries to evaluate.
69    pub queries: Vec<GoldenQuery>,
70    /// All documents in the corpus.
71    pub documents: Vec<GoldenDocument>,
72}
73
74/// Metrics computed for a single query.
75#[derive(Debug, Clone, Serialize, Deserialize)]
76pub struct QueryMetrics {
77    pub query_id: String,
78    pub recall_at_10: f32,
79    pub recall_at_5: f32,
80    pub mrr: f32,
81    pub ndcg_at_10: f32,
82    pub citation_accuracy: f32,
83    pub abstention_correct: bool,
84    pub retrieved_count: usize,
85    pub relevant_count: usize,
86}
87
88/// Aggregated evaluation report.
89#[derive(Debug, Clone, Serialize, Deserialize)]
90pub struct EvalReport {
91    pub per_query: Vec<QueryMetrics>,
92    pub macro_recall_at_10: f32,
93    pub macro_recall_at_5: f32,
94    pub macro_mrr: f32,
95    pub macro_ndcg_at_10: f32,
96    pub macro_citation_accuracy: f32,
97    pub macro_abstention_correctness: f32,
98    pub total_queries: usize,
99    pub answerable_queries: usize,
100    pub unanswerable_queries: usize,
101}
102
103/// Load a golden set from a JSON file.
104pub fn load_golden_set<P: AsRef<Path>>(path: P) -> Result<GoldenSet, EvalError> {
105    let content = fs::read_to_string(path)?;
106    let set: GoldenSet = serde_json::from_str(&content)?;
107    set.validate()?;
108    Ok(set)
109}
110
111/// Build an in-memory index from the golden set documents.
112/// Returns the connection and a mapping from golden set document index to database chunk IDs.
113pub fn build_golden_index(
114    golden_set: &GoldenSet,
115    embedder: SharedEmbedder,
116) -> Result<(Connection, HashMap<usize, Vec<i64>>), EvalError> {
117    use crate::core::index::{init_db, Indexer};
118    use std::env;
119
120    // Create a temporary directory for the database and document files
121    let temp_dir = env::temp_dir().join(format!("paper_eval_{}", std::process::id()));
122    let db_path = temp_dir.join("eval.db");
123    let docs_dir = temp_dir.join("docs");
124    fs::create_dir_all(&docs_dir)?;
125
126    // Write all documents to the filesystem
127    for doc in &golden_set.documents {
128        let doc_path = docs_dir.join(&doc.path);
129        fs::create_dir_all(doc_path.parent().unwrap())?;
130        fs::write(&doc_path, &doc.content)?;
131    }
132
133    // Open connection and index
134    {
135        let conn = Connection::open(&db_path)?;
136        init_db(&conn)?;
137        let indexer = Indexer::new(conn, Some(embedder))?;
138        indexer.index_roots(&[docs_dir])?;
139    }
140    // Re-open the connection after indexing (Indexer takes ownership)
141    let conn = Connection::open(&db_path)?;
142
143    // Build mapping from golden set document index to chunk IDs
144    // by querying the database for source_path
145    let mut doc_to_chunks: HashMap<usize, Vec<i64>> = HashMap::new();
146
147    // Build a map from filename to golden set document index
148    let path_to_doc_idx: HashMap<String, usize> = golden_set
149        .documents
150        .iter()
151        .enumerate()
152        .map(|(i, d)| (d.path.clone(), i))
153        .collect();
154
155    {
156        let mut stmt = conn.inner().prepare("SELECT id, source_path FROM chunks")?;
157        let rows = stmt.query_map([], |row| {
158            let id: i64 = row.get(0)?;
159            let source_path: String = row.get(1)?;
160            Ok((id, source_path))
161        })?;
162
163        for row in rows {
164            let (chunk_id, source_path) = row?;
165            // Extract filename from source_path (which is the full path)
166            let filename = std::path::Path::new(&source_path)
167                .file_name()
168                .and_then(|n| n.to_str())
169                .unwrap_or(&source_path)
170                .to_string();
171            if let Some(&doc_idx) = path_to_doc_idx.get(&filename) {
172                doc_to_chunks.entry(doc_idx).or_default().push(chunk_id);
173            }
174        }
175    } // stmt dropped here, releasing borrow on conn
176
177    Ok((conn, doc_to_chunks))
178}
179
180/// Run the full evaluation against a golden set.
181pub fn run_eval(
182    pipeline: &SearchPipeline,
183    golden_set: &GoldenSet,
184    doc_to_chunks: &HashMap<usize, Vec<i64>>,
185) -> Result<EvalReport, EvalError> {
186    let mut per_query = Vec::with_capacity(golden_set.queries.len());
187
188    for query in &golden_set.queries {
189        let metrics = evaluate_query(pipeline, query, &golden_set.documents, doc_to_chunks)?;
190        per_query.push(metrics);
191    }
192
193    let total = per_query.len() as f32;
194    let answerable = per_query.iter().filter(|q| q.relevant_count > 0).count() as f32;
195    let unanswerable = total - answerable;
196
197    let macro_recall_at_10 = per_query.iter().map(|q| q.recall_at_10).sum::<f32>() / total;
198    let macro_recall_at_5 = per_query.iter().map(|q| q.recall_at_5).sum::<f32>() / total;
199    let macro_mrr = per_query.iter().map(|q| q.mrr).sum::<f32>() / total;
200    let macro_ndcg_at_10 = per_query.iter().map(|q| q.ndcg_at_10).sum::<f32>() / total;
201    let macro_citation_accuracy =
202        per_query.iter().map(|q| q.citation_accuracy).sum::<f32>() / total;
203    let macro_abstention_correctness = per_query
204        .iter()
205        .map(|q| q.abstention_correct as u8 as f32)
206        .sum::<f32>()
207        / total;
208
209    Ok(EvalReport {
210        per_query,
211        macro_recall_at_10,
212        macro_recall_at_5,
213        macro_mrr,
214        macro_ndcg_at_10,
215        macro_citation_accuracy,
216        macro_abstention_correctness,
217        total_queries: total as usize,
218        answerable_queries: answerable as usize,
219        unanswerable_queries: unanswerable as usize,
220    })
221}
222
223/// Evaluate a single query against the golden set.
224fn evaluate_query(
225    pipeline: &SearchPipeline,
226    query: &GoldenQuery,
227    documents: &[GoldenDocument],
228    doc_to_chunks: &HashMap<usize, Vec<i64>>,
229) -> Result<QueryMetrics, EvalError> {
230    // Run search
231    let roots: Vec<PathBuf> = vec![]; // Search all roots (empty = no filter)
232    let hits = pipeline.search(&query.text, &roots)?;
233
234    // Translate qrels from golden set document indices to actual database chunk IDs
235    let mut translated_qrels: HashMap<i64, u8> = HashMap::new();
236    for (&doc_idx, &rel) in &query.qrels {
237        if let Some(chunk_ids) = doc_to_chunks.get(&(doc_idx as usize)) {
238            for &chunk_id in chunk_ids {
239                translated_qrels.insert(chunk_id, rel);
240            }
241        }
242    }
243
244    // Compute relevance judgments using translated qrels
245    let relevant_ids: Vec<i64> = translated_qrels
246        .iter()
247        .filter(|(_, &rel)| rel > 0)
248        .map(|(&id, _)| id)
249        .collect();
250
251    let retrieved_ids: Vec<i64> = hits.iter().map(|h| h.source_id).collect();
252
253    // Recall@k
254    let recall_at_10 = recall_at_k(&retrieved_ids, &relevant_ids, 10);
255    let recall_at_5 = recall_at_k(&retrieved_ids, &relevant_ids, 5);
256
257    // MRR (Mean Reciprocal Rank)
258    let mrr = compute_mrr(&retrieved_ids, &relevant_ids);
259
260    // nDCG@10
261    let ndcg_at_10 = compute_ndcg_at_k(&retrieved_ids, &translated_qrels, 10);
262
263    // Citation accuracy: for each hit in the answer (confidence >= Medium),
264    // verify that the cited line range in the source document actually contains
265    // the answer text (or at least relevant content).
266    let citation_accuracy = compute_citation_accuracy(&hits, query, documents, doc_to_chunks)?;
267
268    // Abstention correctness: when confidence is Low, answer should be not_found
269    let abstention_correct = compute_abstention_correctness(&hits, query.answerable);
270
271    Ok(QueryMetrics {
272        query_id: query.id.clone(),
273        recall_at_10,
274        recall_at_5,
275        mrr,
276        ndcg_at_10,
277        citation_accuracy,
278        abstention_correct,
279        retrieved_count: hits.len(),
280        relevant_count: relevant_ids.len(),
281    })
282}
283
284/// Recall@k: fraction of relevant documents retrieved in top k.
285fn recall_at_k(retrieved: &[i64], relevant: &[i64], k: usize) -> f32 {
286    if relevant.is_empty() {
287        return 1.0; // Vacuously true: no relevant docs to find
288    }
289    let top_k: std::collections::HashSet<_> = retrieved.iter().take(k).collect();
290    let found = relevant.iter().filter(|id| top_k.contains(id)).count();
291    found as f32 / relevant.len() as f32
292}
293
294/// Mean Reciprocal Rank: 1/rank of first relevant document, 0 if none found.
295fn compute_mrr(retrieved: &[i64], relevant: &[i64]) -> f32 {
296    let relevant_set: std::collections::HashSet<_> = relevant.iter().collect();
297    for (rank, id) in retrieved.iter().enumerate() {
298        if relevant_set.contains(id) {
299            return 1.0 / (rank + 1) as f32;
300        }
301    }
302    0.0
303}
304
305/// nDCG@k: normalized Discounted Cumulative Gain.
306/// Gain: 0 for not relevant, 1 for relevant, 3 for highly relevant (2^rel - 1).
307fn compute_ndcg_at_k(retrieved: &[i64], qrels: &HashMap<i64, u8>, k: usize) -> f32 {
308    let mut dcg = 0.0f32;
309    for (i, &id) in retrieved.iter().take(k).enumerate() {
310        let rel = qrels.get(&id).copied().unwrap_or(0) as f32;
311        let gain = 2.0f32.powf(rel) - 1.0; // 0, 1, 3 for rel 0, 1, 2
312        dcg += gain / ((i + 2) as f32).log2();
313    }
314
315    // Ideal DCG: sort all relevant by relevance descending
316    let mut ideal_gains: Vec<f32> = qrels
317        .values()
318        .filter(|&&r| r > 0)
319        .map(|&r| 2.0f32.powf(r as f32) - 1.0)
320        .collect();
321    ideal_gains.sort_by(|a, b| b.partial_cmp(a).unwrap());
322
323    let mut idcg = 0.0f32;
324    for (i, gain) in ideal_gains.iter().take(k).enumerate() {
325        idcg += gain / ((i + 2) as f32).log2();
326    }
327
328    if idcg == 0.0 {
329        1.0 // No relevant docs, perfect score vacuously
330    } else {
331        dcg / idcg
332    }
333}
334
335/// Citation accuracy: verify that the top supported hit (used as the answer text
336/// per Answer::from_hits) has a cited line range that actually contains the
337/// answer text (or at least relevant content).
338fn compute_citation_accuracy(
339    hits: &[Hit],
340    query: &GoldenQuery,
341    documents: &[GoldenDocument],
342    doc_to_chunks: &HashMap<usize, Vec<i64>>,
343) -> Result<f32, EvalError> {
344    // Filter to hits with confidence >= Medium (these are the ones used in answers)
345    let supported_hits: Vec<&Hit> = hits
346        .iter()
347        .filter(|h| !matches!(h.confidence, crate::core::search::Confidence::Low))
348        .collect();
349
350    if supported_hits.is_empty() {
351        // No answer produced - this is correct for unanswerable queries
352        return Ok(if query.answerable { 0.0 } else { 1.0 });
353    }
354
355    // Answer::from_hits uses the FIRST supported hit as the answer text.
356    // We evaluate citation accuracy on this top hit only, since that's what
357    // the user sees as the answer. The additional citations are supplementary.
358    let top_hit = supported_hits[0];
359
360    // Build reverse mapping: chunk_id -> document index
361    let mut chunk_to_doc: HashMap<i64, usize> = HashMap::new();
362    for (doc_idx, chunk_ids) in doc_to_chunks {
363        for &chunk_id in chunk_ids {
364            chunk_to_doc.insert(chunk_id, *doc_idx);
365        }
366    }
367
368    if let Some(&doc_idx) = chunk_to_doc.get(&top_hit.source_id) {
369        if let Some(doc) = documents.get(doc_idx) {
370            // Extract the cited line range from the document
371            let lines: Vec<&str> = doc.content.lines().collect();
372            let start = (top_hit.line_start - 1).max(0) as usize;
373            let end = (top_hit.line_end as usize).min(lines.len());
374
375            if start < lines.len() && start <= end {
376                let cited_text: String = lines[start..end].join("\n");
377
378                // Check if cited text contains the expected snippet or query terms
379                let is_correct = if let Some(expected) = &query.expected_snippet {
380                    cited_text.to_lowercase().contains(&expected.to_lowercase())
381                } else {
382                    // Fallback: check if cited text shares significant terms with query
383                    let query_lower = query.text.to_lowercase();
384                    let query_terms: Vec<String> = query_lower
385                        .split_whitespace()
386                        .filter(|t| t.len() >= 3)
387                        .map(|s| s.to_string())
388                        .collect();
389                    let cited_lower = cited_text.to_lowercase();
390                    if query_terms.is_empty() {
391                        false
392                    } else {
393                        let matches = query_terms
394                            .iter()
395                            .filter(|t| cited_lower.contains(*t))
396                            .count();
397                        matches as f32 / query_terms.len() as f32 > 0.3
398                    }
399                };
400
401                return Ok(if is_correct { 1.0 } else { 0.0 });
402            }
403        }
404    }
405
406    Ok(0.0)
407}
408
409/// Abstention correctness: when all hits have Low confidence (or no hits),
410/// the system should abstain (not_found = true).
411/// For unanswerable queries, this is correct behavior.
412/// For answerable queries, this is a false negative.
413fn compute_abstention_correctness(hits: &[Hit], answerable: bool) -> bool {
414    let has_high_or_medium = hits
415        .iter()
416        .any(|h| !matches!(h.confidence, crate::core::search::Confidence::Low));
417
418    if answerable {
419        // Should find an answer (at least Medium confidence)
420        has_high_or_medium
421    } else {
422        // Should abstain (no High/Medium confidence hits)
423        !has_high_or_medium
424    }
425}
426
427impl GoldenSet {
428    /// Validate the golden set for consistency.
429    fn validate(&self) -> Result<(), EvalError> {
430        if self.queries.is_empty() {
431            return Err(EvalError::InvalidGoldenSet(
432                "No queries in golden set".into(),
433            ));
434        }
435        if self.documents.is_empty() {
436            return Err(EvalError::InvalidGoldenSet(
437                "No documents in golden set".into(),
438            ));
439        }
440
441        // Check that all qrels reference existing document IDs
442        let doc_ids: std::collections::HashSet<i64> = self
443            .documents
444            .iter()
445            .enumerate()
446            .map(|(i, _)| i as i64)
447            .collect();
448
449        for query in &self.queries {
450            for &chunk_id in query.qrels.keys() {
451                if !doc_ids.contains(&chunk_id) {
452                    return Err(EvalError::InvalidGoldenSet(format!(
453                        "Query {} references unknown chunk_id {}",
454                        query.id, chunk_id
455                    )));
456                }
457            }
458        }
459
460        Ok(())
461    }
462}
463
464impl EvalReport {
465    /// Save the report as JSON.
466    pub fn save_json<P: AsRef<Path>>(&self, path: P) -> Result<(), EvalError> {
467        let json = serde_json::to_string_pretty(self)?;
468        fs::write(path, json)?;
469        Ok(())
470    }
471
472    /// Check if the report passes CI gates.
473    /// Returns (passed, failure_reasons).
474    pub fn check_ci_gates(&self) -> (bool, Vec<String>) {
475        let mut reasons = Vec::new();
476
477        if self.macro_recall_at_10 < 0.75 {
478            reasons.push(format!("recall@10 {:.3} < 0.75", self.macro_recall_at_10));
479        }
480        if self.macro_citation_accuracy < 0.95 {
481            reasons.push(format!(
482                "citation_accuracy {:.3} < 0.95",
483                self.macro_citation_accuracy
484            ));
485        }
486
487        (reasons.is_empty(), reasons)
488    }
489}
490
491/// Create a SearchPipeline configured for evaluation with HashEmbedder.
492pub fn make_eval_pipeline(conn: Connection) -> SearchPipeline {
493    use crate::core::search::SharedEmbedder;
494    use std::sync::Arc;
495
496    struct HashEmbedder;
497    impl QueryEmbedder for HashEmbedder {
498        fn embed(&self, text: &str) -> anyhow::Result<Vec<f32>> {
499            let mut v = vec![0.0f32; crate::core::index::EMBEDDING_DIMS];
500            for token in text
501                .to_lowercase()
502                .split(|c: char| !c.is_alphanumeric())
503                .filter(|t| t.len() > 2)
504            {
505                let mut h = DefaultHasher::new();
506                token.hash(&mut h);
507                let idx = (h.finish() as usize) % v.len();
508                v[idx] += 1.0;
509            }
510            let norm = v.iter().map(|x| x * x).sum::<f32>().sqrt();
511            if norm > 0.0 {
512                for x in &mut v {
513                    *x /= norm;
514                }
515            }
516            Ok(v)
517        }
518
519        fn model_id(&self) -> &str {
520            "test-hash-embedder"
521        }
522    }
523
524    let embedder: SharedEmbedder = Arc::new(HashEmbedder);
525    SearchPipeline::new(conn, Some(embedder), None, SearchConfig::default())
526}
527
528#[cfg(test)]
529mod tests {
530    use super::*;
531    use crate::core::search::Confidence;
532
533    #[test]
534    fn test_recall_at_k() {
535        let retrieved = vec![1, 2, 3, 4, 5];
536        let relevant = vec![2, 4, 6];
537        assert!((recall_at_k(&retrieved, &relevant, 5) - 2.0 / 3.0).abs() < 0.001);
538        assert!((recall_at_k(&retrieved, &relevant, 3) - 1.0 / 3.0).abs() < 0.001);
539        assert_eq!(recall_at_k(&retrieved, &[], 5), 1.0);
540    }
541
542    #[test]
543    fn test_mrr() {
544        let retrieved = vec![1, 2, 3, 4, 5];
545        let relevant = vec![3, 5];
546        assert!((compute_mrr(&retrieved, &relevant) - 1.0 / 3.0).abs() < 0.001);
547
548        let relevant = vec![6, 7];
549        assert_eq!(compute_mrr(&retrieved, &relevant), 0.0);
550    }
551
552    #[test]
553    fn test_ndcg_at_k() {
554        let retrieved = vec![1, 2, 3];
555        let mut qrels = HashMap::new();
556        qrels.insert(1, 2); // highly relevant
557        qrels.insert(2, 1); // relevant
558        qrels.insert(3, 0); // not relevant
559
560        let ndcg = compute_ndcg_at_k(&retrieved, &qrels, 3);
561        // DCG = 3/log2(2) + 1/log2(3) + 0/log2(4) = 3 + 0.63 = 3.63
562        // IDCG = 3/log2(2) + 1/log2(3) = 3.63
563        // nDCG = 1.0
564        assert!((ndcg - 1.0).abs() < 0.01);
565
566        // Swap order: relevant first, then highly relevant
567        let retrieved = vec![2, 1, 3];
568        let ndcg = compute_ndcg_at_k(&retrieved, &qrels, 3);
569        // DCG = 1/log2(2) + 3/log2(3) = 1 + 1.89 = 2.89
570        // nDCG = 2.89/3.63 = 0.796
571        assert!((ndcg - 0.796).abs() < 0.01);
572    }
573
574    #[test]
575    fn test_abstention_correctness() {
576        // Answerable query with High confidence hit -> correct
577        let hits = vec![Hit {
578            source_id: 1,
579            path: "test.txt".into(),
580            source_path: "test.txt".into(),
581            page: 0,
582            line_start: 1,
583            line_end: 1,
584            anchor: None,
585            section: None,
586            snippet: "test".into(),
587            content: "test".into(),
588            score: 1.0,
589            why_matched: "test".into(),
590            confidence: Confidence::High,
591            engine: crate::core::search::Engine::Local,
592        }];
593        assert!(compute_abstention_correctness(&hits, true));
594
595        // Answerable query with only Low confidence -> incorrect (should have answered)
596        let hits = vec![Hit {
597            source_id: 1,
598            path: "test.txt".into(),
599            source_path: "test.txt".into(),
600            page: 0,
601            line_start: 1,
602            line_end: 1,
603            anchor: None,
604            section: None,
605            snippet: "test".into(),
606            content: "test".into(),
607            score: 0.1,
608            why_matched: "test".into(),
609            confidence: Confidence::Low,
610            engine: crate::core::search::Engine::Local,
611        }];
612        assert!(!compute_abstention_correctness(&hits, true));
613
614        // Unanswerable query with no hits -> correct
615        let hits = vec![];
616        assert!(compute_abstention_correctness(&hits, false));
617
618        // Unanswerable query with Low confidence only -> correct
619        let hits = vec![Hit {
620            source_id: 1,
621            path: "test.txt".into(),
622            source_path: "test.txt".into(),
623            page: 0,
624            line_start: 1,
625            line_end: 1,
626            anchor: None,
627            section: None,
628            snippet: "test".into(),
629            content: "test".into(),
630            score: 0.1,
631            why_matched: "test".into(),
632            confidence: Confidence::Low,
633            engine: crate::core::search::Engine::Local,
634        }];
635        assert!(compute_abstention_correctness(&hits, false));
636
637        // Unanswerable query with Medium confidence -> incorrect (should have abstained)
638        let hits = vec![Hit {
639            source_id: 1,
640            path: "test.txt".into(),
641            source_path: "test.txt".into(),
642            page: 0,
643            line_start: 1,
644            line_end: 1,
645            anchor: None,
646            section: None,
647            snippet: "test".into(),
648            content: "test".into(),
649            score: 0.5,
650            why_matched: "test".into(),
651            confidence: Confidence::Medium,
652            engine: crate::core::search::Engine::Local,
653        }];
654        assert!(!compute_abstention_correctness(&hits, false));
655    }
656}