Skip to main content

paper/core/
eval.rs

1//! Offline evaluation harness for retrieval quality.
2//!
3//! Uses a deterministic `HashEmbedder` (from `tests/common/mod.rs`) so CI never
4//! downloads models. Metrics: recall@k, MRR, nDCG@10, citation accuracy,
5//! abstention correctness.
6
7use crate::core::config::SearchConfig;
8use crate::core::index::Connection;
9use crate::core::search::{Hit, QueryEmbedder, SearchPipeline, SharedEmbedder};
10use serde::{Deserialize, Serialize};
11use std::collections::hash_map::DefaultHasher;
12use std::collections::HashMap;
13use std::fs;
14use std::hash::{Hash, Hasher};
15use std::path::{Path, PathBuf};
16use tempfile::TempDir;
17use thiserror::Error;
18
19/// Errors that can occur during evaluation.
20#[derive(Debug, Error)]
21pub enum EvalError {
22    #[error("I/O error: {0}")]
23    Io(#[from] std::io::Error),
24    #[error("JSON parse error: {0}")]
25    Json(#[from] serde_json::Error),
26    #[error("Search error: {0}")]
27    Search(#[from] anyhow::Error),
28    #[error("Database error: {0}")]
29    Database(#[from] rusqlite::Error),
30    #[error("Citation resolution failed: {0}")]
31    Citation(String),
32    #[error("Invalid golden set: {0}")]
33    InvalidGoldenSet(String),
34}
35
36/// A single query in the golden set.
37#[derive(Debug, Clone, Serialize, Deserialize)]
38pub struct GoldenQuery {
39    /// Unique query identifier.
40    pub id: String,
41    /// The query text.
42    pub text: String,
43    /// Relevance judgments: chunk_id -> relevance (0=not relevant, 1=relevant, 2=highly relevant).
44    pub qrels: HashMap<i64, u8>,
45    /// Whether this query should have an answer (false = unanswerable, tests abstention).
46    pub answerable: bool,
47    /// Expected answer text snippet (for citation accuracy verification).
48    #[serde(default)]
49    pub expected_snippet: Option<String>,
50}
51
52/// A document in the golden set corpus.
53#[derive(Debug, Clone, Serialize, Deserialize)]
54pub struct GoldenDocument {
55    /// Document identifier (used as chunk_id in the index).
56    pub id: i64,
57    /// Path to the document file.
58    pub path: String,
59    /// Full text content of the document.
60    pub content: String,
61    /// Optional page number (for paged formats).
62    #[serde(default)]
63    pub page: i64,
64}
65
66/// The complete golden evaluation set.
67#[derive(Debug, Clone, Serialize, Deserialize)]
68pub struct GoldenSet {
69    /// All queries to evaluate.
70    pub queries: Vec<GoldenQuery>,
71    /// All documents in the corpus.
72    pub documents: Vec<GoldenDocument>,
73}
74
75/// Metrics computed for a single query.
76#[derive(Debug, Clone, Serialize, Deserialize)]
77pub struct QueryMetrics {
78    pub query_id: String,
79    pub recall_at_10: f32,
80    pub recall_at_5: f32,
81    pub mrr: f32,
82    pub ndcg_at_10: f32,
83    pub citation_accuracy: f32,
84    pub abstention_correct: bool,
85    pub retrieved_count: usize,
86    pub relevant_count: usize,
87}
88
89/// Aggregated evaluation report.
90#[derive(Debug, Clone, Serialize, Deserialize)]
91pub struct EvalReport {
92    pub per_query: Vec<QueryMetrics>,
93    pub macro_recall_at_10: f32,
94    pub macro_recall_at_5: f32,
95    pub macro_mrr: f32,
96    pub macro_ndcg_at_10: f32,
97    pub macro_citation_accuracy: f32,
98    pub macro_abstention_correctness: f32,
99    pub total_queries: usize,
100    pub answerable_queries: usize,
101    pub unanswerable_queries: usize,
102}
103
104/// Load a golden set from a JSON file.
105pub fn load_golden_set<P: AsRef<Path>>(path: P) -> Result<GoldenSet, EvalError> {
106    let content = fs::read_to_string(path)?;
107    let set: GoldenSet = serde_json::from_str(&content)?;
108    set.validate()?;
109    Ok(set)
110}
111
112/// Build an in-memory index from the golden set documents.
113/// Returns the connection and a mapping from golden set document index to database chunk IDs.
114pub fn build_golden_index(
115    golden_set: &GoldenSet,
116    embedder: SharedEmbedder,
117) -> Result<(Connection, HashMap<usize, Vec<i64>>), EvalError> {
118    use crate::core::index::{init_db, Indexer};
119
120    // Create a temporary directory for the database and document files.
121    // `TempDir` is dropped at the end of this function, cleaning up the index
122    // and document files instead of leaving them under `env::temp_dir()`
123    // (M18). The returned `Connection` holds an open fd to the db file, which
124    // remains valid on Unix after the directory entry is removed.
125    let temp_dir = TempDir::new()?;
126    let db_path = temp_dir.path().join("eval.db");
127    let docs_dir = temp_dir.path().join("docs");
128    fs::create_dir_all(&docs_dir)?;
129
130    // Write all documents to the filesystem
131    for doc in &golden_set.documents {
132        let doc_path = docs_dir.join(&doc.path);
133        fs::create_dir_all(doc_path.parent().unwrap())?;
134        fs::write(&doc_path, &doc.content)?;
135    }
136
137    // Open connection and index
138    {
139        let conn = Connection::open(&db_path)?;
140        init_db(&conn)?;
141        let indexer = Indexer::new(conn, Some(embedder))?;
142        indexer.index_roots(&[docs_dir])?;
143    }
144    // Re-open the connection after indexing (Indexer takes ownership)
145    let conn = Connection::open(&db_path)?;
146
147    // Build mapping from golden set document index to chunk IDs
148    // by querying the database for source_path
149    let mut doc_to_chunks: HashMap<usize, Vec<i64>> = HashMap::new();
150
151    // Build a map from filename to golden set document index
152    let path_to_doc_idx: HashMap<String, usize> = golden_set
153        .documents
154        .iter()
155        .enumerate()
156        .map(|(i, d)| (d.path.clone(), i))
157        .collect();
158
159    {
160        let mut stmt = conn.inner().prepare("SELECT id, source_path FROM chunks")?;
161        let rows = stmt.query_map([], |row| {
162            let id: i64 = row.get(0)?;
163            let source_path: String = row.get(1)?;
164            Ok((id, source_path))
165        })?;
166
167        for row in rows {
168            let (chunk_id, source_path) = row?;
169            // Extract filename from source_path (which is the full path)
170            let filename = std::path::Path::new(&source_path)
171                .file_name()
172                .and_then(|n| n.to_str())
173                .unwrap_or(&source_path)
174                .to_string();
175            if let Some(&doc_idx) = path_to_doc_idx.get(&filename) {
176                doc_to_chunks.entry(doc_idx).or_default().push(chunk_id);
177            }
178        }
179    } // stmt dropped here, releasing borrow on conn
180
181    Ok((conn, doc_to_chunks))
182}
183
184/// Run the full evaluation against a golden set.
185pub fn run_eval(
186    pipeline: &SearchPipeline,
187    golden_set: &GoldenSet,
188    doc_to_chunks: &HashMap<usize, Vec<i64>>,
189) -> Result<EvalReport, EvalError> {
190    let mut per_query = Vec::with_capacity(golden_set.queries.len());
191
192    for query in &golden_set.queries {
193        let metrics = evaluate_query(pipeline, query, &golden_set.documents, doc_to_chunks)?;
194        per_query.push(metrics);
195    }
196
197    let total = per_query.len() as f32;
198    let answerable = per_query.iter().filter(|q| q.relevant_count > 0).count() as f32;
199    let unanswerable = total - answerable;
200
201    let macro_recall_at_10 = per_query.iter().map(|q| q.recall_at_10).sum::<f32>() / total;
202    let macro_recall_at_5 = per_query.iter().map(|q| q.recall_at_5).sum::<f32>() / total;
203    let macro_mrr = per_query.iter().map(|q| q.mrr).sum::<f32>() / total;
204    let macro_ndcg_at_10 = per_query.iter().map(|q| q.ndcg_at_10).sum::<f32>() / total;
205    let macro_citation_accuracy =
206        per_query.iter().map(|q| q.citation_accuracy).sum::<f32>() / total;
207    let macro_abstention_correctness = per_query
208        .iter()
209        .map(|q| q.abstention_correct as u8 as f32)
210        .sum::<f32>()
211        / total;
212
213    Ok(EvalReport {
214        per_query,
215        macro_recall_at_10,
216        macro_recall_at_5,
217        macro_mrr,
218        macro_ndcg_at_10,
219        macro_citation_accuracy,
220        macro_abstention_correctness,
221        total_queries: total as usize,
222        answerable_queries: answerable as usize,
223        unanswerable_queries: unanswerable as usize,
224    })
225}
226
227/// Evaluate a single query against the golden set.
228fn evaluate_query(
229    pipeline: &SearchPipeline,
230    query: &GoldenQuery,
231    documents: &[GoldenDocument],
232    doc_to_chunks: &HashMap<usize, Vec<i64>>,
233) -> Result<QueryMetrics, EvalError> {
234    // Run search
235    let roots: Vec<PathBuf> = vec![]; // Search all roots (empty = no filter)
236    let hits = pipeline.search(&query.text, &roots)?;
237
238    // Translate qrels from golden set document indices to actual database chunk IDs
239    let mut translated_qrels: HashMap<i64, u8> = HashMap::new();
240    for (&doc_idx, &rel) in &query.qrels {
241        if let Some(chunk_ids) = doc_to_chunks.get(&(doc_idx as usize)) {
242            for &chunk_id in chunk_ids {
243                translated_qrels.insert(chunk_id, rel);
244            }
245        }
246    }
247
248    // Compute relevance judgments using translated qrels
249    let relevant_ids: Vec<i64> = translated_qrels
250        .iter()
251        .filter(|(_, &rel)| rel > 0)
252        .map(|(&id, _)| id)
253        .collect();
254
255    let retrieved_ids: Vec<i64> = hits.iter().map(|h| h.source_id).collect();
256
257    // Recall@k
258    let recall_at_10 = recall_at_k(&retrieved_ids, &relevant_ids, 10);
259    let recall_at_5 = recall_at_k(&retrieved_ids, &relevant_ids, 5);
260
261    // MRR (Mean Reciprocal Rank)
262    let mrr = compute_mrr(&retrieved_ids, &relevant_ids);
263
264    // nDCG@10
265    let ndcg_at_10 = compute_ndcg_at_k(&retrieved_ids, &translated_qrels, 10);
266
267    // Citation accuracy: for each hit in the answer (confidence >= Medium),
268    // verify that the cited line range in the source document actually contains
269    // the answer text (or at least relevant content).
270    let citation_accuracy = compute_citation_accuracy(&hits, query, documents, doc_to_chunks)?;
271
272    // Abstention correctness: when confidence is Low, answer should be not_found
273    let abstention_correct = compute_abstention_correctness(&hits, query.answerable);
274
275    Ok(QueryMetrics {
276        query_id: query.id.clone(),
277        recall_at_10,
278        recall_at_5,
279        mrr,
280        ndcg_at_10,
281        citation_accuracy,
282        abstention_correct,
283        retrieved_count: hits.len(),
284        relevant_count: relevant_ids.len(),
285    })
286}
287
288/// Recall@k: fraction of relevant documents retrieved in top k.
289fn recall_at_k(retrieved: &[i64], relevant: &[i64], k: usize) -> f32 {
290    if relevant.is_empty() {
291        return 1.0; // Vacuously true: no relevant docs to find
292    }
293    let top_k: std::collections::HashSet<_> = retrieved.iter().take(k).collect();
294    let found = relevant.iter().filter(|id| top_k.contains(id)).count();
295    found as f32 / relevant.len() as f32
296}
297
298/// Mean Reciprocal Rank: 1/rank of first relevant document, 0 if none found.
299fn compute_mrr(retrieved: &[i64], relevant: &[i64]) -> f32 {
300    let relevant_set: std::collections::HashSet<_> = relevant.iter().collect();
301    for (rank, id) in retrieved.iter().enumerate() {
302        if relevant_set.contains(id) {
303            return 1.0 / (rank + 1) as f32;
304        }
305    }
306    0.0
307}
308
309/// nDCG@k: normalized Discounted Cumulative Gain.
310/// Gain: 0 for not relevant, 1 for relevant, 3 for highly relevant (2^rel - 1).
311fn compute_ndcg_at_k(retrieved: &[i64], qrels: &HashMap<i64, u8>, k: usize) -> f32 {
312    let mut dcg = 0.0f32;
313    for (i, &id) in retrieved.iter().take(k).enumerate() {
314        let rel = qrels.get(&id).copied().unwrap_or(0) as f32;
315        let gain = 2.0f32.powf(rel) - 1.0; // 0, 1, 3 for rel 0, 1, 2
316        dcg += gain / ((i + 2) as f32).log2();
317    }
318
319    // Ideal DCG: sort all relevant by relevance descending
320    let mut ideal_gains: Vec<f32> = qrels
321        .values()
322        .filter(|&&r| r > 0)
323        .map(|&r| 2.0f32.powf(r as f32) - 1.0)
324        .collect();
325    ideal_gains.sort_by(|a, b| b.partial_cmp(a).unwrap());
326
327    let mut idcg = 0.0f32;
328    for (i, gain) in ideal_gains.iter().take(k).enumerate() {
329        idcg += gain / ((i + 2) as f32).log2();
330    }
331
332    if idcg == 0.0 {
333        1.0 // No relevant docs, perfect score vacuously
334    } else {
335        dcg / idcg
336    }
337}
338
339/// Citation accuracy: verify that the top supported hit (used as the answer text
340/// per Answer::from_hits) has a cited line range that actually contains the
341/// answer text (or at least relevant content).
342fn compute_citation_accuracy(
343    hits: &[Hit],
344    query: &GoldenQuery,
345    documents: &[GoldenDocument],
346    doc_to_chunks: &HashMap<usize, Vec<i64>>,
347) -> Result<f32, EvalError> {
348    // Filter to hits with confidence >= Medium (these are the ones used in answers)
349    let supported_hits: Vec<&Hit> = hits
350        .iter()
351        .filter(|h| !matches!(h.confidence, crate::core::search::Confidence::Low))
352        .collect();
353
354    if supported_hits.is_empty() {
355        // No answer produced - this is correct for unanswerable queries
356        return Ok(if query.answerable { 0.0 } else { 1.0 });
357    }
358
359    // Answer::from_hits uses the FIRST supported hit as the answer text.
360    // We evaluate citation accuracy on this top hit only, since that's what
361    // the user sees as the answer. The additional citations are supplementary.
362    let top_hit = supported_hits[0];
363
364    // Build reverse mapping: chunk_id -> document index
365    let mut chunk_to_doc: HashMap<i64, usize> = HashMap::new();
366    for (doc_idx, chunk_ids) in doc_to_chunks {
367        for &chunk_id in chunk_ids {
368            chunk_to_doc.insert(chunk_id, *doc_idx);
369        }
370    }
371
372    if let Some(&doc_idx) = chunk_to_doc.get(&top_hit.source_id) {
373        if let Some(doc) = documents.get(doc_idx) {
374            // Extract the cited line range from the document
375            let lines: Vec<&str> = doc.content.lines().collect();
376            let start = (top_hit.line_start - 1).max(0) as usize;
377            let end = (top_hit.line_end as usize).min(lines.len());
378
379            if start < lines.len() && start <= end {
380                let cited_text: String = lines[start..end].join("\n");
381
382                // Check if cited text contains the expected snippet or query terms
383                let is_correct = if let Some(expected) = &query.expected_snippet {
384                    cited_text.to_lowercase().contains(&expected.to_lowercase())
385                } else {
386                    // Fallback: check if cited text shares significant terms with query
387                    let query_lower = query.text.to_lowercase();
388                    let query_terms: Vec<String> = query_lower
389                        .split_whitespace()
390                        .filter(|t| t.len() >= 3)
391                        .map(|s| s.to_string())
392                        .collect();
393                    let cited_lower = cited_text.to_lowercase();
394                    if query_terms.is_empty() {
395                        false
396                    } else {
397                        let matches = query_terms
398                            .iter()
399                            .filter(|t| cited_lower.contains(*t))
400                            .count();
401                        matches as f32 / query_terms.len() as f32 > 0.3
402                    }
403                };
404
405                return Ok(if is_correct { 1.0 } else { 0.0 });
406            }
407        }
408    }
409
410    Ok(0.0)
411}
412
413/// Abstention correctness: when all hits have Low confidence (or no hits),
414/// the system should abstain (not_found = true).
415/// For unanswerable queries, this is correct behavior.
416/// For answerable queries, this is a false negative.
417fn compute_abstention_correctness(hits: &[Hit], answerable: bool) -> bool {
418    let has_high_or_medium = hits
419        .iter()
420        .any(|h| !matches!(h.confidence, crate::core::search::Confidence::Low));
421
422    if answerable {
423        // Should find an answer (at least Medium confidence)
424        has_high_or_medium
425    } else {
426        // Should abstain (no High/Medium confidence hits)
427        !has_high_or_medium
428    }
429}
430
431impl GoldenSet {
432    /// Validate the golden set for consistency.
433    fn validate(&self) -> Result<(), EvalError> {
434        if self.queries.is_empty() {
435            return Err(EvalError::InvalidGoldenSet(
436                "No queries in golden set".into(),
437            ));
438        }
439        if self.documents.is_empty() {
440            return Err(EvalError::InvalidGoldenSet(
441                "No documents in golden set".into(),
442            ));
443        }
444
445        // Check that all qrels reference existing document IDs
446        let doc_ids: std::collections::HashSet<i64> = self
447            .documents
448            .iter()
449            .enumerate()
450            .map(|(i, _)| i as i64)
451            .collect();
452
453        for query in &self.queries {
454            for &chunk_id in query.qrels.keys() {
455                if !doc_ids.contains(&chunk_id) {
456                    return Err(EvalError::InvalidGoldenSet(format!(
457                        "Query {} references unknown chunk_id {}",
458                        query.id, chunk_id
459                    )));
460                }
461            }
462        }
463
464        Ok(())
465    }
466}
467
468impl EvalReport {
469    /// Save the report as JSON.
470    pub fn save_json<P: AsRef<Path>>(&self, path: P) -> Result<(), EvalError> {
471        let json = serde_json::to_string_pretty(self)?;
472        fs::write(path, json)?;
473        Ok(())
474    }
475
476    /// Check if the report passes CI gates.
477    /// Returns (passed, failure_reasons).
478    pub fn check_ci_gates(&self) -> (bool, Vec<String>) {
479        let mut reasons = Vec::new();
480
481        if self.macro_recall_at_10 < 0.75 {
482            reasons.push(format!("recall@10 {:.3} < 0.75", self.macro_recall_at_10));
483        }
484        if self.macro_citation_accuracy < 0.95 {
485            reasons.push(format!(
486                "citation_accuracy {:.3} < 0.95",
487                self.macro_citation_accuracy
488            ));
489        }
490
491        (reasons.is_empty(), reasons)
492    }
493}
494
495/// Create a SearchPipeline configured for evaluation with HashEmbedder.
496pub fn make_eval_pipeline(conn: Connection) -> SearchPipeline {
497    use crate::core::search::SharedEmbedder;
498    use std::sync::Arc;
499
500    struct HashEmbedder;
501    impl QueryEmbedder for HashEmbedder {
502        fn embed(&self, text: &str) -> anyhow::Result<Vec<f32>> {
503            let mut v = vec![0.0f32; crate::core::index::EMBEDDING_DIMS];
504            for token in text
505                .to_lowercase()
506                .split(|c: char| !c.is_alphanumeric())
507                .filter(|t| t.len() > 2)
508            {
509                let mut h = DefaultHasher::new();
510                token.hash(&mut h);
511                let idx = (h.finish() as usize) % v.len();
512                v[idx] += 1.0;
513            }
514            let norm = v.iter().map(|x| x * x).sum::<f32>().sqrt();
515            if norm > 0.0 {
516                for x in &mut v {
517                    *x /= norm;
518                }
519            }
520            Ok(v)
521        }
522
523        fn model_id(&self) -> &str {
524            "test-hash-embedder"
525        }
526    }
527
528    let embedder: SharedEmbedder = Arc::new(HashEmbedder);
529    SearchPipeline::new(conn, Some(embedder), None, SearchConfig::default())
530}
531
532#[cfg(test)]
533mod tests {
534    use super::*;
535    use crate::core::search::Confidence;
536
537    #[test]
538    fn test_recall_at_k() {
539        let retrieved = vec![1, 2, 3, 4, 5];
540        let relevant = vec![2, 4, 6];
541        assert!((recall_at_k(&retrieved, &relevant, 5) - 2.0 / 3.0).abs() < 0.001);
542        assert!((recall_at_k(&retrieved, &relevant, 3) - 1.0 / 3.0).abs() < 0.001);
543        assert_eq!(recall_at_k(&retrieved, &[], 5), 1.0);
544    }
545
546    #[test]
547    fn test_mrr() {
548        let retrieved = vec![1, 2, 3, 4, 5];
549        let relevant = vec![3, 5];
550        assert!((compute_mrr(&retrieved, &relevant) - 1.0 / 3.0).abs() < 0.001);
551
552        let relevant = vec![6, 7];
553        assert_eq!(compute_mrr(&retrieved, &relevant), 0.0);
554    }
555
556    #[test]
557    fn test_ndcg_at_k() {
558        let retrieved = vec![1, 2, 3];
559        let mut qrels = HashMap::new();
560        qrels.insert(1, 2); // highly relevant
561        qrels.insert(2, 1); // relevant
562        qrels.insert(3, 0); // not relevant
563
564        let ndcg = compute_ndcg_at_k(&retrieved, &qrels, 3);
565        // DCG = 3/log2(2) + 1/log2(3) + 0/log2(4) = 3 + 0.63 = 3.63
566        // IDCG = 3/log2(2) + 1/log2(3) = 3.63
567        // nDCG = 1.0
568        assert!((ndcg - 1.0).abs() < 0.01);
569
570        // Swap order: relevant first, then highly relevant
571        let retrieved = vec![2, 1, 3];
572        let ndcg = compute_ndcg_at_k(&retrieved, &qrels, 3);
573        // DCG = 1/log2(2) + 3/log2(3) = 1 + 1.89 = 2.89
574        // nDCG = 2.89/3.63 = 0.796
575        assert!((ndcg - 0.796).abs() < 0.01);
576    }
577
578    #[test]
579    fn test_abstention_correctness() {
580        // Answerable query with High confidence hit -> correct
581        let hits = vec![Hit {
582            source_id: 1,
583            path: "test.txt".into(),
584            source_path: "test.txt".into(),
585            page: 0,
586            line_start: 1,
587            line_end: 1,
588            anchor: None,
589            section: None,
590            snippet: "test".into(),
591            content: "test".into(),
592            score: 1.0,
593            why_matched: "test".into(),
594            confidence: Confidence::High,
595            engine: crate::core::search::Engine::Local,
596        }];
597        assert!(compute_abstention_correctness(&hits, true));
598
599        // Answerable query with only Low confidence -> incorrect (should have answered)
600        let hits = vec![Hit {
601            source_id: 1,
602            path: "test.txt".into(),
603            source_path: "test.txt".into(),
604            page: 0,
605            line_start: 1,
606            line_end: 1,
607            anchor: None,
608            section: None,
609            snippet: "test".into(),
610            content: "test".into(),
611            score: 0.1,
612            why_matched: "test".into(),
613            confidence: Confidence::Low,
614            engine: crate::core::search::Engine::Local,
615        }];
616        assert!(!compute_abstention_correctness(&hits, true));
617
618        // Unanswerable query with no hits -> correct
619        let hits = vec![];
620        assert!(compute_abstention_correctness(&hits, false));
621
622        // Unanswerable query with Low confidence only -> correct
623        let hits = vec![Hit {
624            source_id: 1,
625            path: "test.txt".into(),
626            source_path: "test.txt".into(),
627            page: 0,
628            line_start: 1,
629            line_end: 1,
630            anchor: None,
631            section: None,
632            snippet: "test".into(),
633            content: "test".into(),
634            score: 0.1,
635            why_matched: "test".into(),
636            confidence: Confidence::Low,
637            engine: crate::core::search::Engine::Local,
638        }];
639        assert!(compute_abstention_correctness(&hits, false));
640
641        // Unanswerable query with Medium confidence -> incorrect (should have abstained)
642        let hits = vec![Hit {
643            source_id: 1,
644            path: "test.txt".into(),
645            source_path: "test.txt".into(),
646            page: 0,
647            line_start: 1,
648            line_end: 1,
649            anchor: None,
650            section: None,
651            snippet: "test".into(),
652            content: "test".into(),
653            score: 0.5,
654            why_matched: "test".into(),
655            confidence: Confidence::Medium,
656            engine: crate::core::search::Engine::Local,
657        }];
658        assert!(!compute_abstention_correctness(&hits, false));
659    }
660}