Skip to main content

second_brain_api/eval/
mod.rs

1pub mod bootstrap;
2pub mod caching_store;
3pub mod metrics;
4
5use std::collections::HashSet;
6use std::io::Write;
7use std::path::Path;
8use std::sync::Mutex;
9use std::sync::atomic::{AtomicUsize, Ordering};
10use std::thread;
11
12use anyhow::Result;
13use serde::{Deserialize, Serialize};
14use uuid::Uuid;
15
16use second_brain_core::embedding::Embedder;
17use second_brain_core::index::Indexes;
18use second_brain_core::query::{QueryEngine, QueryFilters, QueryRequest};
19use second_brain_core::retrieval::{RerankScorer, Reranker, RetrievalTunables, Retriever};
20use second_brain_core::sqlite_store::SqliteStore;
21use second_brain_core::store::{Store, StoreAdmin};
22
23/// Index-backed pieces the hybrid engines need beside the (possibly caching)
24/// Store: candidates come from the real indexes over the snapshot SqliteStore,
25/// ranking runs against the caching wrapper.
26pub struct HybridContext<'a> {
27    pub sqlite: &'a SqliteStore,
28    pub indexes: &'a Indexes,
29    pub reranker: Option<&'a Reranker>,
30    pub tunables: RetrievalTunables,
31}
32
33#[derive(Debug, Clone, Deserialize, Serialize)]
34pub struct EvalQuery {
35    pub query_id: String,
36    pub query: String,
37    pub query_variant: String,
38    pub seed_memory_id: Uuid,
39    pub memory_type: String,
40    pub relevant_memory_ids: Vec<Uuid>,
41    #[serde(default)]
42    pub note: Option<String>,
43    #[serde(default)]
44    pub tags: Vec<String>,
45    /// Project the gold memory is filed under.
46    #[serde(default)]
47    pub source_project: Option<String>,
48}
49
50#[derive(Debug, Clone, Serialize, Deserialize)]
51pub struct QueryRecord {
52    pub query_id: String,
53    pub use_prefix: bool,
54    pub ranked_ids: Vec<Uuid>,
55    pub scores: Vec<f32>,
56    pub first_relevant_rank: Option<usize>,
57    pub gold_raw_rank: Option<usize>,
58    pub gold_raw_similarity: Option<f32>,
59    /// Engine-call wall time only. Under run_arm_parallel this includes
60    /// rerank-mutex wait, so solo latency must come from a serial run.
61    #[serde(default)]
62    pub latency_ms: Option<u64>,
63}
64
65#[derive(Debug, Clone, Serialize, Deserialize)]
66pub struct ArmMetrics {
67    pub recall_at_1: f32,
68    pub recall_at_3: f32,
69    pub recall_at_5: f32,
70    pub mrr: f32,
71    pub precision_at_5: f32,
72}
73
74#[derive(Debug, Clone, Serialize, Deserialize)]
75pub struct AggregateReport {
76    pub bare: ArmMetrics,
77    pub prefixed: ArmMetrics,
78    pub delta_recall_at_3_ci: (f32, f32),
79    pub delta_mrr_ci: (f32, f32),
80    pub gated_out_rate_bare: f32,
81    pub gated_out_rate_prefixed: f32,
82}
83
84#[derive(Debug, Clone, Serialize, Deserialize)]
85pub struct GatePoint {
86    pub threshold: f32,
87    pub recall_at_1: f32,
88    pub recall_at_3: f32,
89    pub recall_at_5: f32,
90    pub precision_proxy: f32,
91}
92
93#[derive(Debug, Clone, Serialize, Deserialize)]
94pub struct GateSweepReport {
95    pub frontier: Vec<GatePoint>,
96    pub baseline_threshold: f32,
97    pub chosen_threshold: f32,
98    pub chosen_beats_baseline: bool,
99}
100
101#[derive(Debug, Clone, Serialize, Deserialize)]
102pub struct CorpusEntry {
103    pub id: Uuid,
104    pub content: String,
105    pub memory_type: String,
106    pub created_at: String,
107    pub project_path: Option<String>,
108}
109
110const BASELINE_THRESHOLD: f32 = 0.59;
111
112pub fn load_eval_set(path: &Path) -> Result<Vec<EvalQuery>> {
113    let text = std::fs::read_to_string(path)?;
114    let mut out = Vec::new();
115    for line in text.lines() {
116        let trimmed = line.trim();
117        if trimmed.is_empty() {
118            continue;
119        }
120        out.push(serde_json::from_str(trimmed)?);
121    }
122    Ok(out)
123}
124
125pub fn run_arm(
126    store: &SqliteStore,
127    embedder: &Embedder,
128    queries: &[EvalQuery],
129    use_prefix: bool,
130    limit: usize,
131) -> Result<Vec<QueryRecord>> {
132    let engine = QueryEngine::new(store);
133    let mut records = Vec::with_capacity(queries.len());
134
135    for q in queries {
136        let embedding = if use_prefix {
137            embedder.embed_query(&q.query)?
138        } else {
139            embedder.embed(&q.query)?
140        };
141
142        let relevant: HashSet<Uuid> = q.relevant_memory_ids.iter().copied().collect();
143
144        let request = QueryRequest {
145            text: q.query.clone(),
146            embedding: embedding.clone(),
147            limit,
148            filters: QueryFilters {
149                entity_names: crate::link_query_entities(&q.query),
150                ..QueryFilters::default()
151            },
152        };
153        let started = std::time::Instant::now();
154        let results = engine.recall(&request)?;
155        let latency_ms = started.elapsed().as_millis() as u64;
156
157        let ranked_ids: Vec<Uuid> = results.iter().map(|r| r.memory.id).collect();
158        let scores: Vec<f32> = results.iter().map(|r| r.score).collect();
159        let first_relevant_rank = ranked_ids
160            .iter()
161            .position(|id| relevant.contains(id))
162            .map(|idx| idx + 1);
163
164        let raw = store.vector_search(&embedding, limit * 3)?;
165        let mut gold_raw_rank = None;
166        let mut gold_raw_similarity = None;
167        for (idx, (mem, sim)) in raw.iter().enumerate() {
168            if relevant.contains(&mem.id) {
169                gold_raw_rank = Some(idx + 1);
170                gold_raw_similarity = Some(*sim);
171                break;
172            }
173        }
174
175        records.push(QueryRecord {
176            query_id: q.query_id.clone(),
177            use_prefix,
178            ranked_ids,
179            scores,
180            first_relevant_rank,
181            gold_raw_rank,
182            gold_raw_similarity,
183            latency_ms: Some(latency_ms),
184        });
185    }
186
187    Ok(records)
188}
189
190pub struct EmbeddedQuery {
191    pub query: EvalQuery,
192    pub relevant: HashSet<Uuid>,
193    pub bare_embedding: Vec<f32>,
194    pub prefixed_embedding: Vec<f32>,
195}
196
197// The Embedder wraps the model in a Mutex, so embedding cannot run in parallel.
198// We pre-compute every embedding serially here, in batches, to keep the model
199// out of the parallel recall section entirely.
200pub fn embed_all_queries(embedder: &Embedder, queries: &[EvalQuery]) -> Result<Vec<EmbeddedQuery>> {
201    let bare_texts: Vec<&str> = queries.iter().map(|q| q.query.as_str()).collect();
202    let bare = embedder.embed_batch(&bare_texts)?;
203
204    let prefixed_owned: Vec<String> = queries
205        .iter()
206        .map(|q| second_brain_core::embedding::query_prompt(&q.query))
207        .collect();
208    let prefixed_texts: Vec<&str> = prefixed_owned.iter().map(|s| s.as_str()).collect();
209    let prefixed = embedder.embed_batch(&prefixed_texts)?;
210
211    let mut out = Vec::with_capacity(queries.len());
212    for (i, q) in queries.iter().enumerate() {
213        out.push(EmbeddedQuery {
214            query: q.clone(),
215            relevant: q.relevant_memory_ids.iter().copied().collect(),
216            bare_embedding: bare[i].clone(),
217            prefixed_embedding: prefixed[i].clone(),
218        });
219    }
220    Ok(out)
221}
222
223fn record_for<S: Store + Sync>(
224    embedded: &EmbeddedQuery,
225    store: &S,
226    use_prefix: bool,
227    limit: usize,
228    hybrid: Option<&HybridContext<'_>>,
229) -> Result<QueryRecord> {
230    let embedding = if use_prefix {
231        &embedded.prefixed_embedding
232    } else {
233        &embedded.bare_embedding
234    };
235
236    let request = QueryRequest {
237        text: embedded.query.query.clone(),
238        embedding: embedding.clone(),
239        limit,
240        filters: QueryFilters {
241            entity_names: crate::link_query_entities(&embedded.query.query),
242            ..QueryFilters::default()
243        },
244    };
245    let started = std::time::Instant::now();
246    let results = match hybrid {
247        None => QueryEngine::new(store).recall(&request)?,
248        Some(ctx) => {
249            let retriever = Retriever {
250                tunables: ctx.tunables,
251            };
252            let pool = retriever.candidates(ctx.indexes, ctx.sqlite, &request)?;
253            retriever.rank(
254                store,
255                &request,
256                &pool,
257                ctx.reranker.map(|r| r as &dyn RerankScorer),
258            )?
259        }
260    };
261    let latency_ms = started.elapsed().as_millis() as u64;
262
263    let ranked_ids: Vec<Uuid> = results.iter().map(|r| r.memory.id).collect();
264    let scores: Vec<f32> = results.iter().map(|r| r.score).collect();
265    let first_relevant_rank = ranked_ids
266        .iter()
267        .position(|id| embedded.relevant.contains(id))
268        .map(|idx| idx + 1);
269
270    let raw = store.vector_search(embedding, limit * 3)?;
271    let mut gold_raw_rank = None;
272    let mut gold_raw_similarity = None;
273    for (idx, (mem, sim)) in raw.iter().enumerate() {
274        if embedded.relevant.contains(&mem.id) {
275            gold_raw_rank = Some(idx + 1);
276            gold_raw_similarity = Some(*sim);
277            break;
278        }
279    }
280
281    Ok(QueryRecord {
282        query_id: embedded.query.query_id.clone(),
283        use_prefix,
284        ranked_ids,
285        scores,
286        first_relevant_rank,
287        gold_raw_rank,
288        gold_raw_similarity,
289        latency_ms: Some(latency_ms),
290    })
291}
292
293// SqliteStore serializes every store call behind its connection mutex, so
294// sharing &store across scoped threads is safe; concurrent reads queue rather
295// than race. The Embedder is absent here on purpose: embeddings are
296// pre-computed in embed_all_queries.
297pub fn run_arm_parallel<S: Store + Sync>(
298    store: &S,
299    embedded: &[EmbeddedQuery],
300    use_prefix: bool,
301    limit: usize,
302    hybrid: Option<&HybridContext<'_>>,
303) -> Result<Vec<QueryRecord>> {
304    let total = embedded.len();
305    if total == 0 {
306        return Ok(Vec::new());
307    }
308
309    let workers = thread::available_parallelism()
310        .map(|n| n.get().saturating_sub(1).max(1))
311        .unwrap_or(1);
312    let chunk_size = total.div_ceil(workers);
313
314    let done = AtomicUsize::new(0);
315    let collected: Mutex<Vec<(usize, QueryRecord)>> = Mutex::new(Vec::with_capacity(total));
316    let error: Mutex<Option<anyhow::Error>> = Mutex::new(None);
317
318    thread::scope(|scope| {
319        for chunk in 0..workers {
320            let start = chunk * chunk_size;
321            if start >= total {
322                break;
323            }
324            let end = (start + chunk_size).min(total);
325            let done = &done;
326            let collected = &collected;
327            let error = &error;
328            scope.spawn(move || {
329                let mut local: Vec<(usize, QueryRecord)> = Vec::with_capacity(end - start);
330                for (offset, eq) in embedded[start..end].iter().enumerate() {
331                    if error.lock().unwrap().is_some() {
332                        return;
333                    }
334                    match record_for(eq, store, use_prefix, limit, hybrid) {
335                        Ok(rec) => local.push((start + offset, rec)),
336                        Err(e) => {
337                            *error.lock().unwrap() = Some(e);
338                            return;
339                        }
340                    }
341                    // eprintln progress because long runs were previously invisible.
342                    let n = done.fetch_add(1, Ordering::Relaxed) + 1;
343                    if n.is_multiple_of(25) || n == total {
344                        eprintln!("  {n}/{total} queries");
345                    }
346                }
347                collected.lock().unwrap().extend(local);
348            });
349        }
350    });
351
352    if let Some(e) = error.into_inner().unwrap() {
353        return Err(e);
354    }
355
356    let mut indexed = collected.into_inner().unwrap();
357    indexed.sort_by_key(|(i, _)| *i);
358    Ok(indexed.into_iter().map(|(_, r)| r).collect())
359}
360
361pub fn aggregate(
362    bare: &[QueryRecord],
363    prefixed: &[QueryRecord],
364    relevant_sets: &std::collections::HashMap<String, HashSet<Uuid>>,
365) -> AggregateReport {
366    const AGG_SEED: u64 = 0x4B1D_C0DE;
367    let empty: HashSet<Uuid> = HashSet::new();
368
369    let per_query = |rec: &QueryRecord| -> (f32, f32, f32, f32, f32) {
370        let rel = relevant_sets.get(&rec.query_id).unwrap_or(&empty);
371        (
372            metrics::recall_at_k(&rec.ranked_ids, rel, 1),
373            metrics::recall_at_k(&rec.ranked_ids, rel, 3),
374            metrics::recall_at_k(&rec.ranked_ids, rel, 5),
375            metrics::mrr(&rec.ranked_ids, rel),
376            metrics::precision_at_k(&rec.ranked_ids, rel, 5),
377        )
378    };
379
380    let arm = |records: &[QueryRecord]| -> ArmMetrics {
381        if records.is_empty() {
382            return ArmMetrics {
383                recall_at_1: 0.0,
384                recall_at_3: 0.0,
385                recall_at_5: 0.0,
386                mrr: 0.0,
387                precision_at_5: 0.0,
388            };
389        }
390        let n = records.len() as f32;
391        let mut acc = (0.0, 0.0, 0.0, 0.0, 0.0);
392        for r in records {
393            let (r1, r3, r5, m, p5) = per_query(r);
394            acc.0 += r1;
395            acc.1 += r3;
396            acc.2 += r5;
397            acc.3 += m;
398            acc.4 += p5;
399        }
400        ArmMetrics {
401            recall_at_1: acc.0 / n,
402            recall_at_3: acc.1 / n,
403            recall_at_5: acc.2 / n,
404            mrr: acc.3 / n,
405            precision_at_5: acc.4 / n,
406        }
407    };
408
409    let bare_idx: std::collections::HashMap<&str, &QueryRecord> =
410        bare.iter().map(|r| (r.query_id.as_str(), r)).collect();
411
412    let mut delta_r3 = Vec::new();
413    let mut delta_mrr = Vec::new();
414    for p_rec in prefixed {
415        if let Some(b_rec) = bare_idx.get(p_rec.query_id.as_str()) {
416            let (_, p_r3, _, p_mrr, _) = per_query(p_rec);
417            let (_, b_r3, _, b_mrr, _) = per_query(b_rec);
418            delta_r3.push(p_r3 - b_r3);
419            delta_mrr.push(p_mrr - b_mrr);
420        }
421    }
422
423    let gated_rate = |records: &[QueryRecord]| -> f32 {
424        let flags: Vec<bool> = records
425            .iter()
426            .map(|r| match (r.gold_raw_rank, r.gold_raw_similarity) {
427                (Some(_), Some(sim)) => sim < BASELINE_THRESHOLD,
428                _ => false,
429            })
430            .collect();
431        metrics::gated_out_rate(&flags)
432    };
433
434    AggregateReport {
435        bare: arm(bare),
436        prefixed: arm(prefixed),
437        delta_recall_at_3_ci: bootstrap::paired_bootstrap_ci(&delta_r3, 10000, 0.95, AGG_SEED),
438        delta_mrr_ci: bootstrap::paired_bootstrap_ci(&delta_mrr, 10000, 0.95, AGG_SEED),
439        gated_out_rate_bare: gated_rate(bare),
440        gated_out_rate_prefixed: gated_rate(prefixed),
441    }
442}
443
444pub fn gate_sweep(prefixed: &[QueryRecord]) -> GateSweepReport {
445    const GRID_SEED: u64 = 0x5EED_6A7E;
446
447    let recalled_at = |rec: &QueryRecord, k: usize, t: f32| -> f32 {
448        match (rec.gold_raw_rank, rec.gold_raw_similarity) {
449            (Some(rank), Some(sim)) if rank <= k && sim >= t => 1.0,
450            _ => 0.0,
451        }
452    };
453
454    let mean = |vals: &[f32]| -> f32 {
455        if vals.is_empty() {
456            0.0
457        } else {
458            vals.iter().sum::<f32>() / vals.len() as f32
459        }
460    };
461
462    let baseline_r3: Vec<f32> = prefixed
463        .iter()
464        .map(|r| recalled_at(r, 3, BASELINE_THRESHOLD))
465        .collect();
466    let baseline_r3_mean = mean(&baseline_r3);
467
468    let mut frontier = Vec::with_capacity(41);
469    let mut chosen_threshold = BASELINE_THRESHOLD;
470    let mut chosen_beats_baseline = false;
471    let mut best_recall_at_3 = baseline_r3_mean;
472
473    for step in 0..=40u32 {
474        let t = 0.40 + step as f32 * 0.01;
475
476        let r1: Vec<f32> = prefixed.iter().map(|r| recalled_at(r, 1, t)).collect();
477        let r3: Vec<f32> = prefixed.iter().map(|r| recalled_at(r, 3, t)).collect();
478        let r5: Vec<f32> = prefixed.iter().map(|r| recalled_at(r, 5, t)).collect();
479
480        let recall_at_3 = mean(&r3);
481        let recall_at_5 = mean(&r5);
482
483        frontier.push(GatePoint {
484            threshold: t,
485            recall_at_1: mean(&r1),
486            recall_at_3,
487            recall_at_5,
488            precision_proxy: recall_at_5 / 5.0,
489        });
490
491        let deltas: Vec<f32> = r3
492            .iter()
493            .zip(baseline_r3.iter())
494            .map(|(t_val, b_val)| t_val - b_val)
495            .collect();
496        let (lo, _hi) = bootstrap::paired_bootstrap_ci(&deltas, 2000, 0.95, GRID_SEED);
497
498        if lo > 0.0 && recall_at_3 > best_recall_at_3 + 1e-6 {
499            best_recall_at_3 = recall_at_3;
500            chosen_threshold = t;
501            chosen_beats_baseline = true;
502        }
503    }
504
505    GateSweepReport {
506        frontier,
507        baseline_threshold: BASELINE_THRESHOLD,
508        chosen_threshold,
509        chosen_beats_baseline,
510    }
511}
512
513/// One gold query scored for the exhausted-scope floor: the cosine of its
514/// gold memory, which a recall scoped to the gold's own project must admit,
515/// and the best cosine in an unrelated project, which a recall scoped there
516/// should answer with nothing.
517#[derive(Debug, Clone, Serialize, Deserialize)]
518pub struct ScopedPair {
519    pub query_id: String,
520    pub gold: f32,
521    pub foreign_project: String,
522    pub foreign_best: f32,
523}
524
525#[derive(Debug, Clone, Serialize, Deserialize)]
526pub struct FloorPoint {
527    pub threshold: f32,
528    /// Share of gold memories at or above the floor.
529    pub gold_admitted: f32,
530    /// Share of foreign-scope answers left empty by the floor.
531    pub foreign_empty: f32,
532}
533
534#[derive(Debug, Clone, Serialize, Deserialize)]
535pub struct FloorSweepReport {
536    pub pairs: usize,
537    pub frontier: Vec<FloorPoint>,
538    /// The highest floor that still admits `FLOOR_GOLD_TARGET` of the gold
539    /// memories. Balancing the two rates instead (Youden's J) picked 0.71 on
540    /// the 2026-09 corpus and dropped a third of gold answers from their own
541    /// project, which is the one failure a caller who named the scope sees.
542    pub chosen_threshold: f32,
543}
544
545/// A named scope is the caller asserting where the answer is, so the floor
546/// may cost at most one gold answer in a hundred.
547pub const FLOOR_GOLD_TARGET: f32 = 0.99;
548
549pub fn exhausted_floor_sweep(pairs: &[ScopedPair]) -> FloorSweepReport {
550    let n = pairs.len().max(1) as f32;
551    let mut frontier = Vec::with_capacity(41);
552    let mut chosen = 0.40_f32;
553    for step in 0..=40u32 {
554        let t = 0.40 + step as f32 * 0.01;
555        let gold_admitted = pairs.iter().filter(|p| p.gold >= t).count() as f32 / n;
556        let foreign_empty = pairs.iter().filter(|p| p.foreign_best < t).count() as f32 / n;
557        if gold_admitted >= FLOOR_GOLD_TARGET {
558            chosen = t;
559        }
560        frontier.push(FloorPoint {
561            threshold: t,
562            gold_admitted,
563            foreign_empty,
564        });
565    }
566    FloorSweepReport {
567        pairs: pairs.len(),
568        frontier,
569        chosen_threshold: chosen,
570    }
571}
572
573/// Scores every gold query whose gold memory is live in `store` against its
574/// own vector and against the best match in a foreign project: one of
575/// `projects` other than the gold's, picked by the query's position so the
576/// pairing is deterministic and spread over every project.
577pub fn scoped_pairs(
578    store: &SqliteStore,
579    queries: &[EmbeddedQuery],
580    projects: &[String],
581) -> Result<Vec<ScopedPair>> {
582    let mut pairs = Vec::new();
583    for (i, q) in queries.iter().enumerate() {
584        let Some(own) = q.query.source_project.as_deref() else {
585            continue;
586        };
587        let Some(gold) = store.get_memory_with_embedding(q.query.seed_memory_id)? else {
588            continue;
589        };
590        if gold.embedding.is_empty() || gold.tier != second_brain_core::schema::Tier::Hot {
591            continue;
592        }
593        let foreign: Vec<&String> = projects.iter().filter(|p| p.as_str() != own).collect();
594        if foreign.is_empty() {
595            continue;
596        }
597        let foreign_project = foreign[i % foreign.len()].clone();
598        let emb = &q.prefixed_embedding;
599        let foreign_best = store
600            .filtered_vector_search(emb, None, None, Some(&foreign_project), 1)?
601            .first()
602            .map(|(_, score)| *score)
603            .unwrap_or(0.0);
604        pairs.push(ScopedPair {
605            query_id: q.query.query_id.clone(),
606            gold: second_brain_core::retrieval::cosine(emb, &gold.embedding),
607            foreign_project,
608            foreign_best,
609        });
610    }
611    Ok(pairs)
612}
613
614pub fn extract_corpus(store: &SqliteStore, out: &Path) -> Result<usize> {
615    let memories = store.all_memories_with_embeddings()?;
616    let mut file = std::fs::File::create(out)?;
617    let mut count = 0;
618    // Only rows recall can return are eligible as gold: an archived or
619    // superseded seed would score as a miss no matter how the ranker did.
620    for m in memories.iter().filter(|m| {
621        m.tier == second_brain_core::schema::Tier::Hot
622            && m.superseded_by.is_none()
623            && !m.embedding.is_empty()
624    }) {
625        let entry = CorpusEntry {
626            id: m.id,
627            content: m.content.clone(),
628            memory_type: format!("{:?}", m.memory_type).to_lowercase(),
629            created_at: m.created_at.to_rfc3339(),
630            project_path: m.project_path.clone(),
631        };
632        writeln!(file, "{}", serde_json::to_string(&entry)?)?;
633        count += 1;
634    }
635    Ok(count)
636}
637
638#[cfg(test)]
639mod tests {
640    use super::*;
641    use std::io::Write;
642
643    fn pair(gold: f32, foreign_best: f32) -> ScopedPair {
644        ScopedPair {
645            query_id: "q".to_string(),
646            gold,
647            foreign_project: "p".to_string(),
648            foreign_best,
649        }
650    }
651
652    #[test]
653    fn the_floor_sweep_picks_the_threshold_separating_gold_from_foreign() {
654        let pairs = [pair(0.70, 0.50), pair(0.65, 0.52), pair(0.60, 0.48)];
655        let report = exhausted_floor_sweep(&pairs);
656        assert_eq!(report.pairs, 3);
657        let chosen = report.chosen_threshold;
658        assert!(
659            (chosen - 0.60).abs() < 1e-4,
660            "the highest floor that still admits every gold memory: {chosen}"
661        );
662        let at = |t: f32| {
663            report
664                .frontier
665                .iter()
666                .find(|p| (p.threshold - t).abs() < 1e-4)
667                .unwrap()
668                .clone()
669        };
670        assert!((at(0.66).gold_admitted - 1.0 / 3.0).abs() < 1e-4);
671        assert!((at(0.50).foreign_empty - 1.0 / 3.0).abs() < 1e-4);
672    }
673
674    fn record(
675        id_rank: Option<usize>,
676        raw_rank: Option<usize>,
677        raw_sim: Option<f32>,
678    ) -> QueryRecord {
679        QueryRecord {
680            query_id: "q".to_string(),
681            use_prefix: true,
682            ranked_ids: Vec::new(),
683            scores: Vec::new(),
684            first_relevant_rank: id_rank,
685            gold_raw_rank: raw_rank,
686            gold_raw_similarity: raw_sim,
687            latency_ms: None,
688        }
689    }
690
691    #[test]
692    fn load_eval_set_parses_one_object_per_line() {
693        let dir = std::env::temp_dir();
694        let path = dir.join(format!("eval_set_{}.jsonl", Uuid::new_v4()));
695        let id_a = Uuid::new_v4();
696        let id_b = Uuid::new_v4();
697        let line1 = format!(
698            r#"{{"query_id":"q1","query":"kuzu choice","query_variant":"literal","seed_memory_id":"{id_a}","memory_type":"decision","relevant_memory_ids":["{id_a}"]}}"#
699        );
700        let line2 = format!(
701            r#"{{"query_id":"q2","query":"sync design","query_variant":"paraphrase","seed_memory_id":"{id_b}","memory_type":"architecture","relevant_memory_ids":["{id_b}","{id_a}"],"tags":["sync"]}}"#
702        );
703        let mut f = std::fs::File::create(&path).unwrap();
704        writeln!(f, "{line1}").unwrap();
705        writeln!(f, "{line2}").unwrap();
706        drop(f);
707
708        let queries = load_eval_set(&path).unwrap();
709        std::fs::remove_file(&path).ok();
710
711        assert_eq!(queries.len(), 2);
712        assert_eq!(queries[0].query_id, "q1");
713        assert_eq!(queries[0].seed_memory_id, id_a);
714        assert_eq!(queries[1].relevant_memory_ids.len(), 2);
715        assert_eq!(queries[1].tags, vec!["sync".to_string()]);
716    }
717
718    #[test]
719    fn load_eval_set_tolerates_blank_lines() {
720        let dir = std::env::temp_dir();
721        let path = dir.join(format!("eval_blank_{}.jsonl", Uuid::new_v4()));
722        let id = Uuid::new_v4();
723        let line = format!(
724            r#"{{"query_id":"q1","query":"x","query_variant":"v","seed_memory_id":"{id}","memory_type":"semantic","relevant_memory_ids":["{id}"]}}"#
725        );
726        std::fs::write(&path, format!("\n{line}\n\n")).unwrap();
727
728        let queries = load_eval_set(&path).unwrap();
729        std::fs::remove_file(&path).ok();
730
731        assert_eq!(queries.len(), 1);
732    }
733
734    #[test]
735    fn gate_sweep_emits_full_grid_and_monotone_recall() {
736        let records = vec![
737            record(Some(1), Some(1), Some(0.85)),
738            record(Some(2), Some(2), Some(0.62)),
739            record(Some(4), Some(4), Some(0.55)),
740            record(None, None, None),
741        ];
742
743        let report = gate_sweep(&records);
744
745        // grid 0.40..=0.80 step 0.01 inclusive is 41 points.
746        assert_eq!(report.frontier.len(), 41);
747        assert!((report.frontier.first().unwrap().threshold - 0.40).abs() < 1e-4);
748        assert!((report.frontier.last().unwrap().threshold - 0.80).abs() < 1e-4);
749
750        for w in report.frontier.windows(2) {
751            assert!(
752                w[0].recall_at_3 >= w[1].recall_at_3 - 1e-6,
753                "recall must not increase as the gate tightens"
754            );
755        }
756    }
757
758    #[test]
759    fn gate_sweep_recall_reflects_raw_rank_and_similarity() {
760        let records = vec![
761            record(Some(1), Some(1), Some(0.85)),
762            record(Some(2), Some(2), Some(0.62)),
763            record(Some(4), Some(4), Some(0.55)),
764            record(None, None, None),
765        ];
766
767        let report = gate_sweep(&records);
768
769        let at = |t: f32| {
770            report
771                .frontier
772                .iter()
773                .find(|p| (p.threshold - t).abs() < 1e-4)
774                .unwrap()
775        };
776
777        // T=0.50: golds at sim 0.85, 0.62, 0.55 survive; ranks 1,2,4. recall@3 covers
778        // the first two (rank<=3 and sim>=0.50) so 2/4 = 0.5; recall@5 covers all three = 0.75.
779        let p050 = at(0.50);
780        assert!(
781            (p050.recall_at_1 - 0.25).abs() < 1e-6,
782            "recall@1 was {}",
783            p050.recall_at_1
784        );
785        assert!(
786            (p050.recall_at_3 - 0.5).abs() < 1e-6,
787            "recall@3 was {}",
788            p050.recall_at_3
789        );
790        assert!(
791            (p050.recall_at_5 - 0.75).abs() < 1e-6,
792            "recall@5 was {}",
793            p050.recall_at_5
794        );
795
796        // T=0.70: only the 0.85 gold survives, at rank 1.
797        let p070 = at(0.70);
798        assert!((p070.recall_at_1 - 0.25).abs() < 1e-6);
799        assert!((p070.recall_at_3 - 0.25).abs() < 1e-6);
800        assert!((p070.recall_at_5 - 0.25).abs() < 1e-6);
801    }
802
803    #[test]
804    fn gate_sweep_keeps_baseline_when_nothing_beats_it() {
805        let records = vec![record(Some(1), Some(1), Some(0.85))];
806        let report = gate_sweep(&records);
807        assert!((report.baseline_threshold - 0.59).abs() < 1e-6);
808        assert!(!report.chosen_beats_baseline);
809        assert!((report.chosen_threshold - 0.59).abs() < 1e-6);
810    }
811}