Skip to main content

second_brain_api/eval/
mod.rs

1pub mod bootstrap;
2pub mod caching_store;
3pub mod metrics;
4
5use std::collections::HashSet;
6use std::io::Write;
7use std::path::Path;
8use std::sync::Mutex;
9use std::sync::atomic::{AtomicUsize, Ordering};
10use std::thread;
11
12use anyhow::Result;
13use serde::{Deserialize, Serialize};
14use uuid::Uuid;
15
16use second_brain_core::embedding::Embedder;
17use second_brain_core::kuzu_store::KuzuStore;
18use second_brain_core::query::{QueryEngine, QueryFilters, QueryRequest};
19use second_brain_core::store::Store;
20
21#[derive(Debug, Clone, Deserialize, Serialize)]
22pub struct EvalQuery {
23    pub query_id: String,
24    pub query: String,
25    pub query_variant: String,
26    pub seed_memory_id: Uuid,
27    pub memory_type: String,
28    pub relevant_memory_ids: Vec<Uuid>,
29    #[serde(default)]
30    pub note: Option<String>,
31    #[serde(default)]
32    pub tags: Vec<String>,
33}
34
35#[derive(Debug, Clone, Serialize, Deserialize)]
36pub struct QueryRecord {
37    pub query_id: String,
38    pub use_prefix: bool,
39    pub ranked_ids: Vec<Uuid>,
40    pub scores: Vec<f32>,
41    pub first_relevant_rank: Option<usize>,
42    pub gold_raw_rank: Option<usize>,
43    pub gold_raw_similarity: Option<f32>,
44}
45
46#[derive(Debug, Clone, Serialize, Deserialize)]
47pub struct ArmMetrics {
48    pub recall_at_1: f32,
49    pub recall_at_3: f32,
50    pub recall_at_5: f32,
51    pub mrr: f32,
52    pub precision_at_5: f32,
53}
54
55#[derive(Debug, Clone, Serialize, Deserialize)]
56pub struct AggregateReport {
57    pub bare: ArmMetrics,
58    pub prefixed: ArmMetrics,
59    pub delta_recall_at_3_ci: (f32, f32),
60    pub delta_mrr_ci: (f32, f32),
61    pub gated_out_rate_bare: f32,
62    pub gated_out_rate_prefixed: f32,
63}
64
65#[derive(Debug, Clone, Serialize, Deserialize)]
66pub struct GatePoint {
67    pub threshold: f32,
68    pub recall_at_1: f32,
69    pub recall_at_3: f32,
70    pub recall_at_5: f32,
71    pub precision_proxy: f32,
72}
73
74#[derive(Debug, Clone, Serialize, Deserialize)]
75pub struct GateSweepReport {
76    pub frontier: Vec<GatePoint>,
77    pub baseline_threshold: f32,
78    pub chosen_threshold: f32,
79    pub chosen_beats_baseline: bool,
80}
81
82#[derive(Debug, Clone, Serialize, Deserialize)]
83pub struct CorpusEntry {
84    pub id: Uuid,
85    pub content: String,
86    pub memory_type: String,
87    pub created_at: String,
88    pub project_path: Option<String>,
89}
90
91const BASELINE_THRESHOLD: f32 = 0.59;
92
93pub fn load_eval_set(path: &Path) -> Result<Vec<EvalQuery>> {
94    let text = std::fs::read_to_string(path)?;
95    let mut out = Vec::new();
96    for line in text.lines() {
97        let trimmed = line.trim();
98        if trimmed.is_empty() {
99            continue;
100        }
101        out.push(serde_json::from_str(trimmed)?);
102    }
103    Ok(out)
104}
105
106pub fn run_arm(
107    store: &KuzuStore,
108    embedder: &Embedder,
109    queries: &[EvalQuery],
110    use_prefix: bool,
111    limit: usize,
112) -> Result<Vec<QueryRecord>> {
113    let engine = QueryEngine::new(store);
114    let mut records = Vec::with_capacity(queries.len());
115
116    for q in queries {
117        let embedding = if use_prefix {
118            embedder.embed_query(&q.query)?
119        } else {
120            embedder.embed(&q.query)?
121        };
122
123        let relevant: HashSet<Uuid> = q.relevant_memory_ids.iter().copied().collect();
124
125        let request = QueryRequest {
126            text: q.query.clone(),
127            embedding: embedding.clone(),
128            limit,
129            filters: QueryFilters {
130                entity_names: crate::link_query_entities(&q.query),
131                ..QueryFilters::default()
132            },
133        };
134        let results = engine.recall(&request)?;
135
136        let ranked_ids: Vec<Uuid> = results.iter().map(|r| r.memory.id).collect();
137        let scores: Vec<f32> = results.iter().map(|r| r.score).collect();
138        let first_relevant_rank = ranked_ids
139            .iter()
140            .position(|id| relevant.contains(id))
141            .map(|idx| idx + 1);
142
143        let raw = store.vector_search(&embedding, limit * 3)?;
144        let mut gold_raw_rank = None;
145        let mut gold_raw_similarity = None;
146        for (idx, (mem, sim)) in raw.iter().enumerate() {
147            if relevant.contains(&mem.id) {
148                gold_raw_rank = Some(idx + 1);
149                gold_raw_similarity = Some(*sim);
150                break;
151            }
152        }
153
154        records.push(QueryRecord {
155            query_id: q.query_id.clone(),
156            use_prefix,
157            ranked_ids,
158            scores,
159            first_relevant_rank,
160            gold_raw_rank,
161            gold_raw_similarity,
162        });
163    }
164
165    Ok(records)
166}
167
168pub struct EmbeddedQuery {
169    pub query: EvalQuery,
170    pub relevant: HashSet<Uuid>,
171    pub bare_embedding: Vec<f32>,
172    pub prefixed_embedding: Vec<f32>,
173}
174
175// The Embedder wraps the model in a Mutex, so embedding cannot run in parallel.
176// We pre-compute every embedding serially here, in batches, to keep the model
177// out of the parallel recall section entirely.
178pub fn embed_all_queries(
179    embedder: &Embedder,
180    queries: &[EvalQuery],
181) -> Result<Vec<EmbeddedQuery>> {
182    let bare_texts: Vec<&str> = queries.iter().map(|q| q.query.as_str()).collect();
183    let bare = embedder.embed_batch(&bare_texts)?;
184
185    let prefixed_owned: Vec<String> = queries
186        .iter()
187        .map(|q| second_brain_core::embedding::query_prompt(&q.query))
188        .collect();
189    let prefixed_texts: Vec<&str> = prefixed_owned.iter().map(|s| s.as_str()).collect();
190    let prefixed = embedder.embed_batch(&prefixed_texts)?;
191
192    let mut out = Vec::with_capacity(queries.len());
193    for (i, q) in queries.iter().enumerate() {
194        out.push(EmbeddedQuery {
195            query: q.clone(),
196            relevant: q.relevant_memory_ids.iter().copied().collect(),
197            bare_embedding: bare[i].clone(),
198            prefixed_embedding: prefixed[i].clone(),
199        });
200    }
201    Ok(out)
202}
203
204fn record_for<S: Store + Sync>(
205    embedded: &EmbeddedQuery,
206    store: &S,
207    use_prefix: bool,
208    limit: usize,
209) -> Result<QueryRecord> {
210    let engine = QueryEngine::new(store);
211    let embedding = if use_prefix {
212        &embedded.prefixed_embedding
213    } else {
214        &embedded.bare_embedding
215    };
216
217    let request = QueryRequest {
218        text: embedded.query.query.clone(),
219        embedding: embedding.clone(),
220        limit,
221        filters: QueryFilters {
222            entity_names: crate::link_query_entities(&embedded.query.query),
223            ..QueryFilters::default()
224        },
225    };
226    let results = engine.recall(&request)?;
227
228    let ranked_ids: Vec<Uuid> = results.iter().map(|r| r.memory.id).collect();
229    let scores: Vec<f32> = results.iter().map(|r| r.score).collect();
230    let first_relevant_rank = ranked_ids
231        .iter()
232        .position(|id| embedded.relevant.contains(id))
233        .map(|idx| idx + 1);
234
235    let raw = store.vector_search(embedding, limit * 3)?;
236    let mut gold_raw_rank = None;
237    let mut gold_raw_similarity = None;
238    for (idx, (mem, sim)) in raw.iter().enumerate() {
239        if embedded.relevant.contains(&mem.id) {
240            gold_raw_rank = Some(idx + 1);
241            gold_raw_similarity = Some(*sim);
242            break;
243        }
244    }
245
246    Ok(QueryRecord {
247        query_id: embedded.query.query_id.clone(),
248        use_prefix,
249        ranked_ids,
250        scores,
251        first_relevant_rank,
252        gold_raw_rank,
253        gold_raw_similarity,
254    })
255}
256
257// Each store call creates its own Kuzu Connection (KuzuStore::conn does
258// Connection::new per call), which is the same concurrent-read pattern the
259// daemon uses to serve overlapping recall requests, so sharing &store across
260// scoped threads is safe. The Embedder is absent here on purpose: embeddings
261// are pre-computed in embed_all_queries.
262pub fn run_arm_parallel<S: Store + Sync>(
263    store: &S,
264    embedded: &[EmbeddedQuery],
265    use_prefix: bool,
266    limit: usize,
267) -> Result<Vec<QueryRecord>> {
268    let total = embedded.len();
269    if total == 0 {
270        return Ok(Vec::new());
271    }
272
273    let workers = thread::available_parallelism()
274        .map(|n| n.get().saturating_sub(1).max(1))
275        .unwrap_or(1);
276    let chunk_size = total.div_ceil(workers);
277
278    let done = AtomicUsize::new(0);
279    let collected: Mutex<Vec<(usize, QueryRecord)>> = Mutex::new(Vec::with_capacity(total));
280    let error: Mutex<Option<anyhow::Error>> = Mutex::new(None);
281
282    thread::scope(|scope| {
283        for chunk in 0..workers {
284            let start = chunk * chunk_size;
285            if start >= total {
286                break;
287            }
288            let end = (start + chunk_size).min(total);
289            let done = &done;
290            let collected = &collected;
291            let error = &error;
292            scope.spawn(move || {
293                let mut local: Vec<(usize, QueryRecord)> = Vec::with_capacity(end - start);
294                for (offset, eq) in embedded[start..end].iter().enumerate() {
295                    if error.lock().unwrap().is_some() {
296                        return;
297                    }
298                    match record_for(eq, store, use_prefix, limit) {
299                        Ok(rec) => local.push((start + offset, rec)),
300                        Err(e) => {
301                            *error.lock().unwrap() = Some(e);
302                            return;
303                        }
304                    }
305                    // eprintln progress because long runs were previously invisible.
306                    let n = done.fetch_add(1, Ordering::Relaxed) + 1;
307                    if n % 25 == 0 || n == total {
308                        eprintln!("  {n}/{total} queries");
309                    }
310                }
311                collected.lock().unwrap().extend(local);
312            });
313        }
314    });
315
316    if let Some(e) = error.into_inner().unwrap() {
317        return Err(e);
318    }
319
320    let mut indexed = collected.into_inner().unwrap();
321    indexed.sort_by_key(|(i, _)| *i);
322    Ok(indexed.into_iter().map(|(_, r)| r).collect())
323}
324
325pub fn aggregate(
326    bare: &[QueryRecord],
327    prefixed: &[QueryRecord],
328    relevant_sets: &std::collections::HashMap<String, HashSet<Uuid>>,
329) -> AggregateReport {
330    const AGG_SEED: u64 = 0x4B1D_C0DE;
331    let empty: HashSet<Uuid> = HashSet::new();
332
333    let per_query = |rec: &QueryRecord| -> (f32, f32, f32, f32, f32) {
334        let rel = relevant_sets.get(&rec.query_id).unwrap_or(&empty);
335        (
336            metrics::recall_at_k(&rec.ranked_ids, rel, 1),
337            metrics::recall_at_k(&rec.ranked_ids, rel, 3),
338            metrics::recall_at_k(&rec.ranked_ids, rel, 5),
339            metrics::mrr(&rec.ranked_ids, rel),
340            metrics::precision_at_k(&rec.ranked_ids, rel, 5),
341        )
342    };
343
344    let arm = |records: &[QueryRecord]| -> ArmMetrics {
345        if records.is_empty() {
346            return ArmMetrics {
347                recall_at_1: 0.0,
348                recall_at_3: 0.0,
349                recall_at_5: 0.0,
350                mrr: 0.0,
351                precision_at_5: 0.0,
352            };
353        }
354        let n = records.len() as f32;
355        let mut acc = (0.0, 0.0, 0.0, 0.0, 0.0);
356        for r in records {
357            let (r1, r3, r5, m, p5) = per_query(r);
358            acc.0 += r1;
359            acc.1 += r3;
360            acc.2 += r5;
361            acc.3 += m;
362            acc.4 += p5;
363        }
364        ArmMetrics {
365            recall_at_1: acc.0 / n,
366            recall_at_3: acc.1 / n,
367            recall_at_5: acc.2 / n,
368            mrr: acc.3 / n,
369            precision_at_5: acc.4 / n,
370        }
371    };
372
373    let bare_idx: std::collections::HashMap<&str, &QueryRecord> =
374        bare.iter().map(|r| (r.query_id.as_str(), r)).collect();
375
376    let mut delta_r3 = Vec::new();
377    let mut delta_mrr = Vec::new();
378    for p_rec in prefixed {
379        if let Some(b_rec) = bare_idx.get(p_rec.query_id.as_str()) {
380            let (_, p_r3, _, p_mrr, _) = per_query(p_rec);
381            let (_, b_r3, _, b_mrr, _) = per_query(b_rec);
382            delta_r3.push(p_r3 - b_r3);
383            delta_mrr.push(p_mrr - b_mrr);
384        }
385    }
386
387    let gated_rate = |records: &[QueryRecord]| -> f32 {
388        let flags: Vec<bool> = records
389            .iter()
390            .map(|r| match (r.gold_raw_rank, r.gold_raw_similarity) {
391                (Some(_), Some(sim)) => sim < BASELINE_THRESHOLD,
392                _ => false,
393            })
394            .collect();
395        metrics::gated_out_rate(&flags)
396    };
397
398    AggregateReport {
399        bare: arm(bare),
400        prefixed: arm(prefixed),
401        delta_recall_at_3_ci: bootstrap::paired_bootstrap_ci(&delta_r3, 10000, 0.95, AGG_SEED),
402        delta_mrr_ci: bootstrap::paired_bootstrap_ci(&delta_mrr, 10000, 0.95, AGG_SEED),
403        gated_out_rate_bare: gated_rate(bare),
404        gated_out_rate_prefixed: gated_rate(prefixed),
405    }
406}
407
408pub fn gate_sweep(prefixed: &[QueryRecord]) -> GateSweepReport {
409    const GRID_SEED: u64 = 0x5EED_6A7E;
410
411    let recalled_at = |rec: &QueryRecord, k: usize, t: f32| -> f32 {
412        match (rec.gold_raw_rank, rec.gold_raw_similarity) {
413            (Some(rank), Some(sim)) if rank <= k && sim >= t => 1.0,
414            _ => 0.0,
415        }
416    };
417
418    let mean = |vals: &[f32]| -> f32 {
419        if vals.is_empty() {
420            0.0
421        } else {
422            vals.iter().sum::<f32>() / vals.len() as f32
423        }
424    };
425
426    let baseline_r3: Vec<f32> = prefixed
427        .iter()
428        .map(|r| recalled_at(r, 3, BASELINE_THRESHOLD))
429        .collect();
430    let baseline_r3_mean = mean(&baseline_r3);
431
432    let mut frontier = Vec::with_capacity(41);
433    let mut chosen_threshold = BASELINE_THRESHOLD;
434    let mut chosen_beats_baseline = false;
435    let mut best_recall_at_3 = baseline_r3_mean;
436
437    for step in 0..=40u32 {
438        let t = 0.40 + step as f32 * 0.01;
439
440        let r1: Vec<f32> = prefixed.iter().map(|r| recalled_at(r, 1, t)).collect();
441        let r3: Vec<f32> = prefixed.iter().map(|r| recalled_at(r, 3, t)).collect();
442        let r5: Vec<f32> = prefixed.iter().map(|r| recalled_at(r, 5, t)).collect();
443
444        let recall_at_3 = mean(&r3);
445        let recall_at_5 = mean(&r5);
446
447        frontier.push(GatePoint {
448            threshold: t,
449            recall_at_1: mean(&r1),
450            recall_at_3,
451            recall_at_5,
452            precision_proxy: recall_at_5 / 5.0,
453        });
454
455        let deltas: Vec<f32> = r3
456            .iter()
457            .zip(baseline_r3.iter())
458            .map(|(t_val, b_val)| t_val - b_val)
459            .collect();
460        let (lo, _hi) = bootstrap::paired_bootstrap_ci(&deltas, 2000, 0.95, GRID_SEED);
461
462        if lo > 0.0 && recall_at_3 > best_recall_at_3 + 1e-6 {
463            best_recall_at_3 = recall_at_3;
464            chosen_threshold = t;
465            chosen_beats_baseline = true;
466        }
467    }
468
469    GateSweepReport {
470        frontier,
471        baseline_threshold: BASELINE_THRESHOLD,
472        chosen_threshold,
473        chosen_beats_baseline,
474    }
475}
476
477pub fn extract_corpus(store: &KuzuStore, out: &Path) -> Result<usize> {
478    let memories = store.all_memories_with_embeddings()?;
479    let mut file = std::fs::File::create(out)?;
480    let mut count = 0;
481    for m in &memories {
482        let entry = CorpusEntry {
483            id: m.id,
484            content: m.content.clone(),
485            memory_type: format!("{:?}", m.memory_type).to_lowercase(),
486            created_at: m.created_at.to_rfc3339(),
487            project_path: m.project_path.clone(),
488        };
489        writeln!(file, "{}", serde_json::to_string(&entry)?)?;
490        count += 1;
491    }
492    Ok(count)
493}
494
495#[cfg(test)]
496mod tests {
497    use super::*;
498    use std::io::Write;
499
500    fn record(id_rank: Option<usize>, raw_rank: Option<usize>, raw_sim: Option<f32>) -> QueryRecord {
501        QueryRecord {
502            query_id: "q".to_string(),
503            use_prefix: true,
504            ranked_ids: Vec::new(),
505            scores: Vec::new(),
506            first_relevant_rank: id_rank,
507            gold_raw_rank: raw_rank,
508            gold_raw_similarity: raw_sim,
509        }
510    }
511
512    #[test]
513    fn load_eval_set_parses_one_object_per_line() {
514        let dir = std::env::temp_dir();
515        let path = dir.join(format!("eval_set_{}.jsonl", Uuid::new_v4()));
516        let id_a = Uuid::new_v4();
517        let id_b = Uuid::new_v4();
518        let line1 = format!(
519            r#"{{"query_id":"q1","query":"kuzu choice","query_variant":"literal","seed_memory_id":"{id_a}","memory_type":"decision","relevant_memory_ids":["{id_a}"]}}"#
520        );
521        let line2 = format!(
522            r#"{{"query_id":"q2","query":"sync design","query_variant":"paraphrase","seed_memory_id":"{id_b}","memory_type":"architecture","relevant_memory_ids":["{id_b}","{id_a}"],"tags":["sync"]}}"#
523        );
524        let mut f = std::fs::File::create(&path).unwrap();
525        writeln!(f, "{line1}").unwrap();
526        writeln!(f, "{line2}").unwrap();
527        drop(f);
528
529        let queries = load_eval_set(&path).unwrap();
530        std::fs::remove_file(&path).ok();
531
532        assert_eq!(queries.len(), 2);
533        assert_eq!(queries[0].query_id, "q1");
534        assert_eq!(queries[0].seed_memory_id, id_a);
535        assert_eq!(queries[1].relevant_memory_ids.len(), 2);
536        assert_eq!(queries[1].tags, vec!["sync".to_string()]);
537    }
538
539    #[test]
540    fn load_eval_set_tolerates_blank_lines() {
541        let dir = std::env::temp_dir();
542        let path = dir.join(format!("eval_blank_{}.jsonl", Uuid::new_v4()));
543        let id = Uuid::new_v4();
544        let line = format!(
545            r#"{{"query_id":"q1","query":"x","query_variant":"v","seed_memory_id":"{id}","memory_type":"semantic","relevant_memory_ids":["{id}"]}}"#
546        );
547        std::fs::write(&path, format!("\n{line}\n\n")).unwrap();
548
549        let queries = load_eval_set(&path).unwrap();
550        std::fs::remove_file(&path).ok();
551
552        assert_eq!(queries.len(), 1);
553    }
554
555    #[test]
556    fn gate_sweep_emits_full_grid_and_monotone_recall() {
557        let records = vec![
558            record(Some(1), Some(1), Some(0.85)),
559            record(Some(2), Some(2), Some(0.62)),
560            record(Some(4), Some(4), Some(0.55)),
561            record(None, None, None),
562        ];
563
564        let report = gate_sweep(&records);
565
566        // grid 0.40..=0.80 step 0.01 inclusive is 41 points.
567        assert_eq!(report.frontier.len(), 41);
568        assert!((report.frontier.first().unwrap().threshold - 0.40).abs() < 1e-4);
569        assert!((report.frontier.last().unwrap().threshold - 0.80).abs() < 1e-4);
570
571        for w in report.frontier.windows(2) {
572            assert!(
573                w[0].recall_at_3 >= w[1].recall_at_3 - 1e-6,
574                "recall must not increase as the gate tightens"
575            );
576        }
577    }
578
579    #[test]
580    fn gate_sweep_recall_reflects_raw_rank_and_similarity() {
581        let records = vec![
582            record(Some(1), Some(1), Some(0.85)),
583            record(Some(2), Some(2), Some(0.62)),
584            record(Some(4), Some(4), Some(0.55)),
585            record(None, None, None),
586        ];
587
588        let report = gate_sweep(&records);
589
590        let at = |t: f32| {
591            report
592                .frontier
593                .iter()
594                .find(|p| (p.threshold - t).abs() < 1e-4)
595                .unwrap()
596        };
597
598        // T=0.50: golds at sim 0.85, 0.62, 0.55 survive; ranks 1,2,4. recall@3 covers
599        // the first two (rank<=3 and sim>=0.50) so 2/4 = 0.5; recall@5 covers all three = 0.75.
600        let p050 = at(0.50);
601        assert!((p050.recall_at_1 - 0.25).abs() < 1e-6, "recall@1 was {}", p050.recall_at_1);
602        assert!((p050.recall_at_3 - 0.5).abs() < 1e-6, "recall@3 was {}", p050.recall_at_3);
603        assert!((p050.recall_at_5 - 0.75).abs() < 1e-6, "recall@5 was {}", p050.recall_at_5);
604
605        // T=0.70: only the 0.85 gold survives, at rank 1.
606        let p070 = at(0.70);
607        assert!((p070.recall_at_1 - 0.25).abs() < 1e-6);
608        assert!((p070.recall_at_3 - 0.25).abs() < 1e-6);
609        assert!((p070.recall_at_5 - 0.25).abs() < 1e-6);
610    }
611
612    #[test]
613    fn gate_sweep_keeps_baseline_when_nothing_beats_it() {
614        let records = vec![record(Some(1), Some(1), Some(0.85))];
615        let report = gate_sweep(&records);
616        assert!((report.baseline_threshold - 0.59).abs() < 1e-6);
617        assert!(!report.chosen_beats_baseline);
618        assert!((report.chosen_threshold - 0.59).abs() < 1e-6);
619    }
620}