1pub mod bootstrap;
2pub mod caching_store;
3pub mod metrics;
4
5use std::collections::HashSet;
6use std::io::Write;
7use std::path::Path;
8use std::sync::Mutex;
9use std::sync::atomic::{AtomicUsize, Ordering};
10use std::thread;
11
12use anyhow::Result;
13use serde::{Deserialize, Serialize};
14use uuid::Uuid;
15
16use second_brain_core::embedding::Embedder;
17use second_brain_core::kuzu_store::KuzuStore;
18use second_brain_core::query::{QueryEngine, QueryFilters, QueryRequest};
19use second_brain_core::store::Store;
20
21#[derive(Debug, Clone, Deserialize, Serialize)]
22pub struct EvalQuery {
23 pub query_id: String,
24 pub query: String,
25 pub query_variant: String,
26 pub seed_memory_id: Uuid,
27 pub memory_type: String,
28 pub relevant_memory_ids: Vec<Uuid>,
29 #[serde(default)]
30 pub note: Option<String>,
31 #[serde(default)]
32 pub tags: Vec<String>,
33}
34
35#[derive(Debug, Clone, Serialize, Deserialize)]
36pub struct QueryRecord {
37 pub query_id: String,
38 pub use_prefix: bool,
39 pub ranked_ids: Vec<Uuid>,
40 pub scores: Vec<f32>,
41 pub first_relevant_rank: Option<usize>,
42 pub gold_raw_rank: Option<usize>,
43 pub gold_raw_similarity: Option<f32>,
44}
45
46#[derive(Debug, Clone, Serialize, Deserialize)]
47pub struct ArmMetrics {
48 pub recall_at_1: f32,
49 pub recall_at_3: f32,
50 pub recall_at_5: f32,
51 pub mrr: f32,
52 pub precision_at_5: f32,
53}
54
55#[derive(Debug, Clone, Serialize, Deserialize)]
56pub struct AggregateReport {
57 pub bare: ArmMetrics,
58 pub prefixed: ArmMetrics,
59 pub delta_recall_at_3_ci: (f32, f32),
60 pub delta_mrr_ci: (f32, f32),
61 pub gated_out_rate_bare: f32,
62 pub gated_out_rate_prefixed: f32,
63}
64
65#[derive(Debug, Clone, Serialize, Deserialize)]
66pub struct GatePoint {
67 pub threshold: f32,
68 pub recall_at_1: f32,
69 pub recall_at_3: f32,
70 pub recall_at_5: f32,
71 pub precision_proxy: f32,
72}
73
74#[derive(Debug, Clone, Serialize, Deserialize)]
75pub struct GateSweepReport {
76 pub frontier: Vec<GatePoint>,
77 pub baseline_threshold: f32,
78 pub chosen_threshold: f32,
79 pub chosen_beats_baseline: bool,
80}
81
82#[derive(Debug, Clone, Serialize, Deserialize)]
83pub struct CorpusEntry {
84 pub id: Uuid,
85 pub content: String,
86 pub memory_type: String,
87 pub created_at: String,
88 pub project_path: Option<String>,
89}
90
91const BASELINE_THRESHOLD: f32 = 0.59;
92
93pub fn load_eval_set(path: &Path) -> Result<Vec<EvalQuery>> {
94 let text = std::fs::read_to_string(path)?;
95 let mut out = Vec::new();
96 for line in text.lines() {
97 let trimmed = line.trim();
98 if trimmed.is_empty() {
99 continue;
100 }
101 out.push(serde_json::from_str(trimmed)?);
102 }
103 Ok(out)
104}
105
106pub fn run_arm(
107 store: &KuzuStore,
108 embedder: &Embedder,
109 queries: &[EvalQuery],
110 use_prefix: bool,
111 limit: usize,
112) -> Result<Vec<QueryRecord>> {
113 let engine = QueryEngine::new(store);
114 let mut records = Vec::with_capacity(queries.len());
115
116 for q in queries {
117 let embedding = if use_prefix {
118 embedder.embed_query(&q.query)?
119 } else {
120 embedder.embed(&q.query)?
121 };
122
123 let relevant: HashSet<Uuid> = q.relevant_memory_ids.iter().copied().collect();
124
125 let request = QueryRequest {
126 text: q.query.clone(),
127 embedding: embedding.clone(),
128 limit,
129 filters: QueryFilters {
130 entity_names: crate::link_query_entities(&q.query),
131 ..QueryFilters::default()
132 },
133 };
134 let results = engine.recall(&request)?;
135
136 let ranked_ids: Vec<Uuid> = results.iter().map(|r| r.memory.id).collect();
137 let scores: Vec<f32> = results.iter().map(|r| r.score).collect();
138 let first_relevant_rank = ranked_ids
139 .iter()
140 .position(|id| relevant.contains(id))
141 .map(|idx| idx + 1);
142
143 let raw = store.vector_search(&embedding, limit * 3)?;
144 let mut gold_raw_rank = None;
145 let mut gold_raw_similarity = None;
146 for (idx, (mem, sim)) in raw.iter().enumerate() {
147 if relevant.contains(&mem.id) {
148 gold_raw_rank = Some(idx + 1);
149 gold_raw_similarity = Some(*sim);
150 break;
151 }
152 }
153
154 records.push(QueryRecord {
155 query_id: q.query_id.clone(),
156 use_prefix,
157 ranked_ids,
158 scores,
159 first_relevant_rank,
160 gold_raw_rank,
161 gold_raw_similarity,
162 });
163 }
164
165 Ok(records)
166}
167
168pub struct EmbeddedQuery {
169 pub query: EvalQuery,
170 pub relevant: HashSet<Uuid>,
171 pub bare_embedding: Vec<f32>,
172 pub prefixed_embedding: Vec<f32>,
173}
174
175pub fn embed_all_queries(
179 embedder: &Embedder,
180 queries: &[EvalQuery],
181) -> Result<Vec<EmbeddedQuery>> {
182 let bare_texts: Vec<&str> = queries.iter().map(|q| q.query.as_str()).collect();
183 let bare = embedder.embed_batch(&bare_texts)?;
184
185 let prefixed_owned: Vec<String> = queries
186 .iter()
187 .map(|q| second_brain_core::embedding::query_prompt(&q.query))
188 .collect();
189 let prefixed_texts: Vec<&str> = prefixed_owned.iter().map(|s| s.as_str()).collect();
190 let prefixed = embedder.embed_batch(&prefixed_texts)?;
191
192 let mut out = Vec::with_capacity(queries.len());
193 for (i, q) in queries.iter().enumerate() {
194 out.push(EmbeddedQuery {
195 query: q.clone(),
196 relevant: q.relevant_memory_ids.iter().copied().collect(),
197 bare_embedding: bare[i].clone(),
198 prefixed_embedding: prefixed[i].clone(),
199 });
200 }
201 Ok(out)
202}
203
204fn record_for<S: Store + Sync>(
205 embedded: &EmbeddedQuery,
206 store: &S,
207 use_prefix: bool,
208 limit: usize,
209) -> Result<QueryRecord> {
210 let engine = QueryEngine::new(store);
211 let embedding = if use_prefix {
212 &embedded.prefixed_embedding
213 } else {
214 &embedded.bare_embedding
215 };
216
217 let request = QueryRequest {
218 text: embedded.query.query.clone(),
219 embedding: embedding.clone(),
220 limit,
221 filters: QueryFilters {
222 entity_names: crate::link_query_entities(&embedded.query.query),
223 ..QueryFilters::default()
224 },
225 };
226 let results = engine.recall(&request)?;
227
228 let ranked_ids: Vec<Uuid> = results.iter().map(|r| r.memory.id).collect();
229 let scores: Vec<f32> = results.iter().map(|r| r.score).collect();
230 let first_relevant_rank = ranked_ids
231 .iter()
232 .position(|id| embedded.relevant.contains(id))
233 .map(|idx| idx + 1);
234
235 let raw = store.vector_search(embedding, limit * 3)?;
236 let mut gold_raw_rank = None;
237 let mut gold_raw_similarity = None;
238 for (idx, (mem, sim)) in raw.iter().enumerate() {
239 if embedded.relevant.contains(&mem.id) {
240 gold_raw_rank = Some(idx + 1);
241 gold_raw_similarity = Some(*sim);
242 break;
243 }
244 }
245
246 Ok(QueryRecord {
247 query_id: embedded.query.query_id.clone(),
248 use_prefix,
249 ranked_ids,
250 scores,
251 first_relevant_rank,
252 gold_raw_rank,
253 gold_raw_similarity,
254 })
255}
256
257pub fn run_arm_parallel<S: Store + Sync>(
263 store: &S,
264 embedded: &[EmbeddedQuery],
265 use_prefix: bool,
266 limit: usize,
267) -> Result<Vec<QueryRecord>> {
268 let total = embedded.len();
269 if total == 0 {
270 return Ok(Vec::new());
271 }
272
273 let workers = thread::available_parallelism()
274 .map(|n| n.get().saturating_sub(1).max(1))
275 .unwrap_or(1);
276 let chunk_size = total.div_ceil(workers);
277
278 let done = AtomicUsize::new(0);
279 let collected: Mutex<Vec<(usize, QueryRecord)>> = Mutex::new(Vec::with_capacity(total));
280 let error: Mutex<Option<anyhow::Error>> = Mutex::new(None);
281
282 thread::scope(|scope| {
283 for chunk in 0..workers {
284 let start = chunk * chunk_size;
285 if start >= total {
286 break;
287 }
288 let end = (start + chunk_size).min(total);
289 let done = &done;
290 let collected = &collected;
291 let error = &error;
292 scope.spawn(move || {
293 let mut local: Vec<(usize, QueryRecord)> = Vec::with_capacity(end - start);
294 for (offset, eq) in embedded[start..end].iter().enumerate() {
295 if error.lock().unwrap().is_some() {
296 return;
297 }
298 match record_for(eq, store, use_prefix, limit) {
299 Ok(rec) => local.push((start + offset, rec)),
300 Err(e) => {
301 *error.lock().unwrap() = Some(e);
302 return;
303 }
304 }
305 let n = done.fetch_add(1, Ordering::Relaxed) + 1;
307 if n % 25 == 0 || n == total {
308 eprintln!(" {n}/{total} queries");
309 }
310 }
311 collected.lock().unwrap().extend(local);
312 });
313 }
314 });
315
316 if let Some(e) = error.into_inner().unwrap() {
317 return Err(e);
318 }
319
320 let mut indexed = collected.into_inner().unwrap();
321 indexed.sort_by_key(|(i, _)| *i);
322 Ok(indexed.into_iter().map(|(_, r)| r).collect())
323}
324
325pub fn aggregate(
326 bare: &[QueryRecord],
327 prefixed: &[QueryRecord],
328 relevant_sets: &std::collections::HashMap<String, HashSet<Uuid>>,
329) -> AggregateReport {
330 const AGG_SEED: u64 = 0x4B1D_C0DE;
331 let empty: HashSet<Uuid> = HashSet::new();
332
333 let per_query = |rec: &QueryRecord| -> (f32, f32, f32, f32, f32) {
334 let rel = relevant_sets.get(&rec.query_id).unwrap_or(&empty);
335 (
336 metrics::recall_at_k(&rec.ranked_ids, rel, 1),
337 metrics::recall_at_k(&rec.ranked_ids, rel, 3),
338 metrics::recall_at_k(&rec.ranked_ids, rel, 5),
339 metrics::mrr(&rec.ranked_ids, rel),
340 metrics::precision_at_k(&rec.ranked_ids, rel, 5),
341 )
342 };
343
344 let arm = |records: &[QueryRecord]| -> ArmMetrics {
345 if records.is_empty() {
346 return ArmMetrics {
347 recall_at_1: 0.0,
348 recall_at_3: 0.0,
349 recall_at_5: 0.0,
350 mrr: 0.0,
351 precision_at_5: 0.0,
352 };
353 }
354 let n = records.len() as f32;
355 let mut acc = (0.0, 0.0, 0.0, 0.0, 0.0);
356 for r in records {
357 let (r1, r3, r5, m, p5) = per_query(r);
358 acc.0 += r1;
359 acc.1 += r3;
360 acc.2 += r5;
361 acc.3 += m;
362 acc.4 += p5;
363 }
364 ArmMetrics {
365 recall_at_1: acc.0 / n,
366 recall_at_3: acc.1 / n,
367 recall_at_5: acc.2 / n,
368 mrr: acc.3 / n,
369 precision_at_5: acc.4 / n,
370 }
371 };
372
373 let bare_idx: std::collections::HashMap<&str, &QueryRecord> =
374 bare.iter().map(|r| (r.query_id.as_str(), r)).collect();
375
376 let mut delta_r3 = Vec::new();
377 let mut delta_mrr = Vec::new();
378 for p_rec in prefixed {
379 if let Some(b_rec) = bare_idx.get(p_rec.query_id.as_str()) {
380 let (_, p_r3, _, p_mrr, _) = per_query(p_rec);
381 let (_, b_r3, _, b_mrr, _) = per_query(b_rec);
382 delta_r3.push(p_r3 - b_r3);
383 delta_mrr.push(p_mrr - b_mrr);
384 }
385 }
386
387 let gated_rate = |records: &[QueryRecord]| -> f32 {
388 let flags: Vec<bool> = records
389 .iter()
390 .map(|r| match (r.gold_raw_rank, r.gold_raw_similarity) {
391 (Some(_), Some(sim)) => sim < BASELINE_THRESHOLD,
392 _ => false,
393 })
394 .collect();
395 metrics::gated_out_rate(&flags)
396 };
397
398 AggregateReport {
399 bare: arm(bare),
400 prefixed: arm(prefixed),
401 delta_recall_at_3_ci: bootstrap::paired_bootstrap_ci(&delta_r3, 10000, 0.95, AGG_SEED),
402 delta_mrr_ci: bootstrap::paired_bootstrap_ci(&delta_mrr, 10000, 0.95, AGG_SEED),
403 gated_out_rate_bare: gated_rate(bare),
404 gated_out_rate_prefixed: gated_rate(prefixed),
405 }
406}
407
408pub fn gate_sweep(prefixed: &[QueryRecord]) -> GateSweepReport {
409 const GRID_SEED: u64 = 0x5EED_6A7E;
410
411 let recalled_at = |rec: &QueryRecord, k: usize, t: f32| -> f32 {
412 match (rec.gold_raw_rank, rec.gold_raw_similarity) {
413 (Some(rank), Some(sim)) if rank <= k && sim >= t => 1.0,
414 _ => 0.0,
415 }
416 };
417
418 let mean = |vals: &[f32]| -> f32 {
419 if vals.is_empty() {
420 0.0
421 } else {
422 vals.iter().sum::<f32>() / vals.len() as f32
423 }
424 };
425
426 let baseline_r3: Vec<f32> = prefixed
427 .iter()
428 .map(|r| recalled_at(r, 3, BASELINE_THRESHOLD))
429 .collect();
430 let baseline_r3_mean = mean(&baseline_r3);
431
432 let mut frontier = Vec::with_capacity(41);
433 let mut chosen_threshold = BASELINE_THRESHOLD;
434 let mut chosen_beats_baseline = false;
435 let mut best_recall_at_3 = baseline_r3_mean;
436
437 for step in 0..=40u32 {
438 let t = 0.40 + step as f32 * 0.01;
439
440 let r1: Vec<f32> = prefixed.iter().map(|r| recalled_at(r, 1, t)).collect();
441 let r3: Vec<f32> = prefixed.iter().map(|r| recalled_at(r, 3, t)).collect();
442 let r5: Vec<f32> = prefixed.iter().map(|r| recalled_at(r, 5, t)).collect();
443
444 let recall_at_3 = mean(&r3);
445 let recall_at_5 = mean(&r5);
446
447 frontier.push(GatePoint {
448 threshold: t,
449 recall_at_1: mean(&r1),
450 recall_at_3,
451 recall_at_5,
452 precision_proxy: recall_at_5 / 5.0,
453 });
454
455 let deltas: Vec<f32> = r3
456 .iter()
457 .zip(baseline_r3.iter())
458 .map(|(t_val, b_val)| t_val - b_val)
459 .collect();
460 let (lo, _hi) = bootstrap::paired_bootstrap_ci(&deltas, 2000, 0.95, GRID_SEED);
461
462 if lo > 0.0 && recall_at_3 > best_recall_at_3 + 1e-6 {
463 best_recall_at_3 = recall_at_3;
464 chosen_threshold = t;
465 chosen_beats_baseline = true;
466 }
467 }
468
469 GateSweepReport {
470 frontier,
471 baseline_threshold: BASELINE_THRESHOLD,
472 chosen_threshold,
473 chosen_beats_baseline,
474 }
475}
476
477pub fn extract_corpus(store: &KuzuStore, out: &Path) -> Result<usize> {
478 let memories = store.all_memories_with_embeddings()?;
479 let mut file = std::fs::File::create(out)?;
480 let mut count = 0;
481 for m in &memories {
482 let entry = CorpusEntry {
483 id: m.id,
484 content: m.content.clone(),
485 memory_type: format!("{:?}", m.memory_type).to_lowercase(),
486 created_at: m.created_at.to_rfc3339(),
487 project_path: m.project_path.clone(),
488 };
489 writeln!(file, "{}", serde_json::to_string(&entry)?)?;
490 count += 1;
491 }
492 Ok(count)
493}
494
495#[cfg(test)]
496mod tests {
497 use super::*;
498 use std::io::Write;
499
500 fn record(id_rank: Option<usize>, raw_rank: Option<usize>, raw_sim: Option<f32>) -> QueryRecord {
501 QueryRecord {
502 query_id: "q".to_string(),
503 use_prefix: true,
504 ranked_ids: Vec::new(),
505 scores: Vec::new(),
506 first_relevant_rank: id_rank,
507 gold_raw_rank: raw_rank,
508 gold_raw_similarity: raw_sim,
509 }
510 }
511
512 #[test]
513 fn load_eval_set_parses_one_object_per_line() {
514 let dir = std::env::temp_dir();
515 let path = dir.join(format!("eval_set_{}.jsonl", Uuid::new_v4()));
516 let id_a = Uuid::new_v4();
517 let id_b = Uuid::new_v4();
518 let line1 = format!(
519 r#"{{"query_id":"q1","query":"kuzu choice","query_variant":"literal","seed_memory_id":"{id_a}","memory_type":"decision","relevant_memory_ids":["{id_a}"]}}"#
520 );
521 let line2 = format!(
522 r#"{{"query_id":"q2","query":"sync design","query_variant":"paraphrase","seed_memory_id":"{id_b}","memory_type":"architecture","relevant_memory_ids":["{id_b}","{id_a}"],"tags":["sync"]}}"#
523 );
524 let mut f = std::fs::File::create(&path).unwrap();
525 writeln!(f, "{line1}").unwrap();
526 writeln!(f, "{line2}").unwrap();
527 drop(f);
528
529 let queries = load_eval_set(&path).unwrap();
530 std::fs::remove_file(&path).ok();
531
532 assert_eq!(queries.len(), 2);
533 assert_eq!(queries[0].query_id, "q1");
534 assert_eq!(queries[0].seed_memory_id, id_a);
535 assert_eq!(queries[1].relevant_memory_ids.len(), 2);
536 assert_eq!(queries[1].tags, vec!["sync".to_string()]);
537 }
538
539 #[test]
540 fn load_eval_set_tolerates_blank_lines() {
541 let dir = std::env::temp_dir();
542 let path = dir.join(format!("eval_blank_{}.jsonl", Uuid::new_v4()));
543 let id = Uuid::new_v4();
544 let line = format!(
545 r#"{{"query_id":"q1","query":"x","query_variant":"v","seed_memory_id":"{id}","memory_type":"semantic","relevant_memory_ids":["{id}"]}}"#
546 );
547 std::fs::write(&path, format!("\n{line}\n\n")).unwrap();
548
549 let queries = load_eval_set(&path).unwrap();
550 std::fs::remove_file(&path).ok();
551
552 assert_eq!(queries.len(), 1);
553 }
554
555 #[test]
556 fn gate_sweep_emits_full_grid_and_monotone_recall() {
557 let records = vec![
558 record(Some(1), Some(1), Some(0.85)),
559 record(Some(2), Some(2), Some(0.62)),
560 record(Some(4), Some(4), Some(0.55)),
561 record(None, None, None),
562 ];
563
564 let report = gate_sweep(&records);
565
566 assert_eq!(report.frontier.len(), 41);
568 assert!((report.frontier.first().unwrap().threshold - 0.40).abs() < 1e-4);
569 assert!((report.frontier.last().unwrap().threshold - 0.80).abs() < 1e-4);
570
571 for w in report.frontier.windows(2) {
572 assert!(
573 w[0].recall_at_3 >= w[1].recall_at_3 - 1e-6,
574 "recall must not increase as the gate tightens"
575 );
576 }
577 }
578
579 #[test]
580 fn gate_sweep_recall_reflects_raw_rank_and_similarity() {
581 let records = vec![
582 record(Some(1), Some(1), Some(0.85)),
583 record(Some(2), Some(2), Some(0.62)),
584 record(Some(4), Some(4), Some(0.55)),
585 record(None, None, None),
586 ];
587
588 let report = gate_sweep(&records);
589
590 let at = |t: f32| {
591 report
592 .frontier
593 .iter()
594 .find(|p| (p.threshold - t).abs() < 1e-4)
595 .unwrap()
596 };
597
598 let p050 = at(0.50);
601 assert!((p050.recall_at_1 - 0.25).abs() < 1e-6, "recall@1 was {}", p050.recall_at_1);
602 assert!((p050.recall_at_3 - 0.5).abs() < 1e-6, "recall@3 was {}", p050.recall_at_3);
603 assert!((p050.recall_at_5 - 0.75).abs() < 1e-6, "recall@5 was {}", p050.recall_at_5);
604
605 let p070 = at(0.70);
607 assert!((p070.recall_at_1 - 0.25).abs() < 1e-6);
608 assert!((p070.recall_at_3 - 0.25).abs() < 1e-6);
609 assert!((p070.recall_at_5 - 0.25).abs() < 1e-6);
610 }
611
612 #[test]
613 fn gate_sweep_keeps_baseline_when_nothing_beats_it() {
614 let records = vec![record(Some(1), Some(1), Some(0.85))];
615 let report = gate_sweep(&records);
616 assert!((report.baseline_threshold - 0.59).abs() < 1e-6);
617 assert!(!report.chosen_beats_baseline);
618 assert!((report.chosen_threshold - 0.59).abs() < 1e-6);
619 }
620}