1pub mod bootstrap;
2pub mod caching_store;
3pub mod metrics;
4
5use std::collections::HashSet;
6use std::io::Write;
7use std::path::Path;
8use std::sync::Mutex;
9use std::sync::atomic::{AtomicUsize, Ordering};
10use std::thread;
11
12use anyhow::Result;
13use serde::{Deserialize, Serialize};
14use uuid::Uuid;
15
16use second_brain_core::embedding::Embedder;
17use second_brain_core::index::Indexes;
18use second_brain_core::query::{QueryEngine, QueryFilters, QueryRequest};
19use second_brain_core::retrieval::{RerankScorer, Reranker, RetrievalTunables, Retriever};
20use second_brain_core::sqlite_store::SqliteStore;
21use second_brain_core::store::{Store, StoreAdmin};
22
23pub struct HybridContext<'a> {
27 pub sqlite: &'a SqliteStore,
28 pub indexes: &'a Indexes,
29 pub reranker: Option<&'a Reranker>,
30 pub tunables: RetrievalTunables,
31}
32
33#[derive(Debug, Clone, Deserialize, Serialize)]
34pub struct EvalQuery {
35 pub query_id: String,
36 pub query: String,
37 pub query_variant: String,
38 pub seed_memory_id: Uuid,
39 pub memory_type: String,
40 pub relevant_memory_ids: Vec<Uuid>,
41 #[serde(default)]
42 pub note: Option<String>,
43 #[serde(default)]
44 pub tags: Vec<String>,
45 #[serde(default)]
47 pub source_project: Option<String>,
48}
49
50#[derive(Debug, Clone, Serialize, Deserialize)]
51pub struct QueryRecord {
52 pub query_id: String,
53 pub use_prefix: bool,
54 pub ranked_ids: Vec<Uuid>,
55 pub scores: Vec<f32>,
56 pub first_relevant_rank: Option<usize>,
57 pub gold_raw_rank: Option<usize>,
58 pub gold_raw_similarity: Option<f32>,
59 #[serde(default)]
62 pub latency_ms: Option<u64>,
63}
64
65#[derive(Debug, Clone, Serialize, Deserialize)]
66pub struct ArmMetrics {
67 pub recall_at_1: f32,
68 pub recall_at_3: f32,
69 pub recall_at_5: f32,
70 pub mrr: f32,
71 pub precision_at_5: f32,
72}
73
74#[derive(Debug, Clone, Serialize, Deserialize)]
75pub struct AggregateReport {
76 pub bare: ArmMetrics,
77 pub prefixed: ArmMetrics,
78 pub delta_recall_at_3_ci: (f32, f32),
79 pub delta_mrr_ci: (f32, f32),
80 pub gated_out_rate_bare: f32,
81 pub gated_out_rate_prefixed: f32,
82}
83
84#[derive(Debug, Clone, Serialize, Deserialize)]
85pub struct GatePoint {
86 pub threshold: f32,
87 pub recall_at_1: f32,
88 pub recall_at_3: f32,
89 pub recall_at_5: f32,
90 pub precision_proxy: f32,
91}
92
93#[derive(Debug, Clone, Serialize, Deserialize)]
94pub struct GateSweepReport {
95 pub frontier: Vec<GatePoint>,
96 pub baseline_threshold: f32,
97 pub chosen_threshold: f32,
98 pub chosen_beats_baseline: bool,
99}
100
101#[derive(Debug, Clone, Serialize, Deserialize)]
102pub struct CorpusEntry {
103 pub id: Uuid,
104 pub content: String,
105 pub memory_type: String,
106 pub created_at: String,
107 pub project_path: Option<String>,
108}
109
110const BASELINE_THRESHOLD: f32 = 0.59;
111
112pub fn load_eval_set(path: &Path) -> Result<Vec<EvalQuery>> {
113 let text = std::fs::read_to_string(path)?;
114 let mut out = Vec::new();
115 for line in text.lines() {
116 let trimmed = line.trim();
117 if trimmed.is_empty() {
118 continue;
119 }
120 out.push(serde_json::from_str(trimmed)?);
121 }
122 Ok(out)
123}
124
125pub fn run_arm(
126 store: &SqliteStore,
127 embedder: &Embedder,
128 queries: &[EvalQuery],
129 use_prefix: bool,
130 limit: usize,
131) -> Result<Vec<QueryRecord>> {
132 let engine = QueryEngine::new(store);
133 let mut records = Vec::with_capacity(queries.len());
134
135 for q in queries {
136 let embedding = if use_prefix {
137 embedder.embed_query(&q.query)?
138 } else {
139 embedder.embed(&q.query)?
140 };
141
142 let relevant: HashSet<Uuid> = q.relevant_memory_ids.iter().copied().collect();
143
144 let request = QueryRequest {
145 text: q.query.clone(),
146 embedding: embedding.clone(),
147 limit,
148 filters: QueryFilters {
149 entity_names: crate::link_query_entities(&q.query),
150 ..QueryFilters::default()
151 },
152 };
153 let started = std::time::Instant::now();
154 let results = engine.recall(&request)?;
155 let latency_ms = started.elapsed().as_millis() as u64;
156
157 let ranked_ids: Vec<Uuid> = results.iter().map(|r| r.memory.id).collect();
158 let scores: Vec<f32> = results.iter().map(|r| r.score).collect();
159 let first_relevant_rank = ranked_ids
160 .iter()
161 .position(|id| relevant.contains(id))
162 .map(|idx| idx + 1);
163
164 let raw = store.vector_search(&embedding, limit * 3)?;
165 let mut gold_raw_rank = None;
166 let mut gold_raw_similarity = None;
167 for (idx, (mem, sim)) in raw.iter().enumerate() {
168 if relevant.contains(&mem.id) {
169 gold_raw_rank = Some(idx + 1);
170 gold_raw_similarity = Some(*sim);
171 break;
172 }
173 }
174
175 records.push(QueryRecord {
176 query_id: q.query_id.clone(),
177 use_prefix,
178 ranked_ids,
179 scores,
180 first_relevant_rank,
181 gold_raw_rank,
182 gold_raw_similarity,
183 latency_ms: Some(latency_ms),
184 });
185 }
186
187 Ok(records)
188}
189
190pub struct EmbeddedQuery {
191 pub query: EvalQuery,
192 pub relevant: HashSet<Uuid>,
193 pub bare_embedding: Vec<f32>,
194 pub prefixed_embedding: Vec<f32>,
195}
196
197pub fn embed_all_queries(embedder: &Embedder, queries: &[EvalQuery]) -> Result<Vec<EmbeddedQuery>> {
201 let bare_texts: Vec<&str> = queries.iter().map(|q| q.query.as_str()).collect();
202 let bare = embedder.embed_batch(&bare_texts)?;
203
204 let prefixed_owned: Vec<String> = queries
205 .iter()
206 .map(|q| second_brain_core::embedding::query_prompt(&q.query))
207 .collect();
208 let prefixed_texts: Vec<&str> = prefixed_owned.iter().map(|s| s.as_str()).collect();
209 let prefixed = embedder.embed_batch(&prefixed_texts)?;
210
211 let mut out = Vec::with_capacity(queries.len());
212 for (i, q) in queries.iter().enumerate() {
213 out.push(EmbeddedQuery {
214 query: q.clone(),
215 relevant: q.relevant_memory_ids.iter().copied().collect(),
216 bare_embedding: bare[i].clone(),
217 prefixed_embedding: prefixed[i].clone(),
218 });
219 }
220 Ok(out)
221}
222
223fn record_for<S: Store + Sync>(
224 embedded: &EmbeddedQuery,
225 store: &S,
226 use_prefix: bool,
227 limit: usize,
228 hybrid: Option<&HybridContext<'_>>,
229) -> Result<QueryRecord> {
230 let embedding = if use_prefix {
231 &embedded.prefixed_embedding
232 } else {
233 &embedded.bare_embedding
234 };
235
236 let request = QueryRequest {
237 text: embedded.query.query.clone(),
238 embedding: embedding.clone(),
239 limit,
240 filters: QueryFilters {
241 entity_names: crate::link_query_entities(&embedded.query.query),
242 ..QueryFilters::default()
243 },
244 };
245 let started = std::time::Instant::now();
246 let results = match hybrid {
247 None => QueryEngine::new(store).recall(&request)?,
248 Some(ctx) => {
249 let retriever = Retriever {
250 tunables: ctx.tunables,
251 };
252 let pool = retriever.candidates(ctx.indexes, ctx.sqlite, &request)?;
253 retriever.rank(
254 store,
255 &request,
256 &pool,
257 ctx.reranker.map(|r| r as &dyn RerankScorer),
258 )?
259 }
260 };
261 let latency_ms = started.elapsed().as_millis() as u64;
262
263 let ranked_ids: Vec<Uuid> = results.iter().map(|r| r.memory.id).collect();
264 let scores: Vec<f32> = results.iter().map(|r| r.score).collect();
265 let first_relevant_rank = ranked_ids
266 .iter()
267 .position(|id| embedded.relevant.contains(id))
268 .map(|idx| idx + 1);
269
270 let raw = store.vector_search(embedding, limit * 3)?;
271 let mut gold_raw_rank = None;
272 let mut gold_raw_similarity = None;
273 for (idx, (mem, sim)) in raw.iter().enumerate() {
274 if embedded.relevant.contains(&mem.id) {
275 gold_raw_rank = Some(idx + 1);
276 gold_raw_similarity = Some(*sim);
277 break;
278 }
279 }
280
281 Ok(QueryRecord {
282 query_id: embedded.query.query_id.clone(),
283 use_prefix,
284 ranked_ids,
285 scores,
286 first_relevant_rank,
287 gold_raw_rank,
288 gold_raw_similarity,
289 latency_ms: Some(latency_ms),
290 })
291}
292
293pub fn run_arm_parallel<S: Store + Sync>(
298 store: &S,
299 embedded: &[EmbeddedQuery],
300 use_prefix: bool,
301 limit: usize,
302 hybrid: Option<&HybridContext<'_>>,
303) -> Result<Vec<QueryRecord>> {
304 let total = embedded.len();
305 if total == 0 {
306 return Ok(Vec::new());
307 }
308
309 let workers = thread::available_parallelism()
310 .map(|n| n.get().saturating_sub(1).max(1))
311 .unwrap_or(1);
312 let chunk_size = total.div_ceil(workers);
313
314 let done = AtomicUsize::new(0);
315 let collected: Mutex<Vec<(usize, QueryRecord)>> = Mutex::new(Vec::with_capacity(total));
316 let error: Mutex<Option<anyhow::Error>> = Mutex::new(None);
317
318 thread::scope(|scope| {
319 for chunk in 0..workers {
320 let start = chunk * chunk_size;
321 if start >= total {
322 break;
323 }
324 let end = (start + chunk_size).min(total);
325 let done = &done;
326 let collected = &collected;
327 let error = &error;
328 scope.spawn(move || {
329 let mut local: Vec<(usize, QueryRecord)> = Vec::with_capacity(end - start);
330 for (offset, eq) in embedded[start..end].iter().enumerate() {
331 if error.lock().unwrap().is_some() {
332 return;
333 }
334 match record_for(eq, store, use_prefix, limit, hybrid) {
335 Ok(rec) => local.push((start + offset, rec)),
336 Err(e) => {
337 *error.lock().unwrap() = Some(e);
338 return;
339 }
340 }
341 let n = done.fetch_add(1, Ordering::Relaxed) + 1;
343 if n.is_multiple_of(25) || n == total {
344 eprintln!(" {n}/{total} queries");
345 }
346 }
347 collected.lock().unwrap().extend(local);
348 });
349 }
350 });
351
352 if let Some(e) = error.into_inner().unwrap() {
353 return Err(e);
354 }
355
356 let mut indexed = collected.into_inner().unwrap();
357 indexed.sort_by_key(|(i, _)| *i);
358 Ok(indexed.into_iter().map(|(_, r)| r).collect())
359}
360
361pub fn aggregate(
362 bare: &[QueryRecord],
363 prefixed: &[QueryRecord],
364 relevant_sets: &std::collections::HashMap<String, HashSet<Uuid>>,
365) -> AggregateReport {
366 const AGG_SEED: u64 = 0x4B1D_C0DE;
367 let empty: HashSet<Uuid> = HashSet::new();
368
369 let per_query = |rec: &QueryRecord| -> (f32, f32, f32, f32, f32) {
370 let rel = relevant_sets.get(&rec.query_id).unwrap_or(&empty);
371 (
372 metrics::recall_at_k(&rec.ranked_ids, rel, 1),
373 metrics::recall_at_k(&rec.ranked_ids, rel, 3),
374 metrics::recall_at_k(&rec.ranked_ids, rel, 5),
375 metrics::mrr(&rec.ranked_ids, rel),
376 metrics::precision_at_k(&rec.ranked_ids, rel, 5),
377 )
378 };
379
380 let arm = |records: &[QueryRecord]| -> ArmMetrics {
381 if records.is_empty() {
382 return ArmMetrics {
383 recall_at_1: 0.0,
384 recall_at_3: 0.0,
385 recall_at_5: 0.0,
386 mrr: 0.0,
387 precision_at_5: 0.0,
388 };
389 }
390 let n = records.len() as f32;
391 let mut acc = (0.0, 0.0, 0.0, 0.0, 0.0);
392 for r in records {
393 let (r1, r3, r5, m, p5) = per_query(r);
394 acc.0 += r1;
395 acc.1 += r3;
396 acc.2 += r5;
397 acc.3 += m;
398 acc.4 += p5;
399 }
400 ArmMetrics {
401 recall_at_1: acc.0 / n,
402 recall_at_3: acc.1 / n,
403 recall_at_5: acc.2 / n,
404 mrr: acc.3 / n,
405 precision_at_5: acc.4 / n,
406 }
407 };
408
409 let bare_idx: std::collections::HashMap<&str, &QueryRecord> =
410 bare.iter().map(|r| (r.query_id.as_str(), r)).collect();
411
412 let mut delta_r3 = Vec::new();
413 let mut delta_mrr = Vec::new();
414 for p_rec in prefixed {
415 if let Some(b_rec) = bare_idx.get(p_rec.query_id.as_str()) {
416 let (_, p_r3, _, p_mrr, _) = per_query(p_rec);
417 let (_, b_r3, _, b_mrr, _) = per_query(b_rec);
418 delta_r3.push(p_r3 - b_r3);
419 delta_mrr.push(p_mrr - b_mrr);
420 }
421 }
422
423 let gated_rate = |records: &[QueryRecord]| -> f32 {
424 let flags: Vec<bool> = records
425 .iter()
426 .map(|r| match (r.gold_raw_rank, r.gold_raw_similarity) {
427 (Some(_), Some(sim)) => sim < BASELINE_THRESHOLD,
428 _ => false,
429 })
430 .collect();
431 metrics::gated_out_rate(&flags)
432 };
433
434 AggregateReport {
435 bare: arm(bare),
436 prefixed: arm(prefixed),
437 delta_recall_at_3_ci: bootstrap::paired_bootstrap_ci(&delta_r3, 10000, 0.95, AGG_SEED),
438 delta_mrr_ci: bootstrap::paired_bootstrap_ci(&delta_mrr, 10000, 0.95, AGG_SEED),
439 gated_out_rate_bare: gated_rate(bare),
440 gated_out_rate_prefixed: gated_rate(prefixed),
441 }
442}
443
444pub fn gate_sweep(prefixed: &[QueryRecord]) -> GateSweepReport {
445 const GRID_SEED: u64 = 0x5EED_6A7E;
446
447 let recalled_at = |rec: &QueryRecord, k: usize, t: f32| -> f32 {
448 match (rec.gold_raw_rank, rec.gold_raw_similarity) {
449 (Some(rank), Some(sim)) if rank <= k && sim >= t => 1.0,
450 _ => 0.0,
451 }
452 };
453
454 let mean = |vals: &[f32]| -> f32 {
455 if vals.is_empty() {
456 0.0
457 } else {
458 vals.iter().sum::<f32>() / vals.len() as f32
459 }
460 };
461
462 let baseline_r3: Vec<f32> = prefixed
463 .iter()
464 .map(|r| recalled_at(r, 3, BASELINE_THRESHOLD))
465 .collect();
466 let baseline_r3_mean = mean(&baseline_r3);
467
468 let mut frontier = Vec::with_capacity(41);
469 let mut chosen_threshold = BASELINE_THRESHOLD;
470 let mut chosen_beats_baseline = false;
471 let mut best_recall_at_3 = baseline_r3_mean;
472
473 for step in 0..=40u32 {
474 let t = 0.40 + step as f32 * 0.01;
475
476 let r1: Vec<f32> = prefixed.iter().map(|r| recalled_at(r, 1, t)).collect();
477 let r3: Vec<f32> = prefixed.iter().map(|r| recalled_at(r, 3, t)).collect();
478 let r5: Vec<f32> = prefixed.iter().map(|r| recalled_at(r, 5, t)).collect();
479
480 let recall_at_3 = mean(&r3);
481 let recall_at_5 = mean(&r5);
482
483 frontier.push(GatePoint {
484 threshold: t,
485 recall_at_1: mean(&r1),
486 recall_at_3,
487 recall_at_5,
488 precision_proxy: recall_at_5 / 5.0,
489 });
490
491 let deltas: Vec<f32> = r3
492 .iter()
493 .zip(baseline_r3.iter())
494 .map(|(t_val, b_val)| t_val - b_val)
495 .collect();
496 let (lo, _hi) = bootstrap::paired_bootstrap_ci(&deltas, 2000, 0.95, GRID_SEED);
497
498 if lo > 0.0 && recall_at_3 > best_recall_at_3 + 1e-6 {
499 best_recall_at_3 = recall_at_3;
500 chosen_threshold = t;
501 chosen_beats_baseline = true;
502 }
503 }
504
505 GateSweepReport {
506 frontier,
507 baseline_threshold: BASELINE_THRESHOLD,
508 chosen_threshold,
509 chosen_beats_baseline,
510 }
511}
512
513#[derive(Debug, Clone, Serialize, Deserialize)]
518pub struct ScopedPair {
519 pub query_id: String,
520 pub gold: f32,
521 pub foreign_project: String,
522 pub foreign_best: f32,
523}
524
525#[derive(Debug, Clone, Serialize, Deserialize)]
526pub struct FloorPoint {
527 pub threshold: f32,
528 pub gold_admitted: f32,
530 pub foreign_empty: f32,
532}
533
534#[derive(Debug, Clone, Serialize, Deserialize)]
535pub struct FloorSweepReport {
536 pub pairs: usize,
537 pub frontier: Vec<FloorPoint>,
538 pub chosen_threshold: f32,
543}
544
545pub const FLOOR_GOLD_TARGET: f32 = 0.99;
548
549pub fn exhausted_floor_sweep(pairs: &[ScopedPair]) -> FloorSweepReport {
550 let n = pairs.len().max(1) as f32;
551 let mut frontier = Vec::with_capacity(41);
552 let mut chosen = 0.40_f32;
553 for step in 0..=40u32 {
554 let t = 0.40 + step as f32 * 0.01;
555 let gold_admitted = pairs.iter().filter(|p| p.gold >= t).count() as f32 / n;
556 let foreign_empty = pairs.iter().filter(|p| p.foreign_best < t).count() as f32 / n;
557 if gold_admitted >= FLOOR_GOLD_TARGET {
558 chosen = t;
559 }
560 frontier.push(FloorPoint {
561 threshold: t,
562 gold_admitted,
563 foreign_empty,
564 });
565 }
566 FloorSweepReport {
567 pairs: pairs.len(),
568 frontier,
569 chosen_threshold: chosen,
570 }
571}
572
573pub fn scoped_pairs(
578 store: &SqliteStore,
579 queries: &[EmbeddedQuery],
580 projects: &[String],
581) -> Result<Vec<ScopedPair>> {
582 let mut pairs = Vec::new();
583 for (i, q) in queries.iter().enumerate() {
584 let Some(own) = q.query.source_project.as_deref() else {
585 continue;
586 };
587 let Some(gold) = store.get_memory_with_embedding(q.query.seed_memory_id)? else {
588 continue;
589 };
590 if gold.embedding.is_empty() || gold.tier != second_brain_core::schema::Tier::Hot {
591 continue;
592 }
593 let foreign: Vec<&String> = projects.iter().filter(|p| p.as_str() != own).collect();
594 if foreign.is_empty() {
595 continue;
596 }
597 let foreign_project = foreign[i % foreign.len()].clone();
598 let emb = &q.prefixed_embedding;
599 let foreign_best = store
600 .filtered_vector_search(emb, None, None, Some(&foreign_project), 1)?
601 .first()
602 .map(|(_, score)| *score)
603 .unwrap_or(0.0);
604 pairs.push(ScopedPair {
605 query_id: q.query.query_id.clone(),
606 gold: second_brain_core::retrieval::cosine(emb, &gold.embedding),
607 foreign_project,
608 foreign_best,
609 });
610 }
611 Ok(pairs)
612}
613
614pub fn extract_corpus(store: &SqliteStore, out: &Path) -> Result<usize> {
615 let memories = store.all_memories_with_embeddings()?;
616 let mut file = std::fs::File::create(out)?;
617 let mut count = 0;
618 for m in memories.iter().filter(|m| {
621 m.tier == second_brain_core::schema::Tier::Hot
622 && m.superseded_by.is_none()
623 && !m.embedding.is_empty()
624 }) {
625 let entry = CorpusEntry {
626 id: m.id,
627 content: m.content.clone(),
628 memory_type: format!("{:?}", m.memory_type).to_lowercase(),
629 created_at: m.created_at.to_rfc3339(),
630 project_path: m.project_path.clone(),
631 };
632 writeln!(file, "{}", serde_json::to_string(&entry)?)?;
633 count += 1;
634 }
635 Ok(count)
636}
637
638#[cfg(test)]
639mod tests {
640 use super::*;
641 use std::io::Write;
642
643 fn pair(gold: f32, foreign_best: f32) -> ScopedPair {
644 ScopedPair {
645 query_id: "q".to_string(),
646 gold,
647 foreign_project: "p".to_string(),
648 foreign_best,
649 }
650 }
651
652 #[test]
653 fn the_floor_sweep_picks_the_threshold_separating_gold_from_foreign() {
654 let pairs = [pair(0.70, 0.50), pair(0.65, 0.52), pair(0.60, 0.48)];
655 let report = exhausted_floor_sweep(&pairs);
656 assert_eq!(report.pairs, 3);
657 let chosen = report.chosen_threshold;
658 assert!(
659 (chosen - 0.60).abs() < 1e-4,
660 "the highest floor that still admits every gold memory: {chosen}"
661 );
662 let at = |t: f32| {
663 report
664 .frontier
665 .iter()
666 .find(|p| (p.threshold - t).abs() < 1e-4)
667 .unwrap()
668 .clone()
669 };
670 assert!((at(0.66).gold_admitted - 1.0 / 3.0).abs() < 1e-4);
671 assert!((at(0.50).foreign_empty - 1.0 / 3.0).abs() < 1e-4);
672 }
673
674 fn record(
675 id_rank: Option<usize>,
676 raw_rank: Option<usize>,
677 raw_sim: Option<f32>,
678 ) -> QueryRecord {
679 QueryRecord {
680 query_id: "q".to_string(),
681 use_prefix: true,
682 ranked_ids: Vec::new(),
683 scores: Vec::new(),
684 first_relevant_rank: id_rank,
685 gold_raw_rank: raw_rank,
686 gold_raw_similarity: raw_sim,
687 latency_ms: None,
688 }
689 }
690
691 #[test]
692 fn load_eval_set_parses_one_object_per_line() {
693 let dir = std::env::temp_dir();
694 let path = dir.join(format!("eval_set_{}.jsonl", Uuid::new_v4()));
695 let id_a = Uuid::new_v4();
696 let id_b = Uuid::new_v4();
697 let line1 = format!(
698 r#"{{"query_id":"q1","query":"kuzu choice","query_variant":"literal","seed_memory_id":"{id_a}","memory_type":"decision","relevant_memory_ids":["{id_a}"]}}"#
699 );
700 let line2 = format!(
701 r#"{{"query_id":"q2","query":"sync design","query_variant":"paraphrase","seed_memory_id":"{id_b}","memory_type":"architecture","relevant_memory_ids":["{id_b}","{id_a}"],"tags":["sync"]}}"#
702 );
703 let mut f = std::fs::File::create(&path).unwrap();
704 writeln!(f, "{line1}").unwrap();
705 writeln!(f, "{line2}").unwrap();
706 drop(f);
707
708 let queries = load_eval_set(&path).unwrap();
709 std::fs::remove_file(&path).ok();
710
711 assert_eq!(queries.len(), 2);
712 assert_eq!(queries[0].query_id, "q1");
713 assert_eq!(queries[0].seed_memory_id, id_a);
714 assert_eq!(queries[1].relevant_memory_ids.len(), 2);
715 assert_eq!(queries[1].tags, vec!["sync".to_string()]);
716 }
717
718 #[test]
719 fn load_eval_set_tolerates_blank_lines() {
720 let dir = std::env::temp_dir();
721 let path = dir.join(format!("eval_blank_{}.jsonl", Uuid::new_v4()));
722 let id = Uuid::new_v4();
723 let line = format!(
724 r#"{{"query_id":"q1","query":"x","query_variant":"v","seed_memory_id":"{id}","memory_type":"semantic","relevant_memory_ids":["{id}"]}}"#
725 );
726 std::fs::write(&path, format!("\n{line}\n\n")).unwrap();
727
728 let queries = load_eval_set(&path).unwrap();
729 std::fs::remove_file(&path).ok();
730
731 assert_eq!(queries.len(), 1);
732 }
733
734 #[test]
735 fn gate_sweep_emits_full_grid_and_monotone_recall() {
736 let records = vec![
737 record(Some(1), Some(1), Some(0.85)),
738 record(Some(2), Some(2), Some(0.62)),
739 record(Some(4), Some(4), Some(0.55)),
740 record(None, None, None),
741 ];
742
743 let report = gate_sweep(&records);
744
745 assert_eq!(report.frontier.len(), 41);
747 assert!((report.frontier.first().unwrap().threshold - 0.40).abs() < 1e-4);
748 assert!((report.frontier.last().unwrap().threshold - 0.80).abs() < 1e-4);
749
750 for w in report.frontier.windows(2) {
751 assert!(
752 w[0].recall_at_3 >= w[1].recall_at_3 - 1e-6,
753 "recall must not increase as the gate tightens"
754 );
755 }
756 }
757
758 #[test]
759 fn gate_sweep_recall_reflects_raw_rank_and_similarity() {
760 let records = vec![
761 record(Some(1), Some(1), Some(0.85)),
762 record(Some(2), Some(2), Some(0.62)),
763 record(Some(4), Some(4), Some(0.55)),
764 record(None, None, None),
765 ];
766
767 let report = gate_sweep(&records);
768
769 let at = |t: f32| {
770 report
771 .frontier
772 .iter()
773 .find(|p| (p.threshold - t).abs() < 1e-4)
774 .unwrap()
775 };
776
777 let p050 = at(0.50);
780 assert!(
781 (p050.recall_at_1 - 0.25).abs() < 1e-6,
782 "recall@1 was {}",
783 p050.recall_at_1
784 );
785 assert!(
786 (p050.recall_at_3 - 0.5).abs() < 1e-6,
787 "recall@3 was {}",
788 p050.recall_at_3
789 );
790 assert!(
791 (p050.recall_at_5 - 0.75).abs() < 1e-6,
792 "recall@5 was {}",
793 p050.recall_at_5
794 );
795
796 let p070 = at(0.70);
798 assert!((p070.recall_at_1 - 0.25).abs() < 1e-6);
799 assert!((p070.recall_at_3 - 0.25).abs() < 1e-6);
800 assert!((p070.recall_at_5 - 0.25).abs() < 1e-6);
801 }
802
803 #[test]
804 fn gate_sweep_keeps_baseline_when_nothing_beats_it() {
805 let records = vec![record(Some(1), Some(1), Some(0.85))];
806 let report = gate_sweep(&records);
807 assert!((report.baseline_threshold - 0.59).abs() < 1e-6);
808 assert!(!report.chosen_beats_baseline);
809 assert!((report.chosen_threshold - 0.59).abs() < 1e-6);
810 }
811}