Skip to main content

second_brain_core/retrieval/
mod.rs

1pub mod blend;
2mod onnx_rerank;
3pub mod raw_fallback;
4pub mod rerank;
5pub mod rrf;
6
7pub use rerank::{RerankScorer, RerankTimeoutError, Reranker};
8
9use std::collections::HashMap;
10
11use anyhow::Result;
12use uuid::Uuid;
13
14use crate::index::Indexes;
15use crate::query::{QueryRequest, QueryResult};
16use crate::schema::Memory;
17use crate::sqlite_store::SqliteStore;
18use crate::store::Store;
19use blend::BlendWeights;
20
21/// Ceiling on the widened BM25 fetch a scoped query may ask for.
22const SCOPED_LEXICAL_MAX_K: usize = 5000;
23
24/// Dense and lexical candidate lists, each best-first as (memory id, score).
25#[derive(Debug, Clone, Default)]
26pub struct Candidates {
27    pub dense: Vec<(Uuid, f32)>,
28    pub lexical: Vec<(Uuid, f32)>,
29    /// Set when generation scanned the caller's entire scope, so this pool is
30    /// every memory the filter admits rather than a relevance-ranked slice of
31    /// the store.
32    pub scope_exhausted: bool,
33    /// For a memory one of whose chunks hit the query: (chunk ord, chunk
34    /// cosine) of its best chunk. Rerank scores the window around that ord.
35    pub best_chunk: BestChunks,
36}
37
38/// Memory id to (chunk ord, chunk cosine) of its best-matching chunk.
39pub type BestChunks = HashMap<Uuid, (usize, f32)>;
40
41impl Candidates {
42    pub fn new(dense: Vec<(Uuid, f32)>, lexical: Vec<(Uuid, f32)>) -> Self {
43        Self {
44            dense,
45            lexical,
46            scope_exhausted: false,
47            best_chunk: HashMap::new(),
48        }
49    }
50}
51
52/// Folds chunk hits into the dense list: a memory scores the better of its
53/// own vector and its best chunk, and the list is re-sorted and cut to `k`.
54/// Without chunk hits the list comes back untouched, so a store with no
55/// chunks ranks exactly as before chunks existed.
56fn merge_chunk_hits(
57    dense: Vec<(Uuid, f32)>,
58    chunk_hits: Vec<(Uuid, f32, usize)>,
59    k: usize,
60) -> (Vec<(Uuid, f32)>, BestChunks) {
61    if chunk_hits.is_empty() {
62        return (dense, HashMap::new());
63    }
64    let best_chunk: BestChunks = chunk_hits
65        .iter()
66        .map(|(id, similarity, ord)| (*id, (*ord, *similarity)))
67        .collect();
68    let mut merged = dense;
69    let listed: std::collections::HashSet<Uuid> = merged.iter().map(|(id, _)| *id).collect();
70    for (id, similarity) in merged.iter_mut() {
71        if let Some((_, chunk)) = best_chunk.get(id) {
72            *similarity = similarity.max(*chunk);
73        }
74    }
75    merged.extend(
76        chunk_hits
77            .iter()
78            .filter(|(id, _, _)| !listed.contains(id))
79            .map(|(id, similarity, _)| (*id, *similarity)),
80    );
81    merged.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
82    merged.truncate(k);
83    (merged, best_chunk)
84}
85
86/// `chunk_search` keeps one hit per memory, so it is asked for this many
87/// times `dense_k` chunk hits to leave room for memories with several.
88const CHUNK_K_FACTOR: usize = 3;
89
90#[derive(Debug, Clone, Copy)]
91pub struct RetrievalTunables {
92    pub dense_k: usize,
93    pub lexical_k: usize,
94    pub fuse_top: usize,
95    pub rerank_gate: f32,
96    pub cosine_gate: f32,
97    /// Membership floor, in cosine, for a scope generation scanned whole. It
98    /// sits under `cosine_gate` because naming a scope is the caller
99    /// asserting the haystack, so a weak real match must still come back.
100    /// Calibrated on the gold set (`sb-recall-eval floor-sweep`,
101    /// evals/exhausted-floor.md) as the highest floor that keeps 99% of gold
102    /// memories answerable from their own project. With the BGE query
103    /// instruction applied, the best match in an unrelated project has a
104    /// median cosine of 0.65, so this floor rarely empties an unrelated
105    /// scope; it only drops noise.
106    pub exhausted_cosine_gate: f32,
107    /// How many of each candidate list's top entries always enter the fused
108    /// list, whatever RRF makes of them. 0 is plain RRF.
109    pub fusion_floor: usize,
110    /// Use sentence chunks: as dense candidates (a memory scores its best
111    /// chunk when that beats its own vector), as the cosine primary, and as a
112    /// rerank window scored beside the whole memory. False ranks exactly as
113    /// before chunks existed.
114    pub chunk_rerank: bool,
115    /// Also rerank the whole memory beside its chunk window and keep the
116    /// better score. False scores a chunked memory on its window alone, which
117    /// keeps the rerank call the size it was before chunks.
118    pub rerank_whole_chunked: bool,
119    /// Answer a reranked query with nothing unless some result reaches
120    /// `blend::ABSTAIN_RERANK_BAR`; an answer that has one keeps every result
121    /// past `rerank_gate`. A caller that abstains must treat an empty answer
122    /// as final rather than fall back to an unranked search.
123    pub abstain: bool,
124    /// Order results by `blend::source_rank` before score. On the reranked
125    /// path only results reaching `blend::ABSTAIN_RERANK_BAR` are ordered so;
126    /// the rest follow them by score.
127    pub source_order: bool,
128    pub weights: BlendWeights,
129}
130
131impl Default for RetrievalTunables {
132    fn default() -> Self {
133        Self {
134            dense_k: 150,
135            lexical_k: 150,
136            fuse_top: 100,
137            rerank_gate: 0.30,
138            cosine_gate: blend::MIN_SIMILARITY,
139            exhausted_cosine_gate: blend::EXHAUSTED_MIN_SIMILARITY,
140            fusion_floor: 0,
141            chunk_rerank: true,
142            rerank_whole_chunked: true,
143            abstain: false,
144            source_order: false,
145            weights: BlendWeights::default(),
146        }
147    }
148}
149
150#[derive(Default)]
151pub struct Retriever {
152    pub tunables: RetrievalTunables,
153}
154
155/// Where a project-first answer came from.
156#[derive(Debug, Clone, Copy, PartialEq, Eq)]
157pub enum ProjectScope {
158    Project,
159    /// Nothing in the project cleared the bar, so the answer, possibly empty,
160    /// is from the wide search.
161    OtherProjects,
162}
163
164#[derive(Debug, Clone)]
165pub struct ProjectFirst {
166    pub results: Vec<QueryResult>,
167    pub scope: ProjectScope,
168}
169
170pub fn cosine(a: &[f32], b: &[f32]) -> f32 {
171    if a.len() != b.len() || a.is_empty() {
172        return 0.0;
173    }
174    let dot: f32 = a.iter().zip(b).map(|(x, y)| x * y).sum();
175    let na: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
176    let nb: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
177    if na == 0.0 || nb == 0.0 {
178        return 0.0;
179    }
180    dot / (na * nb)
181}
182
183impl Retriever {
184    /// Index-side candidate generation: dense and lexical top-k, both
185    /// best-first. Separated from rank() so the eval harness can feed rank()
186    /// through its CachingStore while candidates come from real indexes.
187    pub fn candidates(
188        &self,
189        indexes: &Indexes,
190        store: &SqliteStore,
191        request: &QueryRequest,
192    ) -> Result<Candidates> {
193        let source = request.filters.source.as_deref();
194        let memory_type = request.filters.memory_type;
195        let project_path = request.filters.project_path.as_deref();
196        if source.is_some() || memory_type.is_some() || project_path.is_some() {
197            return self.scoped_candidates(
198                indexes,
199                store,
200                request,
201                source,
202                memory_type,
203                project_path,
204            );
205        }
206        let dense_k = self.tunables.dense_k;
207        let dense = indexes.vector_search(store, &request.embedding, dense_k)?;
208        let chunk_hits = if self.tunables.chunk_rerank {
209            indexes.chunk_search(store, &request.embedding, dense_k * CHUNK_K_FACTOR)?
210        } else {
211            Vec::new()
212        };
213        let (dense, best_chunk) = merge_chunk_hits(dense, chunk_hits, dense_k);
214        let lexical = indexes.bm25_search(&request.text, self.tunables.lexical_k)?;
215        Ok(Candidates {
216            best_chunk,
217            ..Candidates::new(dense, lexical)
218        })
219    }
220
221    /// Candidate generation for a query scoped by source, type, and/or
222    /// project.
223    ///
224    /// The unfiltered path picks its shortlist before the filter is known, so
225    /// a memory outside the global top-k cannot be reached however narrow the
226    /// scope is; the scope must therefore constrain generation rather than
227    /// trim its output. Dense becomes an exhaustive cosine scan of the
228    /// matching subset, which is affordable because the store already
229    /// brute-forces `vector_search` over the whole table, making a subset scan
230    /// strictly cheaper. Lexical over-fetches and intersects instead, because
231    /// the tantivy schema carries only body text and cannot filter on source
232    /// or project.
233    #[allow(clippy::too_many_arguments)]
234    fn scoped_candidates(
235        &self,
236        indexes: &Indexes,
237        store: &SqliteStore,
238        request: &QueryRequest,
239        source: Option<&str>,
240        memory_type: Option<crate::schema::MemoryType>,
241        project_path: Option<&str>,
242    ) -> Result<Candidates> {
243        let dense: Vec<(Uuid, f32)> = store
244            .filtered_vector_search(
245                &request.embedding,
246                source,
247                memory_type,
248                project_path,
249                self.tunables.dense_k,
250            )?
251            .into_iter()
252            .map(|(memory, similarity)| (memory.id, similarity))
253            .collect();
254
255        // Exhaustion is judged on memories before chunk hits merge in: it
256        // says the scope held fewer memories than dense_k, which chunks of
257        // those same memories cannot change.
258        let exhausted = dense.len() < self.tunables.dense_k;
259        let chunk_hits = if self.tunables.chunk_rerank {
260            store.filtered_chunk_search(
261                &request.embedding,
262                source,
263                memory_type,
264                project_path,
265                self.tunables.dense_k,
266            )?
267        } else {
268            Vec::new()
269        };
270        let (dense, best_chunk) = merge_chunk_hits(dense, chunk_hits, self.tunables.dense_k);
271
272        // A short list means the scan ran out of scope rather than out of room,
273        // so dense already holds every memory the filter admits and no lexical
274        // sweep can add one. This is the common case for a narrow scope, and it
275        // keeps the scope-wide id read below off that path entirely.
276        if exhausted {
277            return Ok(Candidates {
278                dense,
279                lexical: Vec::new(),
280                scope_exhausted: true,
281                best_chunk,
282            });
283        }
284
285        // Widen the sweep by the scope's inverse selectivity: a scope holding
286        // one row in a thousand needs a proportionally deeper BM25 list before
287        // any in-scope document appears at all. The cap bounds the worst case,
288        // and a scope too narrow to survive it loses nothing, because the dense
289        // side of this pair already scanned it exhaustively.
290        let scoped = store
291            .scoped_count(source, memory_type, project_path)?
292            .max(1);
293        let total = <SqliteStore as Store>::memory_count(store)?.max(scoped);
294        let widen = total.div_ceil(scoped);
295        let k = self
296            .tunables
297            .lexical_k
298            .saturating_mul(widen)
299            .min(SCOPED_LEXICAL_MAX_K);
300
301        // Membership is resolved against the fetched hits, never by reading the
302        // scope out, so this stays bounded by k however large the scope is.
303        let hits = indexes.bm25_search(&request.text, k)?;
304        let hit_ids: Vec<Uuid> = hits.iter().map(|(id, _)| *id).collect();
305        let in_scope = store.retain_in_scope(&hit_ids, source, memory_type, project_path)?;
306        let lexical = hits
307            .into_iter()
308            .filter(|(id, _)| in_scope.contains(id))
309            .take(self.tunables.lexical_k)
310            .collect();
311        Ok(Candidates {
312            best_chunk,
313            ..Candidates::new(dense, lexical)
314        })
315    }
316
317    /// Fuse, gate, and blend pre-computed candidates. With a scorer the
318    /// primary term is its 0..1 cross-encoder score gated at rerank_gate;
319    /// without one it is cosine similarity gated at the legacy cosine_gate,
320    /// recomputed from stored embeddings for candidates that arrived through
321    /// the lexical list only.
322    pub fn rank<S: Store + ?Sized>(
323        &self,
324        store: &S,
325        request: &QueryRequest,
326        pool: &Candidates,
327        reranker: Option<&dyn RerankScorer>,
328    ) -> Result<Vec<QueryResult>> {
329        let mut ranked = self.ranked_shortlist(store, request, pool, reranker)?;
330        ranked.truncate(request.limit);
331        Ok(ranked)
332    }
333
334    /// `rank` before its cut to `request.limit`: at most `3 * request.limit`
335    /// results, blended from the top `3 * request.limit` candidates by
336    /// primary score, in answer order.
337    fn ranked_shortlist<S: Store + ?Sized>(
338        &self,
339        store: &S,
340        request: &QueryRequest,
341        pool: &Candidates,
342        reranker: Option<&dyn RerankScorer>,
343    ) -> Result<Vec<QueryResult>> {
344        let dense_ids: Vec<Uuid> = pool.dense.iter().map(|(id, _)| *id).collect();
345        let lexical_ids: Vec<Uuid> = pool.lexical.iter().map(|(id, _)| *id).collect();
346        let fused = rrf::fuse_with_floor(
347            &[&dense_ids, &lexical_ids],
348            rrf::RRF_K,
349            self.tunables.fuse_top,
350            self.tunables.fusion_floor,
351        );
352        let best_chunk = |id: &Uuid| {
353            self.tunables
354                .chunk_rerank
355                .then(|| pool.best_chunk.get(id).copied())
356                .flatten()
357        };
358        let dense_sim: HashMap<Uuid, f32> = pool.dense.iter().copied().collect();
359
360        let mut candidates: Vec<Memory> = Vec::new();
361        for (id, _) in &fused {
362            let Some(memory) = store.get_memory_with_embedding(*id)? else {
363                continue;
364            };
365            // The indexes lag the store by an outbox drain, so a candidate
366            // can be a row recall must no longer show.
367            if memory.tier != crate::schema::Tier::Hot || memory.superseded_by.is_some() {
368                continue;
369            }
370            if let Some(min_conf) = request.filters.min_confidence
371                && memory.confidence < min_conf
372            {
373                continue;
374            }
375            if let Some(ref source) = request.filters.source
376                && !store.source_admits(&memory, source)?
377            {
378                continue;
379            }
380            if let Some(ref mt) = request.filters.memory_type
381                && &memory.memory_type != mt
382            {
383                continue;
384            }
385            candidates.push(memory);
386        }
387
388        let cosine_primaries = |cands: &[Memory]| -> Vec<f32> {
389            cands
390                .iter()
391                .map(|m| {
392                    let whole = dense_sim
393                        .get(&m.id)
394                        .copied()
395                        .unwrap_or_else(|| cosine(&request.embedding, &m.embedding));
396                    best_chunk(&m.id).map_or(whole, |(_, chunk)| whole.max(chunk))
397                })
398                .collect()
399        };
400
401        // A saturated rerank pool, or an overloaded remote scorer, must
402        // degrade this query, not fail it: the caller gets the non-reranked
403        // hybrid answer (still RRF-fused, still graph/recency blended)
404        // instead of an error, the same way the recall route already
405        // degrades to the legacy engine on other hybrid-path failures.
406        let (primaries, reranked) = match reranker {
407            Some(r) => {
408                // Windows follow the whole memories in the same call; each
409                // candidate's primary is the better of its two scores.
410                let mut windows: Vec<(usize, String)> = Vec::new();
411                for (i, m) in candidates.iter().enumerate() {
412                    if let Some((ord, _)) = best_chunk(&m.id)
413                        && let Some(window) = store.chunk_window(m.id, ord)?
414                    {
415                        windows.push((i, window));
416                    }
417                }
418                let whole_too = self.tunables.rerank_whole_chunked;
419                let mut firsts: Vec<&str> = candidates.iter().map(|m| m.content.as_str()).collect();
420                if !whole_too {
421                    for (i, w) in &windows {
422                        firsts[*i] = w.as_str();
423                    }
424                }
425                let docs: Vec<&str> = firsts
426                    .into_iter()
427                    .chain(
428                        windows
429                            .iter()
430                            .filter(|_| whole_too)
431                            .map(|(_, w)| w.as_str()),
432                    )
433                    .collect();
434                match r.rerank(&request.text, &docs) {
435                    // Zipping a short score list would silently drop the
436                    // unscored tail of the candidates.
437                    Ok(scores) if scores.len() != docs.len() => anyhow::bail!(
438                        "rerank scorer returned {} scores for {} documents",
439                        scores.len(),
440                        docs.len()
441                    ),
442                    Ok(mut scores) => {
443                        let window_scores = scores.split_off(candidates.len());
444                        for ((i, _), score) in windows.iter().zip(window_scores) {
445                            scores[*i] = scores[*i].max(score);
446                        }
447                        (scores, true)
448                    }
449                    Err(e) if e.downcast_ref::<RerankTimeoutError>().is_some() => {
450                        tracing::warn!(
451                            "rerank unavailable ({e}); falling back to the \
452                             non-reranked hybrid path for this query"
453                        );
454                        (cosine_primaries(&candidates), false)
455                    }
456                    Err(e) => return Err(e),
457                }
458            }
459            None => (cosine_primaries(&candidates), false),
460        };
461        // An exhausted scope is every memory the caller's filter admits, so a
462        // gate calibrated for slicing a whole-store ranking would blank the
463        // answer to a question that named its own haystack; a cross-encoder
464        // is no help there either, because it scores a weak real match
465        // ("job title" against a memory stating the job) below outright
466        // nonsense. Membership therefore falls back to a relaxed cosine floor
467        // on that path, while the reranked primary still orders the answer.
468        let cosines = cosine_primaries(&candidates);
469        let admitted = |i: usize, primary: f32| -> bool {
470            if pool.scope_exhausted {
471                cosines[i] >= self.tunables.exhausted_cosine_gate
472            } else if reranked {
473                primary >= self.tunables.rerank_gate
474            } else {
475                primary >= self.tunables.cosine_gate
476            }
477        };
478
479        let query_entity_ids = blend::resolve_query_entities(store, &request.filters.entity_names);
480
481        // Blend over at most limit*3 candidates ranked by the primary term,
482        // mirroring the legacy engine's vector_search(limit*3) pool. Graph and
483        // recency carry enough weight that blending a wider fused pool lets
484        // recent, well-connected memories displace strong matches wholesale.
485        let mut shortlist: Vec<(usize, Memory, f32)> = candidates
486            .into_iter()
487            .zip(primaries)
488            .enumerate()
489            .map(|(i, (memory, primary))| (i, memory, primary))
490            .collect();
491        shortlist.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap_or(std::cmp::Ordering::Equal));
492        shortlist.truncate(request.limit * 3);
493
494        let mut scored: Vec<QueryResult> = Vec::new();
495        for (i, memory, primary) in shortlist {
496            // Membership is gated on the primary term alone, exactly like the
497            // legacy path: graph and recency only re-rank.
498            if !admitted(i, primary) {
499                continue;
500            }
501            let recency = blend::compute_recency(&memory);
502            let graph =
503                blend::compute_graph_relevance(store, &memory, &query_entity_ids).unwrap_or(0.0);
504            let score = blend::blend(primary, graph, recency, self.tunables.weights);
505            let score = blend::apply_project_affinity(
506                score,
507                &memory,
508                request.filters.project_path.as_deref(),
509            );
510            scored.push(QueryResult {
511                memory,
512                score,
513                relevance: primary,
514                path: Vec::new(),
515            });
516        }
517
518        // An exhausted named scope keeps its relaxed floor, for the reason
519        // given at `admitted`.
520        if self.tunables.abstain
521            && reranked
522            && !pool.scope_exhausted
523            && !scored
524                .iter()
525                .any(|r| r.relevance >= blend::ABSTAIN_RERANK_BAR)
526        {
527            scored.clear();
528        }
529
530        // A weak note ranked by its source alone would push a strong raw
531        // match out of the answer.
532        let source_rank = |r: &QueryResult| {
533            if !self.tunables.source_order {
534                (false, 0)
535            } else if reranked && r.relevance < blend::ABSTAIN_RERANK_BAR {
536                (true, 0)
537            } else {
538                (false, blend::source_rank(&r.memory.source))
539            }
540        };
541        scored.sort_by(|a, b| {
542            source_rank(a).cmp(&source_rank(b)).then(
543                b.score
544                    .partial_cmp(&a.score)
545                    .unwrap_or(std::cmp::Ordering::Equal),
546            )
547        });
548        Ok(scored)
549    }
550
551    /// Ranks `project`'s memories first and searches `request`'s wider scope
552    /// only when nothing in the project clears the bar. `generate` makes the
553    /// candidates for either request. The project was not named by the
554    /// caller, so its pass never gets the relaxed floor of an exhausted named
555    /// scope.
556    pub fn rank_project_first<S, G>(
557        &self,
558        store: &S,
559        request: &QueryRequest,
560        project: &str,
561        mut generate: G,
562        reranker: Option<&dyn RerankScorer>,
563    ) -> Result<ProjectFirst>
564    where
565        S: Store + ?Sized,
566        G: FnMut(&QueryRequest) -> Result<Candidates>,
567    {
568        let mut local = request.clone();
569        local.filters.project_path = Some(project.to_string());
570        let mut pool = generate(&local)?;
571        pool.scope_exhausted = false;
572        let results = self.rank(store, &local, &pool, reranker)?;
573        if !results.is_empty() {
574            return Ok(ProjectFirst {
575                results,
576                scope: ProjectScope::Project,
577            });
578        }
579        let pool = generate(request)?;
580        Ok(ProjectFirst {
581            results: self.rank(store, request, &pool, reranker)?,
582            scope: ProjectScope::OtherProjects,
583        })
584    }
585
586    /// `rank_project_first` in one pass, for a caller that cannot afford a
587    /// second rerank: ranks `request`'s scope once and moves `project`'s
588    /// results that reach `blend::ABSTAIN_RERANK_BAR` ahead of the rest, the
589    /// bar that decides `rank_project_first`'s project pass, so a weak local
590    /// match never displaces a strong one from elsewhere. A project memory
591    /// outside the blended shortlist of the wide scope (the top
592    /// `3 * request.limit` by primary score) is not looked for. On
593    /// an answer the scorer could not rerank, relevance is cosine, which the
594    /// bar does not calibrate, so the order then means little.
595    pub fn rank_project_preferred<S, G>(
596        &self,
597        store: &S,
598        request: &QueryRequest,
599        project: &str,
600        mut generate: G,
601        reranker: Option<&dyn RerankScorer>,
602    ) -> Result<ProjectFirst>
603    where
604        S: Store + ?Sized,
605        G: FnMut(&QueryRequest) -> Result<Candidates>,
606    {
607        let pool = generate(request)?;
608        let mut results = self.ranked_shortlist(store, request, &pool, reranker)?;
609        let in_project = |r: &QueryResult| {
610            r.memory.project_path.as_deref() == Some(project)
611                && r.relevance >= blend::ABSTAIN_RERANK_BAR
612        };
613        results.sort_by_key(|r| !in_project(r));
614        results.truncate(request.limit);
615        let scope = if results.first().is_some_and(in_project) {
616            ProjectScope::Project
617        } else {
618            ProjectScope::OtherProjects
619        };
620        Ok(ProjectFirst { results, scope })
621    }
622}
623
624#[cfg(test)]
625mod tests {
626    use super::*;
627    use crate::query::QueryFilters;
628    use crate::schema::{Conversation, Entity, MemoryType, Relation};
629    use crate::store::Store;
630    use chrono::Utc;
631
632    struct StubStore {
633        memories: Vec<Memory>,
634    }
635
636    impl Store for StubStore {
637        fn store_memory(&self, _: &Memory) -> Result<()> {
638            unimplemented!()
639        }
640        fn get_memory(&self, id: Uuid) -> Result<Option<Memory>> {
641            Ok(self.memories.iter().find(|m| m.id == id).cloned())
642        }
643        fn get_memory_with_embedding(&self, id: Uuid) -> Result<Option<Memory>> {
644            self.get_memory(id)
645        }
646        fn delete_memory(&self, _: Uuid) -> Result<()> {
647            unimplemented!()
648        }
649        fn store_entity(&self, _: &Entity) -> Result<()> {
650            unimplemented!()
651        }
652        fn get_entity(&self, _: Uuid) -> Result<Option<Entity>> {
653            Ok(None)
654        }
655        fn find_entity_by_name(&self, _: &str) -> Result<Option<Entity>> {
656            Ok(None)
657        }
658        fn store_conversation(&self, _: &Conversation) -> Result<()> {
659            unimplemented!()
660        }
661        fn store_relation(&self, _: &Relation) -> Result<()> {
662            unimplemented!()
663        }
664        fn get_relations(
665            &self,
666            _: Uuid,
667            _: Option<crate::schema::RelationType>,
668        ) -> Result<Vec<Relation>> {
669            Ok(Vec::new())
670        }
671        fn vector_search(&self, _: &[f32], _: usize) -> Result<Vec<(Memory, f32)>> {
672            unimplemented!()
673        }
674        fn filtered_vector_search(
675            &self,
676            embedding: &[f32],
677            source: Option<&str>,
678            memory_type: Option<MemoryType>,
679            project_path: Option<&str>,
680            limit: usize,
681        ) -> Result<Vec<(Memory, f32)>> {
682            let mut scored: Vec<(Memory, f32)> = self
683                .memories
684                .iter()
685                .filter(|m| source.is_none_or(|s| m.source == s))
686                .filter(|m| memory_type.is_none_or(|t| m.memory_type == t))
687                .filter(|m| project_path.is_none_or(|p| m.project_path.as_deref() == Some(p)))
688                .map(|m| (m.clone(), cosine(embedding, &m.embedding)))
689                .collect();
690            scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
691            scored.truncate(limit);
692            Ok(scored)
693        }
694        fn traverse(&self, _: Uuid, _: u32) -> Result<Vec<(Memory, Vec<Relation>)>> {
695            unimplemented!()
696        }
697        fn memories_by_source(&self, _: &str) -> Result<Vec<Memory>> {
698            unimplemented!()
699        }
700        fn memories_by_type(&self, _: MemoryType) -> Result<Vec<Memory>> {
701            unimplemented!()
702        }
703        fn update_memory(&self, _: &Memory) -> Result<()> {
704            unimplemented!()
705        }
706        fn record_access(&self, _: &Memory) -> Result<()> {
707            unimplemented!()
708        }
709        fn bm25_search(&self, _: &str, _: usize) -> Result<Vec<Memory>> {
710            unimplemented!()
711        }
712        fn memory_count(&self) -> Result<usize> {
713            unimplemented!()
714        }
715        fn all_memory_ids(&self) -> Result<Vec<Uuid>> {
716            unimplemented!()
717        }
718        fn all_relations(&self) -> Result<Vec<Relation>> {
719            unimplemented!()
720        }
721        /// The stub keeps a memory's chunk window in its `context`.
722        fn chunk_window(&self, memory_id: Uuid, _: usize) -> Result<Option<String>> {
723            Ok(self
724                .memories
725                .iter()
726                .find(|m| m.id == memory_id)
727                .and_then(|m| m.context.clone()))
728        }
729    }
730
731    fn memory(content: &str, embedding: Vec<f32>) -> Memory {
732        let mut m = Memory::new(
733            content.to_string(),
734            MemoryType::Semantic,
735            "test".to_string(),
736            String::new(),
737        );
738        m.embedding = embedding;
739        m
740    }
741
742    fn request(embedding: Vec<f32>) -> QueryRequest {
743        QueryRequest {
744            text: "query".to_string(),
745            embedding,
746            limit: 10,
747            filters: QueryFilters::default(),
748        }
749    }
750
751    #[test]
752    fn candidate_on_both_lists_outranks_dense_only_at_equal_similarity() {
753        let both = memory("on both lists", vec![1.0, 0.0]);
754        let dense_only = memory("dense only", vec![1.0, 0.0]);
755        let store = StubStore {
756            memories: vec![both.clone(), dense_only.clone()],
757        };
758        let retriever = Retriever::default();
759
760        let dense = vec![(dense_only.id, 0.9), (both.id, 0.9)];
761        let lexical = vec![(both.id, 5.0)];
762        let results = store_rank(
763            &retriever,
764            &store,
765            &request(vec![1.0, 0.0]),
766            &dense,
767            &lexical,
768        );
769
770        assert_eq!(results[0].memory.id, both.id);
771    }
772
773    #[test]
774    fn low_cosine_candidate_is_gated_out_on_the_no_rerank_path() {
775        let strong = memory("strong match", vec![1.0, 0.0]);
776        let weak = memory("weak match", vec![1.0, 0.0]);
777        let store = StubStore {
778            memories: vec![strong.clone(), weak.clone()],
779        };
780        let retriever = Retriever::default();
781
782        let dense = vec![(strong.id, 0.92), (weak.id, 0.45)];
783        let results = store_rank(&retriever, &store, &request(vec![1.0, 0.0]), &dense, &[]);
784
785        assert_eq!(results.len(), 1);
786        assert_eq!(results[0].memory.id, strong.id);
787    }
788
789    #[test]
790    fn lexical_only_candidate_gets_cosine_from_stored_embedding() {
791        // Arrives only via BM25; its stored embedding matches the query
792        // exactly, so cosine = 1.0 clears the 0.59 gate.
793        let lexical_hit = memory("lexical only", vec![0.6, 0.8]);
794        let store = StubStore {
795            memories: vec![lexical_hit.clone()],
796        };
797        let retriever = Retriever::default();
798
799        let lexical = vec![(lexical_hit.id, 3.0)];
800        let results = store_rank(&retriever, &store, &request(vec![0.6, 0.8]), &[], &lexical);
801
802        assert_eq!(results.len(), 1);
803    }
804
805    #[test]
806    fn blend_only_reorders_the_primary_shortlist() {
807        // Legacy recall blended graph/recency over at most limit*3 candidates
808        // ranked by similarity; a wider pool must not let a fresh but weakly
809        // similar memory displace strong matches, so candidates outside the
810        // top limit*3 by primary are excluded before blending.
811        let old = |content: &str, sim: f32| {
812            let mut m = memory(content, vec![1.0, 0.0]);
813            m.created_at = Utc::now() - chrono::Duration::days(200);
814            m.last_accessed = m.created_at;
815            (m, sim)
816        };
817        let (a, a_sim) = old("strong a", 0.95);
818        let (b, b_sim) = old("strong b", 0.94);
819        let (c, c_sim) = old("strong c", 0.93);
820        let fresh = memory("fresh diluter", vec![1.0, 0.0]);
821        let store = StubStore {
822            memories: vec![a.clone(), b.clone(), c.clone(), fresh.clone()],
823        };
824        let retriever = Retriever::default();
825
826        let dense = vec![
827            (a.id, a_sim),
828            (b.id, b_sim),
829            (c.id, c_sim),
830            (fresh.id, 0.60),
831        ];
832        let mut req = request(vec![1.0, 0.0]);
833        req.limit = 1;
834        let results = retriever
835            .rank(&store, &req, &Candidates::new(dense, Vec::new()), None)
836            .unwrap();
837
838        assert_eq!(results[0].memory.id, a.id);
839    }
840
841    #[test]
842    fn source_filter_still_applies() {
843        let mut filtered = memory("wrong source", vec![1.0, 0.0]);
844        filtered.source = "other".to_string();
845        let kept = memory("right source", vec![1.0, 0.0]);
846        let store = StubStore {
847            memories: vec![filtered.clone(), kept.clone()],
848        };
849        let retriever = Retriever::default();
850
851        let mut req = request(vec![1.0, 0.0]);
852        req.filters.source = Some("test".to_string());
853        let dense = vec![(filtered.id, 0.9), (kept.id, 0.9)];
854        let results = store_rank(&retriever, &store, &req, &dense, &[]);
855
856        assert_eq!(results.len(), 1);
857        assert_eq!(results[0].memory.id, kept.id);
858    }
859
860    #[test]
861    fn archived_and_superseded_candidates_are_dropped_by_rank() {
862        // The outbox drains every few seconds, so a candidate list can name a
863        // row the store has since retired or superseded.
864        let live = memory("live", vec![1.0, 0.0]);
865        let mut archived = memory("archived", vec![1.0, 0.0]);
866        archived.tier = crate::schema::Tier::Archived;
867        let mut superseded = memory("superseded", vec![1.0, 0.0]);
868        superseded.superseded_by = Some(live.id);
869        let store = StubStore {
870            memories: vec![live.clone(), archived.clone(), superseded.clone()],
871        };
872        let retriever = Retriever::default();
873
874        let dense = vec![(archived.id, 0.99), (superseded.id, 0.98), (live.id, 0.9)];
875        let results = store_rank(&retriever, &store, &request(vec![1.0, 0.0]), &dense, &[]);
876
877        let ids: Vec<Uuid> = results.iter().map(|r| r.memory.id).collect();
878        assert_eq!(ids, vec![live.id]);
879    }
880
881    #[test]
882    fn an_exhausted_scope_keeps_a_similarity_floor() {
883        // A cosine of 0.3 is noise in any scope; one just over the floor is
884        // a real but weak match.
885        let w = blend::EXHAUSTED_MIN_SIMILARITY + 0.005;
886        let weak = memory("weak match", vec![w, (1.0f32 - w * w).sqrt()]);
887        let noise = memory("noise", vec![0.3, (1.0f32 - 0.09).sqrt()]);
888        let store = StubStore {
889            memories: vec![weak.clone(), noise.clone()],
890        };
891        let retriever = Retriever::default();
892        let pool = Candidates {
893            dense: vec![(weak.id, w), (noise.id, 0.3)],
894            lexical: Vec::new(),
895            scope_exhausted: true,
896            best_chunk: HashMap::new(),
897        };
898
899        let results = retriever
900            .rank(&store, &request(vec![1.0, 0.0]), &pool, None)
901            .unwrap();
902
903        let ids: Vec<Uuid> = results.iter().map(|r| r.memory.id).collect();
904        assert_eq!(ids, vec![weak.id]);
905    }
906
907    #[test]
908    fn an_exhausted_scope_gates_on_cosine_even_when_reranked() {
909        struct LowScorer;
910        impl RerankScorer for LowScorer {
911            fn rerank(&self, _: &str, docs: &[&str]) -> Result<Vec<f32>> {
912                Ok(docs.iter().map(|_| 0.02).collect())
913            }
914        }
915        let w = blend::EXHAUSTED_MIN_SIMILARITY + 0.005;
916        let weak = memory("weak match", vec![w, (1.0f32 - w * w).sqrt()]);
917        let noise = memory("noise", vec![0.3, (1.0f32 - 0.09).sqrt()]);
918        let store = StubStore {
919            memories: vec![weak.clone(), noise.clone()],
920        };
921        let retriever = Retriever::default();
922        let pool = Candidates {
923            dense: vec![(weak.id, w), (noise.id, 0.3)],
924            lexical: Vec::new(),
925            scope_exhausted: true,
926            best_chunk: HashMap::new(),
927        };
928
929        let results = retriever
930            .rank(&store, &request(vec![1.0, 0.0]), &pool, Some(&LowScorer))
931            .unwrap();
932
933        let ids: Vec<Uuid> = results.iter().map(|r| r.memory.id).collect();
934        assert_eq!(ids, vec![weak.id]);
935    }
936
937    fn store_rank(
938        retriever: &Retriever,
939        store: &StubStore,
940        request: &QueryRequest,
941        dense: &[(Uuid, f32)],
942        lexical: &[(Uuid, f32)],
943    ) -> Vec<QueryResult> {
944        retriever
945            .rank(
946                store,
947                request,
948                &Candidates::new(dense.to_vec(), lexical.to_vec()),
949                None,
950            )
951            .unwrap()
952    }
953
954    /// Scores 0.8 for any document naming the planted fact and 0.1 for the
955    /// rest, and records every call's documents.
956    struct FactScorer {
957        calls: std::sync::Mutex<Vec<Vec<String>>>,
958    }
959
960    impl FactScorer {
961        fn new() -> Self {
962            Self {
963                calls: std::sync::Mutex::new(Vec::new()),
964            }
965        }
966    }
967
968    impl RerankScorer for FactScorer {
969        fn rerank(&self, _: &str, docs: &[&str]) -> Result<Vec<f32>> {
970            self.calls
971                .lock()
972                .unwrap()
973                .push(docs.iter().map(|d| d.to_string()).collect());
974            Ok(docs
975                .iter()
976                .map(|d| if d.contains("fact") { 0.8 } else { 0.1 })
977                .collect())
978        }
979    }
980
981    fn chunked_pool(dense: Vec<(Uuid, f32)>, best_chunk: &[(Uuid, usize, f32)]) -> Candidates {
982        let mut pool = Candidates::new(dense, Vec::new());
983        pool.best_chunk = best_chunk
984            .iter()
985            .map(|(id, ord, sim)| (*id, (*ord, *sim)))
986            .collect();
987        pool
988    }
989
990    #[test]
991    fn the_rerank_primary_is_the_better_of_the_whole_memory_and_its_chunk_window() {
992        let mut buried = memory("a long paragraph about many things", vec![1.0, 0.0]);
993        buried.context = Some("the window holding the fact".to_string());
994        let mut whole = memory("the fact stated outright", vec![1.0, 0.0]);
995        whole.context = Some("a window about nothing".to_string());
996        let plain = memory("nothing relevant here", vec![1.0, 0.0]);
997        let store = StubStore {
998            memories: vec![buried.clone(), whole.clone(), plain.clone()],
999        };
1000        let pool = chunked_pool(
1001            vec![(buried.id, 0.7), (whole.id, 0.7), (plain.id, 0.7)],
1002            &[(buried.id, 2, 0.7), (whole.id, 0, 0.6)],
1003        );
1004        let scorer = FactScorer::new();
1005
1006        let results = Retriever::default()
1007            .rank(&store, &request(vec![1.0, 0.0]), &pool, Some(&scorer))
1008            .unwrap();
1009
1010        let mut ids: Vec<Uuid> = results.iter().map(|r| r.memory.id).collect();
1011        ids.sort();
1012        let mut expected = vec![buried.id, whole.id];
1013        expected.sort();
1014        assert_eq!(ids, expected, "plain reranks at 0.1, under the gate");
1015        let calls = scorer.calls.lock().unwrap();
1016        assert_eq!(calls.len(), 1, "whole memories and windows share one call");
1017        assert_eq!(
1018            calls[0],
1019            vec![
1020                "a long paragraph about many things",
1021                "the fact stated outright",
1022                "nothing relevant here",
1023                "the window holding the fact",
1024                "a window about nothing",
1025            ]
1026        );
1027    }
1028
1029    #[test]
1030    fn without_whole_reranking_a_chunked_memory_is_scored_on_its_window_alone() {
1031        let mut buried = memory("a long paragraph about many things", vec![1.0, 0.0]);
1032        buried.context = Some("the window holding the fact".to_string());
1033        let plain = memory("the fact stated outright", vec![1.0, 0.0]);
1034        let store = StubStore {
1035            memories: vec![buried.clone(), plain.clone()],
1036        };
1037        let pool = chunked_pool(
1038            vec![(buried.id, 0.7), (plain.id, 0.7)],
1039            &[(buried.id, 2, 0.7)],
1040        );
1041        let scorer = FactScorer::new();
1042        let retriever = Retriever {
1043            tunables: RetrievalTunables {
1044                rerank_whole_chunked: false,
1045                ..RetrievalTunables::default()
1046            },
1047        };
1048
1049        let results = retriever
1050            .rank(&store, &request(vec![1.0, 0.0]), &pool, Some(&scorer))
1051            .unwrap();
1052
1053        assert_eq!(results.len(), 2);
1054        assert_eq!(
1055            scorer.calls.lock().unwrap()[0],
1056            vec!["the window holding the fact", "the fact stated outright"]
1057        );
1058    }
1059
1060    #[test]
1061    fn with_chunk_rerank_off_windows_are_neither_scored_nor_counted() {
1062        let mut buried = memory("a long paragraph about many things", vec![1.0, 0.0]);
1063        buried.context = Some("the window holding the fact".to_string());
1064        let store = StubStore {
1065            memories: vec![buried.clone()],
1066        };
1067        let pool = chunked_pool(vec![(buried.id, 0.7)], &[(buried.id, 2, 0.7)]);
1068        let scorer = FactScorer::new();
1069        let retriever = Retriever {
1070            tunables: RetrievalTunables {
1071                chunk_rerank: false,
1072                ..RetrievalTunables::default()
1073            },
1074        };
1075
1076        let results = retriever
1077            .rank(&store, &request(vec![1.0, 0.0]), &pool, Some(&scorer))
1078            .unwrap();
1079
1080        assert!(results.is_empty());
1081        assert_eq!(
1082            scorer.calls.lock().unwrap()[0],
1083            vec!["a long paragraph about many things"]
1084        );
1085    }
1086
1087    #[test]
1088    fn the_cosine_primary_takes_the_best_chunk_when_it_beats_the_memory() {
1089        let buried = memory("far as a whole", vec![0.0, 1.0]);
1090        let store = StubStore {
1091            memories: vec![buried.clone()],
1092        };
1093        let mut pool = chunked_pool(Vec::new(), &[(buried.id, 1, 0.9)]);
1094        pool.lexical = vec![(buried.id, 3.0)];
1095
1096        let results = Retriever::default()
1097            .rank(&store, &request(vec![1.0, 0.0]), &pool, None)
1098            .unwrap();
1099        assert_eq!(results.len(), 1, "cosine 0 as a whole, 0.9 by its chunk");
1100
1101        let mut exhausted = pool.clone();
1102        exhausted.scope_exhausted = true;
1103        let results = Retriever::default()
1104            .rank(&store, &request(vec![1.0, 0.0]), &exhausted, None)
1105            .unwrap();
1106        assert_eq!(results.len(), 1, "the exhausted floor also sees the chunk");
1107    }
1108
1109    /// Scores each document by the first entry of its table whose key the
1110    /// document contains, 0.0 when none does.
1111    struct TableScorer(Vec<(&'static str, f32)>);
1112
1113    impl RerankScorer for TableScorer {
1114        fn rerank(&self, _: &str, docs: &[&str]) -> Result<Vec<f32>> {
1115            Ok(docs
1116                .iter()
1117                .map(|d| {
1118                    self.0
1119                        .iter()
1120                        .find(|(key, _)| d.contains(key))
1121                        .map_or(0.0, |(_, score)| *score)
1122                })
1123                .collect())
1124        }
1125    }
1126
1127    fn abstaining() -> Retriever {
1128        Retriever {
1129            tunables: RetrievalTunables {
1130                abstain: true,
1131                ..RetrievalTunables::default()
1132            },
1133        }
1134    }
1135
1136    fn reranked_ids(
1137        retriever: &Retriever,
1138        store: &StubStore,
1139        dense: Vec<(Uuid, f32)>,
1140        scorer: &TableScorer,
1141    ) -> Vec<Uuid> {
1142        retriever
1143            .rank(
1144                store,
1145                &request(vec![1.0, 0.0]),
1146                &Candidates::new(dense, Vec::new()),
1147                Some(scorer),
1148            )
1149            .unwrap()
1150            .iter()
1151            .map(|r| r.memory.id)
1152            .collect()
1153    }
1154
1155    #[test]
1156    fn abstention_answers_in_full_when_one_result_clears_the_bar() {
1157        let above = memory("above the bar", vec![1.0, 0.0]);
1158        let below = memory("below the bar", vec![1.0, 0.0]);
1159        let store = StubStore {
1160            memories: vec![above.clone(), below.clone()],
1161        };
1162        let scorer = TableScorer(vec![
1163            ("above", blend::ABSTAIN_RERANK_BAR + 0.01),
1164            ("below", blend::ABSTAIN_RERANK_BAR - 0.01),
1165        ]);
1166        assert!(blend::ABSTAIN_RERANK_BAR - 0.01 >= RetrievalTunables::default().rerank_gate);
1167        let dense = vec![(above.id, 0.9), (below.id, 0.9)];
1168
1169        let kept = reranked_ids(&abstaining(), &store, dense, &scorer);
1170        assert_eq!(kept, vec![above.id, below.id]);
1171    }
1172
1173    #[test]
1174    fn abstention_returns_nothing_when_no_result_clears_the_bar_and_only_when_asked() {
1175        let weak = memory("weak match", vec![1.0, 0.0]);
1176        let weaker = memory("weaker match", vec![1.0, 0.0]);
1177        let store = StubStore {
1178            memories: vec![weak.clone(), weaker.clone()],
1179        };
1180        let scorer = TableScorer(vec![
1181            ("weaker", RetrievalTunables::default().rerank_gate + 0.01),
1182            ("weak", blend::ABSTAIN_RERANK_BAR - 0.01),
1183        ]);
1184        let dense = vec![(weak.id, 0.9), (weaker.id, 0.9)];
1185
1186        assert!(reranked_ids(&abstaining(), &store, dense.clone(), &scorer).is_empty());
1187
1188        let unchanged = reranked_ids(&Retriever::default(), &store, dense, &scorer);
1189        assert_eq!(unchanged.len(), 2, "abstention is off by default");
1190    }
1191
1192    #[test]
1193    fn abstention_leaves_the_cosine_path_on_its_gate() {
1194        let weak = memory("weak but over the cosine gate", vec![1.0, 0.0]);
1195        let store = StubStore {
1196            memories: vec![weak.clone()],
1197        };
1198        let dense = vec![(weak.id, blend::MIN_SIMILARITY + 0.01)];
1199        let results = abstaining()
1200            .rank(
1201                &store,
1202                &request(vec![1.0, 0.0]),
1203                &Candidates::new(dense, Vec::new()),
1204                None,
1205            )
1206            .unwrap();
1207        assert_eq!(results.len(), 1);
1208    }
1209
1210    #[test]
1211    fn a_result_carries_the_relevance_its_membership_was_judged_on() {
1212        let m = memory("a scored memory", vec![1.0, 0.0]);
1213        let store = StubStore {
1214            memories: vec![m.clone()],
1215        };
1216        let pool = Candidates::new(vec![(m.id, 0.83)], Vec::new());
1217
1218        let cosine_path = Retriever::default()
1219            .rank(&store, &request(vec![1.0, 0.0]), &pool, None)
1220            .unwrap();
1221        assert!((cosine_path[0].relevance - 0.83).abs() < 1e-6);
1222
1223        let scorer = TableScorer(vec![("scored", 0.71)]);
1224        let reranked = Retriever::default()
1225            .rank(&store, &request(vec![1.0, 0.0]), &pool, Some(&scorer))
1226            .unwrap();
1227        assert!((reranked[0].relevance - 0.71).abs() < 1e-6);
1228    }
1229
1230    #[test]
1231    fn source_order_puts_agent_notes_then_distillates_then_raws() {
1232        let mut note = memory("agent note", vec![1.0, 0.0]);
1233        note.source = crate::schema::AGENT_NOTE_SOURCE.to_string();
1234        let mut distillate = memory("distillate", vec![1.0, 0.0]);
1235        distillate.source = "consolidation".to_string();
1236        let mut raw = memory("raw", vec![1.0, 0.0]);
1237        raw.source = "claude-code".to_string();
1238        let store = StubStore {
1239            memories: vec![note.clone(), distillate.clone(), raw.clone()],
1240        };
1241        let scorer = TableScorer(vec![("raw", 0.95), ("distillate", 0.85), ("note", 0.75)]);
1242        let dense = vec![(raw.id, 0.9), (distillate.id, 0.9), (note.id, 0.9)];
1243
1244        let by_score = reranked_ids(&Retriever::default(), &store, dense.clone(), &scorer);
1245        assert_eq!(by_score, vec![raw.id, distillate.id, note.id]);
1246
1247        let ordered = Retriever {
1248            tunables: RetrievalTunables {
1249                source_order: true,
1250                ..RetrievalTunables::default()
1251            },
1252        };
1253        let by_source = reranked_ids(&ordered, &store, dense, &scorer);
1254        assert_eq!(by_source, vec![note.id, distillate.id, raw.id]);
1255    }
1256
1257    #[test]
1258    fn source_order_never_lifts_a_result_below_the_bar_over_one_above_it() {
1259        let mut weak_note = memory("weak note", vec![1.0, 0.0]);
1260        weak_note.source = crate::schema::AGENT_NOTE_SOURCE.to_string();
1261        let mut distillate = memory("strong distillate", vec![1.0, 0.0]);
1262        distillate.source = "consolidation".to_string();
1263        let mut raw = memory("strong raw", vec![1.0, 0.0]);
1264        raw.source = "claude-code".to_string();
1265        let mut weak_raw = memory("weak raw", vec![1.0, 0.0]);
1266        weak_raw.source = "claude-code".to_string();
1267        let store = StubStore {
1268            memories: vec![
1269                weak_note.clone(),
1270                distillate.clone(),
1271                raw.clone(),
1272                weak_raw.clone(),
1273            ],
1274        };
1275        let scorer = TableScorer(vec![
1276            ("strong raw", 0.95),
1277            ("strong distillate", blend::ABSTAIN_RERANK_BAR + 0.01),
1278            ("weak raw", blend::ABSTAIN_RERANK_BAR - 0.02),
1279            ("weak note", blend::ABSTAIN_RERANK_BAR - 0.05),
1280        ]);
1281        let dense = vec![
1282            (raw.id, 0.9),
1283            (distillate.id, 0.9),
1284            (weak_raw.id, 0.9),
1285            (weak_note.id, 0.9),
1286        ];
1287        let ordered = Retriever {
1288            tunables: RetrievalTunables {
1289                source_order: true,
1290                ..RetrievalTunables::default()
1291            },
1292        };
1293
1294        let ids = reranked_ids(&ordered, &store, dense, &scorer);
1295
1296        assert_eq!(ids, vec![distillate.id, raw.id, weak_raw.id, weak_note.id]);
1297    }
1298
1299    #[test]
1300    fn source_order_ranks_remembered_notes_with_distillates_above_raws() {
1301        assert_eq!(blend::source_rank(crate::schema::AGENT_NOTE_SOURCE), 0);
1302        assert_eq!(blend::source_rank("consolidation"), 1);
1303        for explicit in crate::schema::EXPLICIT_SOURCES {
1304            if *explicit != crate::schema::AGENT_NOTE_SOURCE {
1305                assert_eq!(blend::source_rank(explicit), 1, "{explicit}");
1306            }
1307        }
1308        for raw in ["claude-code", "cursor", "codex", "obsidian"] {
1309            assert_eq!(blend::source_rank(raw), 2, "{raw}");
1310        }
1311    }
1312
1313    const LOCAL: &str = "-work-local";
1314
1315    fn in_project(content: &str, project: &str) -> Memory {
1316        let mut m = memory(content, vec![1.0, 0.0]);
1317        m.project_path = Some(project.to_string());
1318        m
1319    }
1320
1321    /// A pool for whichever scope the request names: every stored memory in
1322    /// it, cosine-scored from the stored vectors, exhausted when scoped.
1323    fn scoped_pool(store: &StubStore, request: &QueryRequest) -> Candidates {
1324        let scope = request.filters.project_path.as_deref();
1325        let dense: Vec<(Uuid, f32)> = store
1326            .memories
1327            .iter()
1328            .filter(|m| scope.is_none_or(|p| m.project_path.as_deref() == Some(p)))
1329            .map(|m| (m.id, cosine(&request.embedding, &m.embedding)))
1330            .collect();
1331        Candidates {
1332            scope_exhausted: scope.is_some(),
1333            ..Candidates::new(dense, Vec::new())
1334        }
1335    }
1336
1337    fn project_first(
1338        store: &StubStore,
1339        scorer: &TableScorer,
1340        scopes: &std::cell::RefCell<Vec<Option<String>>>,
1341    ) -> ProjectFirst {
1342        abstaining()
1343            .rank_project_first(
1344                store,
1345                &request(vec![1.0, 0.0]),
1346                LOCAL,
1347                |r| {
1348                    scopes.borrow_mut().push(r.filters.project_path.clone());
1349                    Ok(scoped_pool(store, r))
1350                },
1351                Some(scorer),
1352            )
1353            .unwrap()
1354    }
1355
1356    fn project_preferred(
1357        store: &StubStore,
1358        scorer: &TableScorer,
1359        scopes: &std::cell::RefCell<Vec<Option<String>>>,
1360    ) -> ProjectFirst {
1361        abstaining()
1362            .rank_project_preferred(
1363                store,
1364                &request(vec![1.0, 0.0]),
1365                LOCAL,
1366                |r| {
1367                    scopes.borrow_mut().push(r.filters.project_path.clone());
1368                    Ok(scoped_pool(store, r))
1369                },
1370                Some(scorer),
1371            )
1372            .unwrap()
1373    }
1374
1375    #[test]
1376    fn project_preferred_ranks_once_and_puts_the_project_first() {
1377        let local = in_project("local weaker", LOCAL);
1378        let elsewhere = in_project("elsewhere stronger", "-work-other");
1379        let store = StubStore {
1380            memories: vec![elsewhere.clone(), local.clone()],
1381        };
1382        let scorer = TableScorer(vec![("weaker", 0.7), ("stronger", 0.95)]);
1383        let scopes = std::cell::RefCell::new(Vec::new());
1384
1385        let answer = project_preferred(&store, &scorer, &scopes);
1386
1387        assert_eq!(answer.scope, ProjectScope::Project);
1388        let ids: Vec<Uuid> = answer.results.iter().map(|r| r.memory.id).collect();
1389        assert_eq!(ids, vec![local.id, elsewhere.id]);
1390        assert_eq!(scopes.into_inner(), vec![None], "one wide pass");
1391    }
1392
1393    #[test]
1394    fn project_preferred_finds_a_project_match_ranked_below_the_limit() {
1395        let first = in_project("elsewhere best", "-work-other");
1396        let second = in_project("elsewhere next", "-work-other");
1397        let local = in_project("local good", LOCAL);
1398        let store = StubStore {
1399            memories: vec![first, second, local.clone()],
1400        };
1401        let scorer = TableScorer(vec![("best", 0.9), ("next", 0.8), ("good", 0.6)]);
1402
1403        let answer = abstaining()
1404            .rank_project_preferred(
1405                &store,
1406                &QueryRequest {
1407                    limit: 1,
1408                    ..request(vec![1.0, 0.0])
1409                },
1410                LOCAL,
1411                |r| Ok(scoped_pool(&store, r)),
1412                Some(&scorer),
1413            )
1414            .unwrap();
1415
1416        assert_eq!(answer.scope, ProjectScope::Project);
1417        let ids: Vec<Uuid> = answer.results.iter().map(|r| r.memory.id).collect();
1418        assert_eq!(ids, vec![local.id]);
1419    }
1420
1421    #[test]
1422    fn project_preferred_looks_no_deeper_than_the_blend_shortlist() {
1423        let others: Vec<Memory> = ["first", "second", "third"]
1424            .iter()
1425            .map(|c| {
1426                let mut m = in_project(&format!("elsewhere {c}"), "-work-other");
1427                // Stale, so the fresher local memory would win a wider blend
1428                // despite its lower primary score; without it this test
1429                // passes on a shortlist of any depth.
1430                m.last_accessed = Utc::now() - chrono::Duration::days(60);
1431                m
1432            })
1433            .collect();
1434        let local = in_project("local fourth", LOCAL);
1435        let mut memories = others.clone();
1436        memories.push(local);
1437        let store = StubStore { memories };
1438        let scorer = TableScorer(vec![
1439            ("first", 0.95),
1440            ("second", 0.9),
1441            ("third", 0.85),
1442            ("fourth", 0.8),
1443        ]);
1444
1445        let answer = abstaining()
1446            .rank_project_preferred(
1447                &store,
1448                &QueryRequest {
1449                    limit: 1,
1450                    ..request(vec![1.0, 0.0])
1451                },
1452                LOCAL,
1453                |r| Ok(scoped_pool(&store, r)),
1454                Some(&scorer),
1455            )
1456            .unwrap();
1457
1458        assert_eq!(answer.scope, ProjectScope::OtherProjects);
1459        let ids: Vec<Uuid> = answer.results.iter().map(|r| r.memory.id).collect();
1460        assert_eq!(ids, vec![others[0].id]);
1461    }
1462
1463    #[test]
1464    fn project_preferred_keeps_a_weak_project_match_behind_a_strong_one_elsewhere() {
1465        let local = in_project("local noise", LOCAL);
1466        let elsewhere = in_project("elsewhere answer", "-work-other");
1467        let store = StubStore {
1468            memories: vec![local.clone(), elsewhere.clone()],
1469        };
1470        let scorer = TableScorer(vec![
1471            ("noise", blend::ABSTAIN_RERANK_BAR - 0.01),
1472            ("answer", 0.9),
1473        ]);
1474        let scopes = std::cell::RefCell::new(Vec::new());
1475
1476        let answer = project_preferred(&store, &scorer, &scopes);
1477
1478        assert_eq!(answer.scope, ProjectScope::OtherProjects);
1479        let ids: Vec<Uuid> = answer.results.iter().map(|r| r.memory.id).collect();
1480        assert_eq!(ids, vec![elsewhere.id, local.id]);
1481        assert_eq!(scopes.into_inner(), vec![None]);
1482    }
1483
1484    #[test]
1485    fn project_first_answers_from_the_project_when_something_there_clears_the_bar() {
1486        let local = in_project("local answer", LOCAL);
1487        let elsewhere = in_project("elsewhere answer", "-work-other");
1488        let store = StubStore {
1489            memories: vec![local.clone(), elsewhere.clone()],
1490        };
1491        let scorer = TableScorer(vec![("answer", 0.9)]);
1492        let scopes = std::cell::RefCell::new(Vec::new());
1493
1494        let answer = project_first(&store, &scorer, &scopes);
1495
1496        assert_eq!(answer.scope, ProjectScope::Project);
1497        let ids: Vec<Uuid> = answer.results.iter().map(|r| r.memory.id).collect();
1498        assert_eq!(ids, vec![local.id]);
1499        assert_eq!(scopes.into_inner(), vec![Some(LOCAL.to_string())]);
1500    }
1501
1502    #[test]
1503    fn project_first_searches_other_projects_when_nothing_local_clears_the_bar() {
1504        let local = in_project("local noise", LOCAL);
1505        let elsewhere = in_project("elsewhere answer", "-work-other");
1506        let store = StubStore {
1507            memories: vec![local.clone(), elsewhere.clone()],
1508        };
1509        let scorer = TableScorer(vec![
1510            ("noise", blend::ABSTAIN_RERANK_BAR - 0.01),
1511            ("answer", 0.9),
1512        ]);
1513        let scopes = std::cell::RefCell::new(Vec::new());
1514
1515        let answer = project_first(&store, &scorer, &scopes);
1516
1517        assert_eq!(answer.scope, ProjectScope::OtherProjects);
1518        let ids: Vec<Uuid> = answer.results.iter().map(|r| r.memory.id).collect();
1519        assert_eq!(ids, vec![elsewhere.id, local.id]);
1520        assert_eq!(scopes.into_inner(), vec![Some(LOCAL.to_string()), None]);
1521    }
1522
1523    #[test]
1524    fn project_first_keeps_the_bar_for_its_implicit_scope() {
1525        // The project was never named by the caller, so the relaxed floor an
1526        // exhausted named scope gets must not admit a local result the
1527        // reranker scores as noise.
1528        let local = in_project("local noise", LOCAL);
1529        let store = StubStore {
1530            memories: vec![local.clone()],
1531        };
1532        let scorer = TableScorer(vec![("noise", 0.02)]);
1533        let scopes = std::cell::RefCell::new(Vec::new());
1534
1535        let answer = project_first(&store, &scorer, &scopes);
1536
1537        assert!(answer.results.is_empty());
1538        assert_eq!(answer.scope, ProjectScope::OtherProjects);
1539    }
1540
1541    #[test]
1542    fn project_first_with_nothing_stored_for_the_project_searches_everywhere() {
1543        let elsewhere = in_project("elsewhere answer", "-work-other");
1544        let store = StubStore {
1545            memories: vec![elsewhere.clone()],
1546        };
1547        let scorer = TableScorer(vec![("answer", 0.9)]);
1548        let scopes = std::cell::RefCell::new(Vec::new());
1549
1550        let answer = project_first(&store, &scorer, &scopes);
1551
1552        assert_eq!(answer.scope, ProjectScope::OtherProjects);
1553        assert_eq!(answer.results.len(), 1);
1554    }
1555}