Skip to main content

second_brain_core/
query.rs

1use std::collections::HashSet;
2
3use anyhow::Result;
4use uuid::Uuid;
5
6use crate::schema::Memory;
7use crate::store::Store;
8
9#[derive(Debug, Clone)]
10pub struct QueryRequest {
11    pub text: String,
12    pub embedding: Vec<f32>,
13    pub limit: usize,
14    pub filters: QueryFilters,
15}
16
17#[derive(Debug, Clone, Default)]
18pub struct QueryFilters {
19    pub source: Option<String>,
20    pub memory_type: Option<crate::schema::MemoryType>,
21    pub min_confidence: Option<f32>,
22    pub entity_names: Vec<String>,
23    pub project_path: Option<String>,
24}
25
26#[derive(Debug, Clone)]
27pub struct QueryResult {
28    pub memory: Memory,
29    pub score: f32,
30    /// The term membership was judged on: the cross-encoder score when the
31    /// answer was reranked, else cosine similarity.
32    pub relevance: f32,
33    pub path: Vec<Uuid>,
34}
35
36use crate::retrieval::blend::{self, MIN_SIMILARITY, PROJECT_AFFINITY_BOOST};
37
38pub struct QueryEngine<'a, S: Store + ?Sized> {
39    store: &'a S,
40    vector_weight: f32,
41    graph_weight: f32,
42    recency_weight: f32,
43}
44
45impl<'a, S: Store + ?Sized> QueryEngine<'a, S> {
46    pub fn new(store: &'a S) -> Self {
47        Self {
48            store,
49            vector_weight: 0.5,
50            graph_weight: 0.3,
51            recency_weight: 0.2,
52        }
53    }
54
55    pub fn with_weights(mut self, vector: f32, graph: f32, recency: f32) -> Self {
56        self.vector_weight = vector;
57        self.graph_weight = graph;
58        self.recency_weight = recency;
59        self
60    }
61
62    pub fn recall(&self, request: &QueryRequest) -> Result<Vec<QueryResult>> {
63        // A scoped recall must scan the scope, not trim an unfiltered top-k:
64        // the filters below can only reject, so anything the unfiltered search
65        // misses stays missing however narrow the scope is.
66        let vector_results = if request.filters.source.is_some()
67            || request.filters.memory_type.is_some()
68            || request.filters.project_path.is_some()
69        {
70            self.store.filtered_vector_search(
71                &request.embedding,
72                request.filters.source.as_deref(),
73                request.filters.memory_type,
74                request.filters.project_path.as_deref(),
75                request.limit * 3,
76            )?
77        } else {
78            self.store
79                .vector_search(&request.embedding, request.limit * 3)?
80        };
81
82        let query_entity_ids = self.resolve_query_entities(&request.filters.entity_names);
83
84        let mut scored: Vec<QueryResult> = Vec::new();
85
86        for (memory, similarity) in vector_results {
87            // Gate membership on similarity alone. Graph and recency only
88            // re-rank, so folding them into the cutoff caps the similarity
89            // term below any useful threshold and would silently drop aged,
90            // unconnected memories that match the query perfectly.
91            if similarity < MIN_SIMILARITY {
92                continue;
93            }
94            if let Some(min_conf) = request.filters.min_confidence
95                && memory.confidence < min_conf
96            {
97                continue;
98            }
99            if let Some(ref source) = request.filters.source
100                && !self.store.source_admits(&memory, source)?
101            {
102                continue;
103            }
104            if let Some(ref mt) = request.filters.memory_type
105                && &memory.memory_type != mt
106            {
107                continue;
108            }
109
110            let recency_score = self.compute_recency(&memory);
111            let graph_score = self
112                .compute_graph_relevance(&memory, &query_entity_ids)
113                .unwrap_or(0.0);
114
115            let mut rank_score = (similarity * self.vector_weight)
116                + (graph_score * self.graph_weight)
117                + (recency_score * self.recency_weight);
118
119            if let Some(ref qp) = request.filters.project_path
120                && memory.project_path.as_deref() == Some(qp.as_str())
121            {
122                rank_score *= PROJECT_AFFINITY_BOOST;
123            }
124
125            scored.push(QueryResult {
126                memory,
127                score: rank_score,
128                relevance: similarity,
129                path: Vec::new(),
130            });
131        }
132
133        scored.sort_by(|a, b| {
134            b.score
135                .partial_cmp(&a.score)
136                .unwrap_or(std::cmp::Ordering::Equal)
137        });
138        scored.truncate(request.limit);
139
140        Ok(scored)
141    }
142
143    fn compute_recency(&self, memory: &Memory) -> f32 {
144        blend::compute_recency(memory)
145    }
146
147    fn resolve_query_entities(&self, names: &[String]) -> HashSet<Uuid> {
148        blend::resolve_query_entities(self.store, names)
149    }
150
151    fn compute_graph_relevance(
152        &self,
153        memory: &Memory,
154        query_entity_ids: &HashSet<Uuid>,
155    ) -> Result<f32> {
156        blend::compute_graph_relevance(self.store, memory, query_entity_ids)
157    }
158
159    pub fn store(&self) -> &S {
160        self.store
161    }
162}
163
164#[cfg(test)]
165mod tests {
166    use super::*;
167    use crate::schema::{Conversation, Entity, MemoryType, Relation, RelationType};
168    use chrono::{Duration, Utc};
169
170    struct StubStore {
171        vector_results: Vec<(Memory, f32)>,
172        relations: Vec<Relation>,
173        entities: Vec<Entity>,
174    }
175
176    impl StubStore {
177        fn new(vector_results: Vec<(Memory, f32)>, relations: Vec<Relation>) -> Self {
178            Self {
179                vector_results,
180                relations,
181                entities: Vec::new(),
182            }
183        }
184    }
185
186    impl Store for StubStore {
187        // Truncates like a real store, so a test can put a memory past the
188        // cutoff and see whether the filters below it can still reach one.
189        fn vector_search(&self, _embedding: &[f32], limit: usize) -> Result<Vec<(Memory, f32)>> {
190            Ok(self.vector_results.iter().take(limit).cloned().collect())
191        }
192
193        // The stub's whole corpus is vector_results, so a scoped exhaustive
194        // scan is that list filtered, not the unfiltered prefix it returns
195        // above.
196        fn filtered_vector_search(
197            &self,
198            _embedding: &[f32],
199            source: Option<&str>,
200            memory_type: Option<MemoryType>,
201            project_path: Option<&str>,
202            limit: usize,
203        ) -> Result<Vec<(Memory, f32)>> {
204            Ok(self
205                .vector_results
206                .iter()
207                .filter(|(m, _)| source.is_none_or(|s| self.source_admits(m, s).unwrap()))
208                .filter(|(m, _)| memory_type.is_none_or(|t| m.memory_type == t))
209                .filter(|(m, _)| project_path.is_none_or(|p| m.project_path.as_deref() == Some(p)))
210                .take(limit)
211                .cloned()
212                .collect())
213        }
214
215        // Stands in for the sqlite lookup through distillation_log: every
216        // distillate here descends from a "cursor" raw.
217        fn source_admits(&self, memory: &Memory, source: &str) -> Result<bool> {
218            Ok(memory.source == source || (memory.source == "consolidation" && source == "cursor"))
219        }
220
221        fn get_relations(
222            &self,
223            node_id: Uuid,
224            relation_type: Option<RelationType>,
225        ) -> Result<Vec<Relation>> {
226            Ok(self
227                .relations
228                .iter()
229                .filter(|r| r.from_id == node_id)
230                .filter(|r| {
231                    relation_type
232                        .map(|rt| rt == r.relation_type)
233                        .unwrap_or(true)
234                })
235                .cloned()
236                .collect())
237        }
238
239        fn store_memory(&self, _m: &Memory) -> Result<()> {
240            unimplemented!()
241        }
242        fn get_memory(&self, _id: Uuid) -> Result<Option<Memory>> {
243            unimplemented!()
244        }
245        fn delete_memory(&self, _id: Uuid) -> Result<()> {
246            unimplemented!()
247        }
248        fn store_entity(&self, _e: &Entity) -> Result<()> {
249            unimplemented!()
250        }
251        fn get_entity(&self, _id: Uuid) -> Result<Option<Entity>> {
252            unimplemented!()
253        }
254        fn find_entity_by_name(&self, name: &str) -> Result<Option<Entity>> {
255            Ok(self.entities.iter().find(|e| e.name == name).cloned())
256        }
257        fn store_conversation(&self, _c: &Conversation) -> Result<()> {
258            unimplemented!()
259        }
260        fn store_relation(&self, _r: &Relation) -> Result<()> {
261            unimplemented!()
262        }
263        fn traverse(&self, _id: Uuid, _depth: u32) -> Result<Vec<(Memory, Vec<Relation>)>> {
264            unimplemented!()
265        }
266        fn memories_by_source(&self, _s: &str) -> Result<Vec<Memory>> {
267            unimplemented!()
268        }
269        fn memories_by_type(&self, _mt: MemoryType) -> Result<Vec<Memory>> {
270            unimplemented!()
271        }
272        fn update_memory(&self, _m: &Memory) -> Result<()> {
273            unimplemented!()
274        }
275        fn record_access(&self, _memory: &Memory) -> Result<()> {
276            unimplemented!()
277        }
278        fn bm25_search(&self, _q: &str, _limit: usize) -> Result<Vec<Memory>> {
279            unimplemented!()
280        }
281        fn get_memory_with_embedding(&self, _id: Uuid) -> Result<Option<Memory>> {
282            unimplemented!()
283        }
284        fn memory_count(&self) -> Result<usize> {
285            unimplemented!()
286        }
287        fn all_memory_ids(&self) -> Result<Vec<Uuid>> {
288            unimplemented!()
289        }
290        fn all_relations(&self) -> Result<Vec<Relation>> {
291            unimplemented!()
292        }
293    }
294
295    fn memory_aged(content: &str, days_old: i64) -> Memory {
296        let when = Utc::now() - Duration::days(days_old);
297        let mut m = Memory::new(
298            content.to_string(),
299            MemoryType::Semantic,
300            "test".to_string(),
301            String::new(),
302        );
303        m.created_at = when;
304        m.last_accessed = when;
305        m
306    }
307
308    fn query() -> QueryRequest {
309        QueryRequest {
310            text: "query text".to_string(),
311            embedding: vec![0.0_f32; 384],
312            limit: 10,
313            filters: QueryFilters::default(),
314        }
315    }
316
317    #[test]
318    fn source_filter_reaches_past_the_unfiltered_cutoff() {
319        // recall() fetches limit*3 vectors, so 30 same-source memories fill the
320        // window completely. A post-filter over that window can only return
321        // nothing; the scoped scan has to look at the whole corpus.
322        let mut corpus: Vec<(Memory, f32)> = (0..30)
323            .map(|i| (memory_aged(&format!("crowding note {i}"), 1), 0.95))
324            .collect();
325        let mut wanted = memory_aged("the memory the caller asked to scope to", 1);
326        wanted.source = "ask".to_string();
327        let wanted_id = wanted.id;
328        corpus.push((wanted, 0.80));
329
330        let store = StubStore::new(corpus, Vec::new());
331        let mut request = query();
332        request.filters.source = Some("ask".to_string());
333
334        let results = QueryEngine::new(&store).recall(&request).unwrap();
335        assert_eq!(results.len(), 1);
336        assert_eq!(results[0].memory.id, wanted_id);
337    }
338
339    #[test]
340    fn source_filter_defers_to_the_store_for_distillates() {
341        let mut distillate = memory_aged("distilled from a cursor transcript", 1);
342        distillate.source = "consolidation".to_string();
343        let mut other = memory_aged("distilled from something else", 1);
344        other.source = "claude-code".to_string();
345        let distillate_id = distillate.id;
346        let store = StubStore::new(vec![(distillate, 0.9), (other, 0.9)], Vec::new());
347        let mut request = query();
348        request.filters.source = Some("cursor".to_string());
349
350        let results = QueryEngine::new(&store).recall(&request).unwrap();
351        let ids: Vec<Uuid> = results.iter().map(|r| r.memory.id).collect();
352        assert_eq!(ids, vec![distillate_id]);
353    }
354
355    #[test]
356    fn aged_unconnected_memory_survives_on_strong_similarity() {
357        // The memory is 180 days old with no relations, so graph and recency
358        // contribute nothing; it must still be recalled because a strong
359        // semantic match is what relevance means.
360        let store = StubStore::new(
361            vec![(
362                memory_aged("kuzu was chosen as the embedded graph store", 180),
363                0.92,
364            )],
365            Vec::new(),
366        );
367        let results = QueryEngine::new(&store).recall(&query()).unwrap();
368        assert_eq!(results.len(), 1);
369    }
370
371    #[test]
372    fn well_connected_low_similarity_memory_is_excluded() {
373        // A fresh, heavily reinforced memory whose similarity is weak must be
374        // dropped, because graph and recency re-rank matches but must not
375        // promote an irrelevant memory past the similarity floor.
376        let weak = memory_aged("an entirely unrelated topic", 0);
377        let weak_id = weak.id;
378        let reinforces = |to| Relation {
379            from_id: weak_id,
380            to_id: to,
381            relation_type: RelationType::Reinforces,
382            strength: 1.0,
383            context: None,
384        };
385        let store = StubStore::new(
386            vec![(weak, 0.45)],
387            vec![
388                reinforces(Uuid::new_v4()),
389                reinforces(Uuid::new_v4()),
390                reinforces(Uuid::new_v4()),
391            ],
392        );
393        let results = QueryEngine::new(&store).recall(&query()).unwrap();
394        assert!(results.is_empty());
395    }
396
397    #[test]
398    fn results_are_ranked_by_blended_score() {
399        // Both memories clear the similarity floor with equal similarity, so
400        // recency must break the tie and rank the fresher memory first.
401        let store = StubStore::new(
402            vec![
403                (memory_aged("older relevant note", 200), 0.80),
404                (memory_aged("fresher relevant note", 0), 0.80),
405            ],
406            Vec::new(),
407        );
408        let results = QueryEngine::new(&store).recall(&query()).unwrap();
409        assert_eq!(results.len(), 2);
410        assert_eq!(results[0].memory.content, "fresher relevant note");
411    }
412
413    fn mentions(from: Uuid, to: Uuid) -> Relation {
414        Relation {
415            from_id: from,
416            to_id: to,
417            relation_type: RelationType::Mentions,
418            strength: 1.0,
419            context: None,
420        }
421    }
422
423    #[test]
424    fn memory_mentioning_a_query_entity_outranks_an_equal_match_that_does_not() {
425        // Two memories, same similarity and same age. One mentions the entity
426        // the query is about; it must rank first because graph proximity is now
427        // query-conditioned rather than a raw connectedness prior.
428        let on_topic = memory_aged("note about the kuzu store", 10);
429        let off_topic = memory_aged("note about something else", 10);
430        let kuzu = Entity::new("kuzu".to_string(), "technology".to_string());
431
432        let mut store = StubStore::new(
433            vec![(off_topic.clone(), 0.80), (on_topic.clone(), 0.80)],
434            vec![mentions(on_topic.id, kuzu.id)],
435        );
436        store.entities = vec![kuzu];
437
438        let mut req = query();
439        req.filters.entity_names = vec!["kuzu".to_string()];
440
441        let results = QueryEngine::new(&store).recall(&req).unwrap();
442        assert_eq!(results.len(), 2);
443        assert_eq!(results[0].memory.content, "note about the kuzu store");
444    }
445
446    #[test]
447    fn unresolvable_query_entity_falls_back_to_structural_ranking() {
448        // An entity name the store has never seen resolves to nothing, so the
449        // graph term reverts to the structural prior and recency breaks the tie
450        // exactly as when no entities are named.
451        let store = StubStore::new(
452            vec![
453                (memory_aged("older relevant note", 200), 0.80),
454                (memory_aged("fresher relevant note", 0), 0.80),
455            ],
456            Vec::new(),
457        );
458        let mut req = query();
459        req.filters.entity_names = vec!["never-seen-entity".to_string()];
460
461        let results = QueryEngine::new(&store).recall(&req).unwrap();
462        assert_eq!(results.len(), 2);
463        assert_eq!(results[0].memory.content, "fresher relevant note");
464    }
465}