llm_kernel/graph/
recall.rs

1//! Smart recall with composite scoring and graph boost.
2
3use std::collections::HashSet;
4
5use rusqlite::Connection;
6
7use crate::error::{KernelError, Result};
8
9use super::algo::{CsrGraph, pagerank_default};
10use super::lifecycle::{parse_iso_to_secs, touch_nodes};
11use super::search::search_nodes_hybrid;
12use super::store::{edges_among, read_nodes};
13use super::types::{NODE_COLUMNS, ScoredNode, escape_like};
14
15/// Weight applied to recency in the composite relevance score.
16pub const W_RECENCY: f64 = 0.20;
17/// Weight applied to node importance in the composite relevance score.
18pub const W_IMPORTANCE: f64 = 0.35;
19/// Weight applied to access frequency in the composite relevance score.
20pub const W_ACCESS: f64 = 0.15;
21/// Weight applied to FTS (full-text search) rank in the composite relevance score.
22pub const W_FTS: f64 = 0.20;
23/// Weight applied to graph-neighbor boost in the composite relevance score.
24pub const W_GRAPH: f64 = 0.10;
25
26/// Structured recall options.
27///
28/// `#[non_exhaustive]` + `Default` lets callers add filters without breaking
29/// struct-literal construction. `tags_any` is the intended symbol-scoped path:
30/// TradingAgentOS stores the symbol as a tag on every node.
31#[derive(Debug, Clone, Default)]
32#[non_exhaustive]
33pub struct RecallOptions {
34    /// Project scope.
35    pub project: Option<String>,
36    /// Free-text hint (lexical match). `None`/empty ⇒ pure structural recall.
37    pub hint: Option<String>,
38    /// Restrict to these node types (e.g. `["decision"]`).
39    pub node_types: Vec<String>,
40    /// Match any of these tags (OR). Use for symbol-scoped recall.
41    pub tags_any: Vec<String>,
42    /// `created >=` (ISO8601).
43    pub since: Option<String>,
44    /// Result cap.
45    pub limit: usize,
46    /// Whether to increment `access_count` on retrieved nodes.
47    ///
48    /// Defaults to `false` (via `#[derive(Default)]`). The [`legacy`](Self::legacy)
49    /// constructor sets this to `true` for backward compatibility with the old
50    /// `smart_recall(project, hint, limit)` signature. New callers building
51    /// `RecallOptions` directly get read-only recall by default — pass `true`
52    /// explicitly to opt into mutation.
53    pub touch: bool,
54}
55
56impl RecallOptions {
57    /// Backward-compatible defaults matching the old `smart_recall(project, hint, limit)`.
58    pub fn legacy(project: Option<&str>, hint: Option<&str>, limit: usize) -> Self {
59        Self {
60            project: project.map(str::to_string),
61            hint: hint.map(str::to_string),
62            limit,
63            touch: true,
64            ..Default::default()
65        }
66    }
67}
68
69/// Scoring: `recency(20%) + importance(35%) + access_freq(15%) + FTS(20%) + graph_boost(10%)`
70///
71/// Stale nodes (tagged "stale") are excluded. Retrieved nodes have their
72/// access_count incremented unless `touch` is false.
73pub fn smart_recall(
74    conn: &Connection,
75    project: Option<&str>,
76    hint: Option<&str>,
77    limit: usize,
78) -> Result<Vec<ScoredNode>> {
79    smart_recall_with(conn, &RecallOptions::legacy(project, hint, limit))
80}
81
82/// Structured recall — see [`RecallOptions`].
83pub fn smart_recall_with(conn: &Connection, opts: &RecallOptions) -> Result<Vec<ScoredNode>> {
84    let limit = opts.limit;
85    let hint = opts.hint.as_deref();
86    let now_secs = std::time::SystemTime::now()
87        .duration_since(std::time::SystemTime::UNIX_EPOCH)
88        .unwrap_or_default()
89        .as_secs();
90
91    // Gather lexical matches if hint is provided.
92    // Uses the hybrid path so short CJK hints (which the trigram tokenizer cannot
93    // match) still contribute — see `search_nodes_hybrid`.
94    let fts_ids: HashSet<String> = if let Some(h) = hint {
95        if !h.is_empty() {
96            search_nodes_hybrid(conn, h, limit * 4)?
97                .into_iter()
98                .map(|n| n.id.clone())
99                .collect()
100        } else {
101            Default::default()
102        }
103    } else {
104        Default::default()
105    };
106
107    // A non-empty hint that matched nothing means nothing is relevant. Returning
108    // the globally-most-important nodes instead (what the candidate query below
109    // does on its own) makes recall answer every query with *something*, which
110    // then gets injected into an LLM prompt as if it were relevant context.
111    if hint.is_some_and(|h| !h.is_empty()) && fts_ids.is_empty() {
112        return Ok(Vec::new());
113    }
114
115    // Fetch candidate nodes (broad set)
116    let candidate_limit = (limit * 4).max(40) as i64;
117    let mut conditions: Vec<String> = vec!["',' || tags || ',' NOT LIKE '%,stale,%'".to_string()];
118    let mut param_vals: Vec<Box<dyn rusqlite::ToSql>> = vec![];
119    if let Some(p) = &opts.project {
120        conditions.push("(',' || projects || ',' LIKE '%,' || ? || ',%' ESCAPE '\\')".to_string());
121        param_vals.push(Box::new(escape_like(p)));
122    }
123    if !opts.node_types.is_empty() {
124        let placeholders = vec!["?"; opts.node_types.len()].join(",");
125        conditions.push(format!("type IN ({placeholders})"));
126        for nt in &opts.node_types {
127            param_vals.push(Box::new(nt.clone()));
128        }
129    }
130    if !opts.tags_any.is_empty() {
131        // OR each requested tag against the tags CSV (e.g. symbol-scoped recall).
132        let tag_clauses: Vec<String> = opts
133            .tags_any
134            .iter()
135            .map(|_| "',' || tags || ',' LIKE '%,' || ? || ',%' ESCAPE '\\'".to_string())
136            .collect();
137        conditions.push(format!("({})", tag_clauses.join(" OR ")));
138        for t in &opts.tags_any {
139            param_vals.push(Box::new(escape_like(t)));
140        }
141    }
142    if let Some(s) = &opts.since {
143        conditions.push("created >= ?".to_string());
144        param_vals.push(Box::new(s.clone()));
145    }
146    let where_clause = format!("WHERE {}", conditions.join(" AND "));
147    let sql = format!(
148        "SELECT {NODE_COLUMNS} FROM nodes {where_clause}
149         ORDER BY importance DESC, updated DESC
150         LIMIT {candidate_limit}"
151    );
152
153    let mut stmt = conn
154        .prepare(&sql)
155        .map_err(|e| KernelError::Store(e.to_string()))?;
156    let refs: Vec<&dyn rusqlite::ToSql> = param_vals.iter().map(|b| b.as_ref()).collect();
157    let mut candidates: Vec<super::types::GraphNode> = stmt
158        .query_map(refs.as_slice(), super::types::row_to_node)
159        .map(|rows| rows.filter_map(|r| r.ok()).collect())
160        .unwrap_or_default();
161
162    // With a hint, lexical relevance gates the result set. The candidate query
163    // above is ordered by importance, so a matching-but-unimportant node could
164    // fall outside it entirely while an unrelated-but-important one ranked top —
165    // the reason `W_FTS` was effectively unreachable on larger graphs.
166    if !fts_ids.is_empty() {
167        candidates.retain(|n| fts_ids.contains(&n.id));
168        // Pull in matches the importance-ordered window missed — in ONE batched
169        // query, not a per-id `read_node` loop (which was both an N+1 and a
170        // filter bypass: the recovered nodes were read with no WHERE clause, so
171        // an FTS hit that happened to fall outside the scope filter — e.g. a
172        // generic node mentioning the symbol when `tags_any` scopes to that
173        // symbol — leaked into results). The same `where_clause` is re-applied
174        // client-side to the recovered nodes, keeping recall scope-tight.
175        let present: HashSet<&str> = candidates.iter().map(|n| n.id.as_str()).collect();
176        let missing: Vec<&str> = fts_ids
177            .iter()
178            .map(String::as_str)
179            .filter(|id| !present.contains(*id))
180            .collect();
181        if !missing.is_empty() {
182            for node in read_nodes(conn, &missing).unwrap_or_default() {
183                if passes_scope_filters(&node, opts) {
184                    candidates.push(node);
185                }
186            }
187        }
188    }
189
190    // Score each candidate
191    let mut scored: Vec<ScoredNode> = candidates
192        .into_iter()
193        .map(|node| {
194            let recency = compute_recency(&node.updated, now_secs);
195            let importance = node.importance;
196            let access_freq = (node.access_count.max(0) as f64 / 20.0).min(1.0);
197            let fts_match = if fts_ids.contains(&node.id) { 1.0 } else { 0.0 };
198
199            let score = W_RECENCY * recency
200                + W_IMPORTANCE * importance
201                + W_ACCESS * access_freq
202                + W_FTS * fts_match;
203
204            ScoredNode { node, score }
205        })
206        .collect();
207
208    scored.sort_by(|a, b| {
209        b.score
210            .partial_cmp(&a.score)
211            .unwrap_or(std::cmp::Ordering::Equal)
212    });
213    scored.truncate(limit);
214
215    // Graph-boost pass: PageRank centrality over the induced subgraph of the
216    // top candidates. Replaces the former neighbor-weight-sum (an approximate
217    // degree centrality) with true PageRank — strong connectors rise, dead
218    // ends sink. The pagerank math is backend-agnostic, so the SQLite and
219    // PostgreSQL recall paths share identical scoring (zero drift).
220    if scored.len() > 1 {
221        const MAX_GRAPH_BOOST_PARTICIPANTS: usize = 100;
222        let candidate_ids: Vec<String> = scored
223            .iter()
224            .take(MAX_GRAPH_BOOST_PARTICIPANTS)
225            .map(|sn| sn.node.id.clone())
226            .collect();
227        let id_refs: Vec<&str> = candidate_ids.iter().map(String::as_str).collect();
228        let sub_edges = edges_among(conn, &id_refs).unwrap_or_default();
229        let csr = CsrGraph::from_edges(&candidate_ids, &sub_edges);
230        let pr = pagerank_default(&csr);
231        let max_pr = pr.iter().copied().fold(0.0_f64, f64::max).max(1e-12);
232        let pr_map: std::collections::HashMap<String, f64> = candidate_ids
233            .iter()
234            .zip(pr.iter())
235            .map(|(id, &s)| (id.clone(), s / max_pr))
236            .collect();
237        for sn in &mut scored {
238            let boost = pr_map.get(&sn.node.id).copied().unwrap_or(0.0);
239            sn.score += W_GRAPH * boost;
240        }
241        scored.sort_by(|a, b| {
242            b.score
243                .partial_cmp(&a.score)
244                .unwrap_or(std::cmp::Ordering::Equal)
245        });
246    }
247
248    // Touch retrieved nodes (gated — LLM-context recall should not mutate state).
249    if opts.touch {
250        let ids: Vec<String> = scored.iter().map(|sn| sn.node.id.clone()).collect();
251        touch_nodes(conn, &ids);
252    }
253
254    Ok(scored)
255}
256
257/// Compute recency score (0.0–1.0) with exponential decay, half-life = 30 days.
258///
259/// Exposed so non-SQLite backends (e.g. the `graph-pg` PostgreSQL backend at
260/// `src/graph/pg.rs`) can score candidates with identical recency math — no
261/// drift across backends.
262pub fn compute_recency(updated: &str, now_secs: u64) -> f64 {
263    let node_secs = parse_iso_to_secs(updated);
264    if node_secs == 0 || node_secs > now_secs {
265        return 0.5;
266    }
267    let age_days = (now_secs - node_secs) as f64 / 86400.0;
268    let half_life = 30.0;
269    (-age_days * (2.0_f64.ln()) / half_life).exp()
270}
271
272/// Re-apply the [`RecallOptions`] scope filters to a node recovered outside the
273/// candidate query (the FTS-window recovery path).
274///
275/// Mirrors the SQL `where_clause` built in [`smart_recall_with`] (stale
276/// exclusion, project, node_types, tags_any, since) so a recovered node can
277/// never widen the result set beyond the requested scope. Kept in lock-step
278/// with that query: if a filter is added there, add it here too.
279fn passes_scope_filters(node: &super::types::GraphNode, opts: &RecallOptions) -> bool {
280    // stale exclusion: tags CSV contains "stale"
281    if node.tags.iter().any(|t| t == "stale") {
282        return false;
283    }
284    if let Some(p) = &opts.project
285        && !node.projects.iter().any(|np| np == p)
286    {
287        return false;
288    }
289    if !opts.node_types.is_empty() && !opts.node_types.contains(&node.node_type) {
290        return false;
291    }
292    if !opts.tags_any.is_empty()
293        && !opts
294            .tags_any
295            .iter()
296            .any(|t| node.tags.iter().any(|nt| nt == t))
297    {
298        return false;
299    }
300    if let Some(s) = &opts.since {
301        // Lexicographic compare is valid for ISO8601/Zulu timestamps of equal
302        // shape (what upsert_node writes). Mismatched shapes would compare
303        // wrong, but the candidate query uses the same `created >= ?` semantics,
304        // so this stays consistent with it.
305        if node.created.as_str() < s.as_str() {
306            return false;
307        }
308    }
309    true
310}
311
312#[cfg(test)]
313mod tests {
314    use super::*;
315    use crate::graph::schema::init_graph_schema;
316    use crate::graph::store::{append_edge, upsert_node};
317    use crate::graph::types::GraphEdge;
318    use rusqlite::Connection;
319
320    fn mem_db() -> Connection {
321        let conn = Connection::open_in_memory().unwrap();
322        init_graph_schema(&conn).unwrap();
323        conn
324    }
325
326    fn test_node(id: &str, importance: f64, tags: Vec<&str>) -> crate::graph::types::GraphNode {
327        crate::graph::types::GraphNode {
328            id: id.to_string(),
329            node_type: "concept".to_string(),
330            title: format!("Node {id}"),
331            body: String::new(),
332            tags: tags.into_iter().map(|s| s.to_string()).collect(),
333            projects: vec![],
334            agents: vec![],
335            created: "2026-01-01T00:00:00Z".to_string(),
336            updated: "2026-06-01T00:00:00Z".to_string(),
337            importance,
338            access_count: 0,
339            accessed_at: String::new(),
340        }
341    }
342
343    #[test]
344    fn recall_returns_nodes() {
345        let conn = mem_db();
346        upsert_node(&conn, &test_node("n1", 0.9, vec![])).unwrap();
347        upsert_node(&conn, &test_node("n2", 0.5, vec![])).unwrap();
348        let results = smart_recall(&conn, None, None, 10).unwrap();
349        assert_eq!(results.len(), 2);
350        // Higher importance first
351        assert_eq!(results[0].node.id, "n1");
352    }
353
354    #[test]
355    fn recall_filters_by_project() {
356        let conn = mem_db();
357        let mut n1 = test_node("n1", 0.7, vec![]);
358        n1.projects = vec!["myproj".to_string()];
359        upsert_node(&conn, &n1).unwrap();
360        upsert_node(&conn, &test_node("n2", 0.7, vec![])).unwrap();
361
362        let results = smart_recall(&conn, Some("myproj"), None, 10).unwrap();
363        assert_eq!(results.len(), 1);
364        assert_eq!(results[0].node.id, "n1");
365    }
366
367    #[test]
368    fn recall_with_hint_uses_fts() {
369        let conn = mem_db();
370        let mut n1 = test_node("n1", 0.5, vec![]);
371        n1.title = "Rust ownership model".to_string();
372        n1.body = "borrow checker rules".to_string();
373        upsert_node(&conn, &n1).unwrap();
374
375        let mut n2 = test_node("n2", 0.9, vec![]);
376        n2.title = "Python GIL".to_string();
377        upsert_node(&conn, &n2).unwrap();
378
379        let results = smart_recall(&conn, None, Some("Rust"), 10).unwrap();
380        // n1 should get FTS boost even though n2 has higher base importance
381        assert!(!results.is_empty());
382    }
383
384    #[test]
385    fn recall_excludes_stale() {
386        let conn = mem_db();
387        upsert_node(&conn, &test_node("n1", 0.9, vec!["stale"])).unwrap();
388        upsert_node(&conn, &test_node("n2", 0.5, vec![])).unwrap();
389        let results = smart_recall(&conn, None, None, 10).unwrap();
390        assert_eq!(results.len(), 1);
391        assert_eq!(results[0].node.id, "n2");
392    }
393
394    #[test]
395    fn recall_touches_access_count() {
396        let conn = mem_db();
397        upsert_node(&conn, &test_node("n1", 0.7, vec![])).unwrap();
398        smart_recall(&conn, None, None, 10).unwrap();
399        let node = crate::graph::store::read_node(&conn, "n1")
400            .unwrap()
401            .unwrap();
402        assert_eq!(node.access_count, 1);
403    }
404
405    #[test]
406    fn recall_graph_boost() {
407        let conn = mem_db();
408        upsert_node(&conn, &test_node("n1", 0.7, vec![])).unwrap();
409        upsert_node(&conn, &test_node("n2", 0.7, vec![])).unwrap();
410        append_edge(
411            &conn,
412            &GraphEdge {
413                id: "e1".into(),
414                source: "n1".into(),
415                target: "n2".into(),
416                relation: "related".into(),
417                weight: 1.0,
418                ts: "2026-01-01T00:00:00Z".into(),
419            },
420        )
421        .unwrap();
422
423        let results = smart_recall(&conn, None, None, 10).unwrap();
424        assert_eq!(results.len(), 2);
425        // With hint=None and identical recency/importance/access, every score
426        // component is equal across n1 and n2 EXCEPT the graph boost. n2 is a
427        // dangling sink (no out-edges) and so accrues higher PageRank than n1
428        // — its sole in-bound rank source — so the boost pass must rank n2
429        // above n1. This is the one assertion that exercises the boost's
430        // actual ranking effect (the pagerank math itself is unit-tested in
431        // algo/pagerank.rs).
432        let n1 = results.iter().find(|s| s.node.id == "n1").unwrap();
433        let n2 = results.iter().find(|s| s.node.id == "n2").unwrap();
434        assert!(
435            n2.score > n1.score,
436            "dangling sink n2 must outrank source n1 via PageRank boost"
437        );
438    }
439
440    #[test]
441    fn recall_project_wildcard_is_escaped() {
442        let conn = mem_db();
443        let mut n1 = test_node("n1", 0.7, vec![]);
444        n1.projects = vec!["myproj".to_string()];
445        upsert_node(&conn, &n1).unwrap();
446        // "my%" would match "myproj" as a LIKE wildcard, but escape_like prevents it
447        let results = smart_recall(&conn, Some("my%"), None, 10).unwrap();
448        assert!(results.is_empty());
449    }
450
451    #[test]
452    fn recall_fts_recovery_respects_tag_scope() {
453        // Regression for the scope-bypass blocker: an FTS hint that also matches
454        // an out-of-scope node must NOT pull that node in via the recovery path.
455        //
456        // Setup: n1 is symbol-scoped (tag "AAPL"), low importance; n2 mentions
457        // "AAPL" in its body but is NOT tagged "AAPL" and has high importance.
458        // With `tags_any = ["AAPL"]` + hint "AAPL", the FTS window recovers both
459        // ids, but only n1 should survive — n2 fails the tag scope filter.
460        let conn = mem_db();
461        let mut n1 = test_node("n1", 0.1, vec!["AAPL"]);
462        n1.title = "AAPL position".to_string();
463        n1.body = "earnings call notes".to_string();
464        upsert_node(&conn, &n1).unwrap();
465
466        let mut n2 = test_node("n2", 0.99, vec![]);
467        n2.title = "Market commentary".to_string();
468        n2.body = "AAPL mentioned in passing".to_string();
469        upsert_node(&conn, &n2).unwrap();
470
471        let opts = RecallOptions {
472            hint: Some("AAPL".to_string()),
473            tags_any: vec!["AAPL".to_string()],
474            limit: 10,
475            ..Default::default()
476        };
477        let results = smart_recall_with(&conn, &opts).unwrap();
478        let ids: Vec<&str> = results.iter().map(|s| s.node.id.as_str()).collect();
479        assert!(
480            ids.iter().all(|id| *id != "n2"),
481            "out-of-scope n2 must not leak via FTS recovery; got {ids:?}"
482        );
483        assert!(
484            ids.contains(&"n1"),
485            "in-scope n1 must be present; got {ids:?}"
486        );
487    }
488
489    #[test]
490    fn recall_fts_recovery_respects_project_scope() {
491        // Same bypass, project dimension: a recovered node in the wrong project
492        // is dropped.
493        let conn = mem_db();
494        let mut n1 = test_node("n1", 0.1, vec![]);
495        n1.title = "AAPL note".to_string();
496        n1.projects = vec!["projA".to_string()];
497        upsert_node(&conn, &n1).unwrap();
498
499        let mut n2 = test_node("n2", 0.99, vec![]);
500        n2.title = "AAPL cross-ref".to_string();
501        n2.projects = vec!["projB".to_string()];
502        upsert_node(&conn, &n2).unwrap();
503
504        let opts = RecallOptions {
505            project: Some("projA".to_string()),
506            hint: Some("AAPL".to_string()),
507            limit: 10,
508            ..Default::default()
509        };
510        let results = smart_recall_with(&conn, &opts).unwrap();
511        let ids: Vec<&str> = results.iter().map(|s| s.node.id.as_str()).collect();
512        assert!(
513            ids.iter().all(|id| *id != "n2"),
514            "wrong-project n2 must not leak via FTS recovery; got {ids:?}"
515        );
516    }
517}
llm_kernel/graph/recall.rs

llm_kernel/graph/
recall.rs
