llm_kernel/graph/
recall.rs

1//! Smart recall with composite scoring and graph boost.
2
3use std::collections::HashSet;
4
5use rusqlite::Connection;
6
7use crate::error::{KernelError, Result};
8
9use super::algo::{CsrGraph, pagerank_default};
10use super::lifecycle::{parse_iso_to_secs, touch_nodes};
11use super::search::search_nodes_hybrid;
12use super::store::{edges_among, read_nodes};
13use super::types::{NODE_COLUMNS, ScoredNode, escape_like};
14
15/// Weight applied to recency in the composite relevance score.
16pub const W_RECENCY: f64 = 0.20;
17/// Weight applied to node importance in the composite relevance score.
18pub const W_IMPORTANCE: f64 = 0.35;
19/// Weight applied to access frequency in the composite relevance score.
20pub const W_ACCESS: f64 = 0.15;
21/// Weight applied to FTS (full-text search) rank in the composite relevance score.
22pub const W_FTS: f64 = 0.20;
23/// Weight applied to graph-neighbor boost in the composite relevance score.
24pub const W_GRAPH: f64 = 0.10;
25
26/// Structured recall options.
27///
28/// `#[non_exhaustive]` + `Default` lets callers add filters without breaking
29/// struct-literal construction. `tags_any` is the intended symbol-scoped path:
30/// TradingAgentOS stores the symbol as a tag on every node.
31#[derive(Debug, Clone, Default)]
32#[non_exhaustive]
33pub struct RecallOptions {
34    /// Project scope.
35    pub project: Option<String>,
36    /// Free-text hint (lexical match). `None`/empty ⇒ pure structural recall.
37    pub hint: Option<String>,
38    /// Restrict to these node types (e.g. `["decision"]`).
39    pub node_types: Vec<String>,
40    /// Match any of these tags (OR). Use for symbol-scoped recall.
41    pub tags_any: Vec<String>,
42    /// `created >=` (ISO8601).
43    pub since: Option<String>,
44    /// Result cap.
45    pub limit: usize,
46    /// Whether to increment `access_count` on retrieved nodes.
47    ///
48    /// Defaults to `false` (via `#[derive(Default)]`). The [`legacy`](Self::legacy)
49    /// constructor sets this to `true` for backward compatibility with the old
50    /// `smart_recall(project, hint, limit)` signature. New callers building
51    /// `RecallOptions` directly get read-only recall by default — pass `true`
52    /// explicitly to opt into mutation.
53    pub touch: bool,
54}
55
56impl RecallOptions {
57    /// Backward-compatible defaults matching the old `smart_recall(project, hint, limit)`.
58    pub fn legacy(project: Option<&str>, hint: Option<&str>, limit: usize) -> Self {
59        Self {
60            project: project.map(str::to_string),
61            hint: hint.map(str::to_string),
62            limit,
63            touch: true,
64            ..Default::default()
65        }
66    }
67}
68
69/// Scoring: `recency(20%) + importance(35%) + access_freq(15%) + FTS(20%) + graph_boost(10%)`
70///
71/// Stale nodes (tagged "stale") are excluded. Retrieved nodes have their
72/// access_count incremented unless `touch` is false.
73pub fn smart_recall(
74    conn: &Connection,
75    project: Option<&str>,
76    hint: Option<&str>,
77    limit: usize,
78) -> Result<Vec<ScoredNode>> {
79    smart_recall_with(conn, &RecallOptions::legacy(project, hint, limit))
80}
81
82/// Structured recall — see [`RecallOptions`].
83pub fn smart_recall_with(conn: &Connection, opts: &RecallOptions) -> Result<Vec<ScoredNode>> {
84    let limit = opts.limit;
85    let hint = opts.hint.as_deref();
86    let now_secs = std::time::SystemTime::now()
87        .duration_since(std::time::SystemTime::UNIX_EPOCH)
88        .unwrap_or_default()
89        .as_secs();
90
91    // Gather lexical matches if hint is provided.
92    // Uses the hybrid path so short CJK hints (which the trigram tokenizer cannot
93    // match) still contribute — see `search_nodes_hybrid`.
94    let fts_ids: HashSet<String> = if let Some(h) = hint {
95        if !h.is_empty() {
96            search_nodes_hybrid(conn, h, limit * 4)?
97                .into_iter()
98                .map(|n| n.id.clone())
99                .collect()
100        } else {
101            Default::default()
102        }
103    } else {
104        Default::default()
105    };
106
107    // A non-empty hint that matched nothing means nothing is relevant. Returning
108    // the globally-most-important nodes instead (what the candidate query below
109    // does on its own) makes recall answer every query with *something*, which
110    // then gets injected into an LLM prompt as if it were relevant context.
111    if hint.is_some_and(|h| !h.is_empty()) && fts_ids.is_empty() {
112        return Ok(Vec::new());
113    }
114
115    // Fetch candidate nodes (broad set)
116    let candidate_limit = (limit * 4).max(40) as i64;
117    let mut conditions: Vec<String> = vec!["',' || tags || ',' NOT LIKE '%,stale,%'".to_string()];
118    let mut param_vals: Vec<Box<dyn rusqlite::ToSql>> = vec![];
119    if let Some(p) = &opts.project {
120        conditions.push("(',' || projects || ',' LIKE '%,' || ? || ',%' ESCAPE '\\')".to_string());
121        param_vals.push(Box::new(escape_like(p)));
122    }
123    if !opts.node_types.is_empty() {
124        let placeholders = vec!["?"; opts.node_types.len()].join(",");
125        conditions.push(format!("type IN ({placeholders})"));
126        for nt in &opts.node_types {
127            param_vals.push(Box::new(nt.clone()));
128        }
129    }
130    if !opts.tags_any.is_empty() {
131        // OR each requested tag against the tags CSV (e.g. symbol-scoped recall).
132        let tag_clauses: Vec<String> = opts
133            .tags_any
134            .iter()
135            .map(|_| "',' || tags || ',' LIKE '%,' || ? || ',%' ESCAPE '\\'".to_string())
136            .collect();
137        conditions.push(format!("({})", tag_clauses.join(" OR ")));
138        for t in &opts.tags_any {
139            param_vals.push(Box::new(escape_like(t)));
140        }
141    }
142    if let Some(s) = &opts.since {
143        conditions.push("created >= ?".to_string());
144        param_vals.push(Box::new(s.clone()));
145    }
146    let where_clause = format!("WHERE {}", conditions.join(" AND "));
147    let sql = format!(
148        "SELECT {NODE_COLUMNS} FROM nodes {where_clause}
149         ORDER BY importance DESC, updated DESC
150         LIMIT {candidate_limit}"
151    );
152
153    let mut stmt = conn
154        .prepare(&sql)
155        .map_err(|e| KernelError::Store(e.to_string()))?;
156    let refs: Vec<&dyn rusqlite::ToSql> = param_vals.iter().map(|b| b.as_ref()).collect();
157    let mut candidates: Vec<super::types::GraphNode> = stmt
158        .query_map(refs.as_slice(), super::types::row_to_node)
159        .map(|rows| rows.filter_map(|r| r.ok()).collect())
160        .unwrap_or_default();
161
162    // With a hint, lexical relevance gates the result set. The candidate query
163    // above is ordered by importance, so a matching-but-unimportant node could
164    // fall outside it entirely while an unrelated-but-important one ranked top —
165    // the reason `W_FTS` was effectively unreachable on larger graphs.
166    if !fts_ids.is_empty() {
167        candidates.retain(|n| fts_ids.contains(&n.id));
168        // Pull in matches the importance-ordered window missed — in ONE batched
169        // query, not a per-id `read_node` loop (which was both an N+1 and a
170        // filter bypass: the recovered nodes were read with no WHERE clause, so
171        // an FTS hit that happened to fall outside the scope filter — e.g. a
172        // generic node mentioning the symbol when `tags_any` scopes to that
173        // symbol — leaked into results). The same `where_clause` is re-applied
174        // client-side to the recovered nodes, keeping recall scope-tight.
175        let present: HashSet<&str> = candidates.iter().map(|n| n.id.as_str()).collect();
176        let missing: Vec<&str> = fts_ids
177            .iter()
178            .map(String::as_str)
179            .filter(|id| !present.contains(*id))
180            .collect();
181        if !missing.is_empty() {
182            for node in read_nodes(conn, &missing).unwrap_or_default() {
183                if passes_scope_filters(&node, opts) {
184                    candidates.push(node);
185                }
186            }
187        }
188    }
189
190    // Score each candidate
191    let mut scored: Vec<ScoredNode> = candidates
192        .into_iter()
193        .map(|node| {
194            let recency = compute_recency(&node.updated, now_secs);
195            let importance = node.importance;
196            let access_freq = (node.access_count.max(0) as f64 / 20.0).min(1.0);
197            let fts_match = if fts_ids.contains(&node.id) { 1.0 } else { 0.0 };
198
199            let score = W_RECENCY * recency
200                + W_IMPORTANCE * importance
201                + W_ACCESS * access_freq
202                + W_FTS * fts_match;
203
204            ScoredNode { node, score }
205        })
206        .collect();
207
208    scored.sort_by(|a, b| {
209        b.score
210            .partial_cmp(&a.score)
211            .unwrap_or(std::cmp::Ordering::Equal)
212    });
213    scored.truncate(limit);
214
215    // Graph-boost pass: PageRank centrality over the induced subgraph of the
216    // top candidates. Replaces the former neighbor-weight-sum (an approximate
217    // degree centrality) with true PageRank — strong connectors rise, dead
218    // ends sink. The pagerank math is backend-agnostic, so the SQLite and
219    // PostgreSQL recall paths share identical scoring (zero drift).
220    if scored.len() > 1 {
221        const MAX_GRAPH_BOOST_PARTICIPANTS: usize = 100;
222        let candidate_ids: Vec<String> = scored
223            .iter()
224            .take(MAX_GRAPH_BOOST_PARTICIPANTS)
225            .map(|sn| sn.node.id.clone())
226            .collect();
227        let id_refs: Vec<&str> = candidate_ids.iter().map(String::as_str).collect();
228        let sub_edges = edges_among(conn, &id_refs).unwrap_or_default();
229        let csr = CsrGraph::from_edges(&candidate_ids, &sub_edges);
230        let pr = pagerank_default(&csr);
231        let max_pr = pr.iter().copied().fold(0.0_f64, f64::max).max(1e-12);
232        let pr_map: std::collections::HashMap<String, f64> = candidate_ids
233            .iter()
234            .zip(pr.iter())
235            .map(|(id, &s)| (id.clone(), s / max_pr))
236            .collect();
237        for sn in &mut scored {
238            let boost = pr_map.get(&sn.node.id).copied().unwrap_or(0.0);
239            sn.score += W_GRAPH * boost;
240        }
241        scored.sort_by(|a, b| {
242            b.score
243                .partial_cmp(&a.score)
244                .unwrap_or(std::cmp::Ordering::Equal)
245        });
246    }
247
248    // Touch retrieved nodes (gated — LLM-context recall should not mutate state).
249    if opts.touch {
250        let ids: Vec<String> = scored.iter().map(|sn| sn.node.id.clone()).collect();
251        touch_nodes(conn, &ids);
252    }
253
254    Ok(scored)
255}
256
257/// Compute recency score (0.0–1.0) with exponential decay, half-life = 30 days.
258///
259/// Exposed so non-SQLite backends (e.g. the `graph-pg` PostgreSQL backend at
260/// `src/graph/pg.rs`) can score candidates with identical recency math — no
261/// drift across backends.
262pub fn compute_recency(updated: &str, now_secs: u64) -> f64 {
263    let node_secs = parse_iso_to_secs(updated);
264    if node_secs == 0 || node_secs > now_secs {
265        return 0.5;
266    }
267    let age_days = (now_secs - node_secs) as f64 / 86400.0;
268    let half_life = 30.0;
269    (-age_days * (2.0_f64.ln()) / half_life).exp()
270}
271
272/// Re-apply the [`RecallOptions`] scope filters to a node recovered outside the
273/// candidate query (the FTS-window recovery path).
274///
275/// Mirrors the SQL `where_clause` built in [`smart_recall_with`] (stale
276/// exclusion, project, node_types, tags_any, since) so a recovered node can
277/// never widen the result set beyond the requested scope. Kept in lock-step
278/// with that query: if a filter is added there, add it here too.
279fn passes_scope_filters(node: &super::types::GraphNode, opts: &RecallOptions) -> bool {
280    // stale exclusion: tags CSV contains "stale"
281    if node.tags.iter().any(|t| t == "stale") {
282        return false;
283    }
284    if let Some(p) = &opts.project
285        && !node.projects.iter().any(|np| np == p)
286    {
287        return false;
288    }
289    if !opts.node_types.is_empty() && !opts.node_types.contains(&node.node_type) {
290        return false;
291    }
292    if !opts.tags_any.is_empty()
293        && !opts
294            .tags_any
295            .iter()
296            .any(|t| node.tags.iter().any(|nt| nt == t))
297    {
298        return false;
299    }
300    if let Some(s) = &opts.since {
301        // Lexicographic compare is valid for ISO8601/Zulu timestamps of equal
302        // shape (what upsert_node writes). Mismatched shapes would compare
303        // wrong, but the candidate query uses the same `created >= ?` semantics,
304        // so this stays consistent with it.
305        if node.created.as_str() < s.as_str() {
306            return false;
307        }
308    }
309    true
310}
311
312#[cfg(test)]
313mod tests {
314    use super::*;
315    use crate::graph::schema::init_graph_schema;
316    use crate::graph::store::{append_edge, upsert_node};
317    use crate::graph::types::GraphEdge;
318    use rusqlite::Connection;
319
320    fn mem_db() -> Connection {
321        let conn = Connection::open_in_memory().unwrap();
322        init_graph_schema(&conn).unwrap();
323        conn
324    }
325
326    fn test_node(id: &str, importance: f64, tags: Vec<&str>) -> crate::graph::types::GraphNode {
327        crate::graph::types::GraphNode {
328            id: id.to_string(),
329            node_type: "concept".to_string(),
330            title: format!("Node {id}"),
331            body: String::new(),
332            tags: tags.into_iter().map(|s| s.to_string()).collect(),
333            projects: vec![],
334            agents: vec![],
335            created: "2026-01-01T00:00:00Z".to_string(),
336            updated: "2026-06-01T00:00:00Z".to_string(),
337            importance,
338            access_count: 0,
339            accessed_at: String::new(),
340            ..Default::default()
341        }
342    }
343
344    #[test]
345    fn recall_returns_nodes() {
346        let conn = mem_db();
347        upsert_node(&conn, &test_node("n1", 0.9, vec![])).unwrap();
348        upsert_node(&conn, &test_node("n2", 0.5, vec![])).unwrap();
349        let results = smart_recall(&conn, None, None, 10).unwrap();
350        assert_eq!(results.len(), 2);
351        // Higher importance first
352        assert_eq!(results[0].node.id, "n1");
353    }
354
355    #[test]
356    fn recall_filters_by_project() {
357        let conn = mem_db();
358        let mut n1 = test_node("n1", 0.7, vec![]);
359        n1.projects = vec!["myproj".to_string()];
360        upsert_node(&conn, &n1).unwrap();
361        upsert_node(&conn, &test_node("n2", 0.7, vec![])).unwrap();
362
363        let results = smart_recall(&conn, Some("myproj"), None, 10).unwrap();
364        assert_eq!(results.len(), 1);
365        assert_eq!(results[0].node.id, "n1");
366    }
367
368    #[test]
369    fn recall_with_hint_uses_fts() {
370        let conn = mem_db();
371        let mut n1 = test_node("n1", 0.5, vec![]);
372        n1.title = "Rust ownership model".to_string();
373        n1.body = "borrow checker rules".to_string();
374        upsert_node(&conn, &n1).unwrap();
375
376        let mut n2 = test_node("n2", 0.9, vec![]);
377        n2.title = "Python GIL".to_string();
378        upsert_node(&conn, &n2).unwrap();
379
380        let results = smart_recall(&conn, None, Some("Rust"), 10).unwrap();
381        // n1 should get FTS boost even though n2 has higher base importance
382        assert!(!results.is_empty());
383    }
384
385    #[test]
386    fn recall_excludes_stale() {
387        let conn = mem_db();
388        upsert_node(&conn, &test_node("n1", 0.9, vec!["stale"])).unwrap();
389        upsert_node(&conn, &test_node("n2", 0.5, vec![])).unwrap();
390        let results = smart_recall(&conn, None, None, 10).unwrap();
391        assert_eq!(results.len(), 1);
392        assert_eq!(results[0].node.id, "n2");
393    }
394
395    #[test]
396    fn recall_touches_access_count() {
397        let conn = mem_db();
398        upsert_node(&conn, &test_node("n1", 0.7, vec![])).unwrap();
399        smart_recall(&conn, None, None, 10).unwrap();
400        let node = crate::graph::store::read_node(&conn, "n1")
401            .unwrap()
402            .unwrap();
403        assert_eq!(node.access_count, 1);
404    }
405
406    #[test]
407    fn recall_graph_boost() {
408        let conn = mem_db();
409        upsert_node(&conn, &test_node("n1", 0.7, vec![])).unwrap();
410        upsert_node(&conn, &test_node("n2", 0.7, vec![])).unwrap();
411        append_edge(
412            &conn,
413            &GraphEdge {
414                id: "e1".into(),
415                source: "n1".into(),
416                target: "n2".into(),
417                relation: "related".into(),
418                weight: 1.0,
419                ts: "2026-01-01T00:00:00Z".into(),
420            },
421        )
422        .unwrap();
423
424        let results = smart_recall(&conn, None, None, 10).unwrap();
425        assert_eq!(results.len(), 2);
426        // With hint=None and identical recency/importance/access, every score
427        // component is equal across n1 and n2 EXCEPT the graph boost. n2 is a
428        // dangling sink (no out-edges) and so accrues higher PageRank than n1
429        // — its sole in-bound rank source — so the boost pass must rank n2
430        // above n1. This is the one assertion that exercises the boost's
431        // actual ranking effect (the pagerank math itself is unit-tested in
432        // algo/pagerank.rs).
433        let n1 = results.iter().find(|s| s.node.id == "n1").unwrap();
434        let n2 = results.iter().find(|s| s.node.id == "n2").unwrap();
435        assert!(
436            n2.score > n1.score,
437            "dangling sink n2 must outrank source n1 via PageRank boost"
438        );
439    }
440
441    #[test]
442    fn recall_project_wildcard_is_escaped() {
443        let conn = mem_db();
444        let mut n1 = test_node("n1", 0.7, vec![]);
445        n1.projects = vec!["myproj".to_string()];
446        upsert_node(&conn, &n1).unwrap();
447        // "my%" would match "myproj" as a LIKE wildcard, but escape_like prevents it
448        let results = smart_recall(&conn, Some("my%"), None, 10).unwrap();
449        assert!(results.is_empty());
450    }
451
452    #[test]
453    fn recall_fts_recovery_respects_tag_scope() {
454        // Regression for the scope-bypass blocker: an FTS hint that also matches
455        // an out-of-scope node must NOT pull that node in via the recovery path.
456        //
457        // Setup: n1 is symbol-scoped (tag "AAPL"), low importance; n2 mentions
458        // "AAPL" in its body but is NOT tagged "AAPL" and has high importance.
459        // With `tags_any = ["AAPL"]` + hint "AAPL", the FTS window recovers both
460        // ids, but only n1 should survive — n2 fails the tag scope filter.
461        let conn = mem_db();
462        let mut n1 = test_node("n1", 0.1, vec!["AAPL"]);
463        n1.title = "AAPL position".to_string();
464        n1.body = "earnings call notes".to_string();
465        upsert_node(&conn, &n1).unwrap();
466
467        let mut n2 = test_node("n2", 0.99, vec![]);
468        n2.title = "Market commentary".to_string();
469        n2.body = "AAPL mentioned in passing".to_string();
470        upsert_node(&conn, &n2).unwrap();
471
472        let opts = RecallOptions {
473            hint: Some("AAPL".to_string()),
474            tags_any: vec!["AAPL".to_string()],
475            limit: 10,
476            ..Default::default()
477        };
478        let results = smart_recall_with(&conn, &opts).unwrap();
479        let ids: Vec<&str> = results.iter().map(|s| s.node.id.as_str()).collect();
480        assert!(
481            ids.iter().all(|id| *id != "n2"),
482            "out-of-scope n2 must not leak via FTS recovery; got {ids:?}"
483        );
484        assert!(
485            ids.contains(&"n1"),
486            "in-scope n1 must be present; got {ids:?}"
487        );
488    }
489
490    #[test]
491    fn recall_fts_recovery_respects_project_scope() {
492        // Same bypass, project dimension: a recovered node in the wrong project
493        // is dropped.
494        let conn = mem_db();
495        let mut n1 = test_node("n1", 0.1, vec![]);
496        n1.title = "AAPL note".to_string();
497        n1.projects = vec!["projA".to_string()];
498        upsert_node(&conn, &n1).unwrap();
499
500        let mut n2 = test_node("n2", 0.99, vec![]);
501        n2.title = "AAPL cross-ref".to_string();
502        n2.projects = vec!["projB".to_string()];
503        upsert_node(&conn, &n2).unwrap();
504
505        let opts = RecallOptions {
506            project: Some("projA".to_string()),
507            hint: Some("AAPL".to_string()),
508            limit: 10,
509            ..Default::default()
510        };
511        let results = smart_recall_with(&conn, &opts).unwrap();
512        let ids: Vec<&str> = results.iter().map(|s| s.node.id.as_str()).collect();
513        assert!(
514            ids.iter().all(|id| *id != "n2"),
515            "wrong-project n2 must not leak via FTS recovery; got {ids:?}"
516        );
517    }
518}
llm_kernel/graph/recall.rs

llm_kernel/graph/
recall.rs
