remem/retrieval/memory_search/
fts.rs

1use anyhow::Result;
2use rusqlite::Connection;
3
4use crate::db;
5use crate::memory::{map_memory_row_pub, Memory};
6use crate::retrieval::memory_search::filters::{push_branch_filter, push_project_filter};
7
8#[derive(Debug, Clone)]
9pub struct FtsMemoryHit {
10    pub memory: Memory,
11    pub score: f64,
12}
13
14/// FTS5 trigram search on memories.
15pub fn search_memories_fts(
16    conn: &Connection,
17    query: &str,
18    project: Option<&str>,
19    memory_type: Option<&str>,
20    limit: i64,
21    offset: i64,
22) -> Result<Vec<Memory>> {
23    search_memories_fts_filtered(
24        conn,
25        query,
26        project,
27        memory_type,
28        limit,
29        offset,
30        false,
31        None,
32    )
33}
34
35pub fn search_memories_fts_filtered(
36    conn: &Connection,
37    query: &str,
38    project: Option<&str>,
39    memory_type: Option<&str>,
40    limit: i64,
41    offset: i64,
42    include_inactive: bool,
43    branch: Option<&str>,
44) -> Result<Vec<Memory>> {
45    Ok(search_memories_fts_hits_filtered(
46        conn,
47        query,
48        project,
49        memory_type,
50        limit,
51        offset,
52        include_inactive,
53        branch,
54    )?
55    .into_iter()
56    .map(|hit| hit.memory)
57    .collect())
58}
59
60pub fn search_memories_fts_hits_filtered(
61    conn: &Connection,
62    query: &str,
63    project: Option<&str>,
64    memory_type: Option<&str>,
65    limit: i64,
66    offset: i64,
67    include_inactive: bool,
68    branch: Option<&str>,
69) -> Result<Vec<FtsMemoryHit>> {
70    let mut conditions = vec!["memories_fts MATCH ?1".to_string()];
71    let mut param_values: Vec<Box<dyn rusqlite::types::ToSql>> = vec![Box::new(query.to_string())];
72
73    let mut idx = 2;
74    conditions.push(crate::memory::memory_current_filter_sql(
75        "m.status",
76        "m.expires_at_epoch",
77        include_inactive,
78    ));
79
80    idx = push_project_filter(
81        "m.project",
82        project,
83        idx,
84        &mut conditions,
85        &mut param_values,
86    );
87    idx = push_branch_filter("m.branch", branch, idx, &mut conditions, &mut param_values);
88    if let Some(memory_type) = memory_type {
89        conditions.push(format!("m.memory_type = ?{idx}"));
90        param_values.push(Box::new(memory_type.to_string()));
91        idx += 1;
92    }
93
94    param_values.push(Box::new(limit));
95    param_values.push(Box::new(offset));
96
97    let sql = format!(
98        "WITH ranked AS (
99             SELECT m.id, m.session_id, m.project, m.topic_key, m.title, m.content,
100                    m.memory_type, m.files, m.created_at_epoch, m.updated_at_epoch,
101                    m.status, m.branch, m.scope,
102                    (bm25(memories_fts, 10.0, 1.0, 3.0) * CASE WHEN m.memory_type IN ('decision','bugfix') THEN 1.5 ELSE 1.0 END) AS rank_score
103             FROM memories m
104             JOIN memories_fts ON memories_fts.rowid = m.id
105             WHERE {}
106         )
107         SELECT id, session_id, project, topic_key, title, content,
108                memory_type, files, created_at_epoch, updated_at_epoch,
109                status, branch, scope, rank_score
110         FROM ranked
111         ORDER BY rank_score
112         LIMIT ?{} OFFSET ?{}",
113        conditions.join(" AND "),
114        idx,
115        idx + 1
116    );
117
118    let mut stmt = conn.prepare(&sql)?;
119    let refs = db::to_sql_refs(&param_values);
120    let rows = stmt.query_map(refs.as_slice(), |row| {
121        Ok(FtsMemoryHit {
122            memory: map_memory_row_pub(row)?,
123            score: row.get(13)?,
124        })
125    })?;
126    crate::db::query::collect_rows(rows)
127}
128
129#[cfg(test)]
130mod tests {
131    use super::*;
132
133    fn setup_conn() -> Connection {
134        let conn = Connection::open_in_memory().unwrap();
135        crate::migrate::run_migrations(&conn).unwrap();
136        conn
137    }
138
139    fn insert_memory(conn: &Connection, id: i64, body: &str, status: &str) {
140        conn.execute(
141            "INSERT INTO memories(id, project, title, content, memory_type, created_at_epoch,
142                updated_at_epoch, status)
143             VALUES (?1, 'proj', 'title', ?2, 'decision', 100, 100, ?3)",
144            rusqlite::params![id, body, status],
145        )
146        .unwrap();
147    }
148
149    /// Reproduction for #236: before v019 a stale row never entered memories_fts,
150    /// so the JOIN dropped it and include_inactive bm25 search returned empty.
151    #[test]
152    fn include_inactive_finds_stale_rows() {
153        let conn = setup_conn();
154        insert_memory(&conn, 1, "deprecated zookeeper approach", "stale");
155
156        // active-only search must hide the stale row
157        let active_only = search_memories_fts_filtered(
158            &conn,
159            "zookeeper",
160            Some("proj"),
161            None,
162            10,
163            0,
164            false,
165            None,
166        )
167        .unwrap();
168        assert!(
169            active_only.is_empty(),
170            "active-only search must hide stale rows: {active_only:?}"
171        );
172
173        // include_inactive must surface the stale row via the bm25 path
174        let with_inactive =
175            search_memories_fts_filtered(&conn, "zookeeper", Some("proj"), None, 10, 0, true, None)
176                .unwrap();
177        assert_eq!(
178            with_inactive.len(),
179            1,
180            "include_inactive must retrieve stale"
181        );
182        assert_eq!(with_inactive[0].status, "stale");
183    }
184
185    /// Active rows must remain retrievable on the default (active-only) path.
186    #[test]
187    fn active_path_still_finds_active_rows() {
188        let conn = setup_conn();
189        insert_memory(&conn, 1, "current kafka pipeline", "active");
190
191        let hits =
192            search_memories_fts_filtered(&conn, "kafka", Some("proj"), None, 10, 0, false, None)
193                .unwrap();
194        assert_eq!(hits.len(), 1);
195        assert_eq!(hits[0].status, "active");
196    }
197}
remem/retrieval/memory_search/fts.rs

remem/retrieval/memory_search/
fts.rs
