sqlite_graphrag/storage/memories/
fts.rs

1//! FTS5 shadow table: query sanitisation, BM25 search and index sync.
2//!
3//! Owns every statement touching `fts_memories`, plus the sanitiser that keeps
4//! raw operator input from reaching the FTS5 query parser.
5
6use super::rows::MemoryRow;
7use crate::errors::AppError;
8use rusqlite::{params, Connection};
9
10/// Preprocesses a raw user query for FTS5 `MATCH`.
11///
12/// Technical separators (`-`, `.`, `_`, `/`) are treated as word boundaries by
13/// the `unicode61` tokenizer.  When the query contains any of these characters
14/// the function builds a compound FTS5 expression:
15///   1. A phrase query with the separated tokens (exact compound matching).
16///   2. Individual prefix terms joined with OR (broader recall).
17///
18/// Queries without separators keep the original `term*` prefix behaviour.
19pub(super) fn preprocess_fts_query(raw: &str) -> String {
20    const SEPARATORS: &[char] = &['-', '.', '_', '/'];
21    const FTS5_SYNTAX: &[char] = &['"', '*', '(', ')', '^', ':'];
22    const FTS5_KEYWORDS: &[&str] = &["OR", "AND", "NOT", "NEAR"];
23
24    let sanitized: String = raw.chars().filter(|c| !FTS5_SYNTAX.contains(c)).collect();
25    let trimmed = sanitized.trim();
26    if trimmed.is_empty() {
27        return String::new();
28    }
29
30    let is_fts_keyword = |t: &str| FTS5_KEYWORDS.iter().any(|kw| kw.eq_ignore_ascii_case(t));
31
32    if !trimmed.chars().any(|c| SEPARATORS.contains(&c)) {
33        return trimmed
34            .split_whitespace()
35            .filter(|t| !is_fts_keyword(t))
36            .map(|t| format!("{t}*"))
37            .collect::<Vec<_>>()
38            .join(" ");
39    }
40    let tokens: Vec<&str> = trimmed
41        .split(|c: char| SEPARATORS.contains(&c) || c.is_whitespace())
42        .filter(|t| !t.is_empty() && !is_fts_keyword(t))
43        .collect();
44    if tokens.is_empty() {
45        return String::new();
46    }
47    let phrase = format!("\"{}\"", tokens.join(" "));
48    let prefix_terms: Vec<String> = tokens.iter().map(|t| format!("{t}*")).collect();
49    format!("{phrase} OR {}", prefix_terms.join(" OR "))
50}
51
52/// Executes an FTS5 search against `fts_memories` with query preprocessing.
53///
54/// Technical separators in the query are converted to phrase + prefix OR
55/// expressions so compound terms like `graphrag-precompact.sh` match correctly.
56///
57/// # Errors
58///
59/// Returns `Err(AppError::Database)` on any `rusqlite` failure.
60pub fn fts_search(
61    conn: &Connection,
62    query: &str,
63    namespace: &str,
64    memory_type: Option<&str>,
65    limit: usize,
66) -> Result<Vec<MemoryRow>, AppError> {
67    let fts_query = preprocess_fts_query(query);
68    if let Some(mt) = memory_type {
69        let mut stmt = conn.prepare_cached(
70            "SELECT m.id, m.namespace, m.name, m.type, m.description, m.body, m.body_hash,
71                    m.session_id, m.source, m.metadata, m.created_at, m.updated_at, m.deleted_at
72             FROM fts_memories fts
73             JOIN memories m ON m.id = fts.rowid
74             WHERE fts_memories MATCH ?1 AND m.namespace = ?2 AND m.type = ?3 AND m.deleted_at IS NULL
75             ORDER BY rank LIMIT ?4",
76        )?;
77        let rows = stmt
78            .query_map(params![fts_query, namespace, mt, limit as i64], |r| {
79                Ok(MemoryRow {
80                    id: r.get(0)?,
81                    namespace: r.get(1)?,
82                    name: r.get(2)?,
83                    memory_type: r.get(3)?,
84                    description: r.get(4)?,
85                    body: r.get(5)?,
86                    body_hash: r.get(6)?,
87                    session_id: r.get(7)?,
88                    source: r.get(8)?,
89                    metadata: r.get(9)?,
90                    created_at: r.get(10)?,
91                    updated_at: r.get(11)?,
92                    deleted_at: r.get(12)?,
93                })
94            })?
95            .collect::<Result<Vec<_>, _>>()?;
96        Ok(rows)
97    } else {
98        let mut stmt = conn.prepare_cached(
99            "SELECT m.id, m.namespace, m.name, m.type, m.description, m.body, m.body_hash,
100                    m.session_id, m.source, m.metadata, m.created_at, m.updated_at, m.deleted_at
101             FROM fts_memories fts
102             JOIN memories m ON m.id = fts.rowid
103             WHERE fts_memories MATCH ?1 AND m.namespace = ?2 AND m.deleted_at IS NULL
104             ORDER BY rank LIMIT ?3",
105        )?;
106        let rows = stmt
107            .query_map(params![fts_query, namespace, limit as i64], |r| {
108                Ok(MemoryRow {
109                    id: r.get(0)?,
110                    namespace: r.get(1)?,
111                    name: r.get(2)?,
112                    memory_type: r.get(3)?,
113                    description: r.get(4)?,
114                    body: r.get(5)?,
115                    body_hash: r.get(6)?,
116                    session_id: r.get(7)?,
117                    source: r.get(8)?,
118                    metadata: r.get(9)?,
119                    created_at: r.get(10)?,
120                    updated_at: r.get(11)?,
121                    deleted_at: r.get(12)?,
122                })
123            })?
124            .collect::<Result<Vec<_>, _>>()?;
125        Ok(rows)
126    }
127}
128
129/// Syncs FTS5 external-content index after an UPDATE on the memories table.
130///
131/// The AFTER UPDATE trigger (`trg_fts_au`) is intentionally absent because
132/// sqlite-vec loaded via `sqlite3_auto_extension` conflicts with FTS5 inside
133/// UPDATE triggers. This function performs the equivalent sync in Rust:
134/// DELETE the old entry, then INSERT the new one (external-content FTS5
135/// tables do not support in-place UPDATE).
136// The three FTS columns twice, before and after: the arity IS the delete/insert
137// pair the external-content index requires, and both triples are already spelled
138// out at every call site from two different sources.
139// The old and new triples are what an FTS5 external-content delete/insert pair
140// requires, and they must stay POSITIONALLY distinct so a caller cannot pass the
141// new text as the old: naming them in one struct would invite exactly that.
142#[allow(clippy::too_many_arguments)]
143pub fn sync_fts_after_update(
144    conn: &Connection,
145    memory_id: i64,
146    old_name: &str,
147    old_desc: &str,
148    old_body: &str,
149    new_name: &str,
150    new_desc: &str,
151    new_body: &str,
152) -> Result<(), AppError> {
153    conn.execute(
154        "INSERT INTO fts_memories(fts_memories, rowid, name, description, body)
155         VALUES('delete', ?1, ?2, ?3, ?4)",
156        params![memory_id, old_name, old_desc, old_body],
157    )?;
158    conn.execute(
159        "INSERT INTO fts_memories(rowid, name, description, body)
160         VALUES(?1, ?2, ?3, ?4)",
161        params![memory_id, new_name, new_desc, new_body],
162    )?;
163    Ok(())
164}
sqlite_graphrag/storage/memories/fts.rs

sqlite_graphrag/storage/memories/
fts.rs
