sqlite_graphrag/storage/memories/
mod.rs

1//! Persistence layer for the `memories` table and its vector companion.
2//!
3//! Functions here encapsulate every SQL statement touching `memories`,
4//! `memory_embeddings` and the FTS5 `fts_memories` shadow table. Callers receive
5//! typed [`MemoryRow`] or [`NewMemory`] values and never build SQL strings.
6
7mod soft_delete;
8pub use soft_delete::{clear_deleted_at, find_by_name_any_state, list_deleted_before, soft_delete};
9
10use crate::embedder::f32_to_bytes;
11use crate::errors::AppError;
12use crate::storage::utils::with_busy_retry;
13use rusqlite::{params, Connection};
14use serde::{Deserialize, Serialize};
15
16/// Input payload for inserting or updating a memory.
17///
18/// `body_hash` must be the BLAKE3 digest of `body`. The `metadata` field is
19/// stored as a TEXT column containing JSON.
20#[derive(Debug, Serialize, Deserialize)]
21pub struct NewMemory {
22    /// Namespace scope.
23    pub namespace: String,
24    /// Name of this item.
25    pub name: String,
26    /// Memory type classification.
27    pub memory_type: String,
28    /// Human-readable description.
29    pub description: String,
30    /// Full text body.
31    pub body: String,
32    /// Body hash.
33    pub body_hash: String,
34    /// Session ID.
35    pub session_id: Option<String>,
36    /// Source side of the relationship.
37    pub source: String,
38    /// Arbitrary metadata.
39    pub metadata: serde_json::Value,
40}
41
42/// Fully materialized row from the `memories` table.
43///
44/// Returned by [`read_by_name`], [`read_full`], [`list`] and [`fts_search`].
45/// The `metadata` field is kept as a JSON string to avoid double parsing.
46#[derive(Debug, Serialize)]
47pub struct MemoryRow {
48    /// Unique identifier.
49    pub id: i64,
50    /// Namespace scope.
51    pub namespace: String,
52    /// Name of this item.
53    pub name: String,
54    /// Memory type classification.
55    pub memory_type: String,
56    /// Human-readable description.
57    pub description: String,
58    /// Full text body.
59    pub body: String,
60    /// Body hash.
61    pub body_hash: String,
62    /// Session ID.
63    pub session_id: Option<String>,
64    /// Source side of the relationship.
65    pub source: String,
66    /// Arbitrary metadata.
67    pub metadata: String,
68    /// Creation timestamp.
69    pub created_at: i64,
70    /// Last-update timestamp.
71    pub updated_at: i64,
72    /// Unix epoch when the memory was soft-deleted, or `None` for active memories.
73    /// Surfaced in `list --include-deleted --json` so LLM consumers can distinguish
74    /// active from soft-deleted rows without a second SQL query (v1.0.37 H7+M9 fix).
75    #[serde(skip_serializing_if = "Option::is_none")]
76    pub deleted_at: Option<i64>,
77}
78
79/// Finds a live memory by `(namespace, name)` and returns key metadata.
80///
81/// # Arguments
82///
83/// - `conn` — open SQLite connection configured with the project pragmas.
84/// - `namespace` — resolved namespace for the lookup.
85/// - `name` — kebab-case memory name.
86///
87/// # Returns
88///
89/// `Ok(Some((id, updated_at, max_version)))` when the memory exists and is
90/// not soft-deleted, `Ok(None)` otherwise.
91///
92/// # Errors
93///
94/// Returns `Err(AppError::Database)` on any `rusqlite` failure.
95pub fn find_by_name(
96    conn: &Connection,
97    namespace: &str,
98    name: &str,
99) -> Result<Option<(i64, i64, i64)>, AppError> {
100    let mut stmt = conn.prepare_cached(
101        "SELECT m.id, m.updated_at, COALESCE(MAX(v.version), 0)
102         FROM memories m
103         LEFT JOIN memory_versions v ON v.memory_id = m.id
104         WHERE m.namespace = ?1 AND m.name = ?2 AND m.deleted_at IS NULL
105         GROUP BY m.id",
106    )?;
107    let result = stmt.query_row(params![namespace, name], |r| {
108        Ok((
109            r.get::<_, i64>(0)?,
110            r.get::<_, i64>(1)?,
111            r.get::<_, i64>(2)?,
112        ))
113    });
114    match result {
115        Ok(row) => Ok(Some(row)),
116        Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
117        Err(e) => Err(AppError::Database(e)),
118    }
119}
120
121/// Looks up a live memory by exact `body_hash` within a namespace.
122///
123/// Used during `remember` to short-circuit semantic duplicates before
124/// spending an embedding call.
125///
126/// # Returns
127///
128/// `Ok(Some(id))` when a live memory with the same hash exists,
129/// `Ok(None)` otherwise.
130///
131/// # Errors
132///
133/// Returns `Err(AppError::Database)` on any `rusqlite` failure.
134pub fn find_by_hash(
135    conn: &Connection,
136    namespace: &str,
137    body_hash: &str,
138) -> Result<Option<i64>, AppError> {
139    let mut stmt = conn.prepare_cached(
140        "SELECT id FROM memories WHERE namespace = ?1 AND body_hash = ?2 AND deleted_at IS NULL",
141    )?;
142    match stmt.query_row(params![namespace, body_hash], |r| r.get(0)) {
143        Ok(id) => Ok(Some(id)),
144        Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
145        Err(e) => Err(AppError::Database(e)),
146    }
147}
148
149/// Inserts a new row into the `memories` table.
150///
151/// # Arguments
152///
153/// - `conn` — active SQLite connection, typically inside a transaction.
154/// - `m` — validated payload including `body_hash` and serialized metadata.
155///
156/// # Returns
157///
158/// The `rowid` assigned to the newly inserted memory.
159///
160/// # Errors
161///
162/// Returns `Err(AppError::Database)` on insertion failure and
163/// `Err(AppError::Json)` if metadata serialization fails.
164pub fn insert(conn: &Connection, m: &NewMemory) -> Result<i64, AppError> {
165    // G29 Passo 2 (v1.0.69): runtime guard for the CHECK constraint on
166    // `source`. Even though `MemorySource` is the typed future, every
167    // legacy `NewMemory { source: "..." }` literal still flows through
168    // this function; validating here keeps the footgun from regressing
169    // for callers that have not yet migrated to the enum.
170    let validated_source = crate::memory_source::validate_source(&m.source)?;
171    conn.execute(
172        "INSERT INTO memories (namespace, name, type, description, body, body_hash, session_id, source, metadata)
173         VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
174        params![
175            m.namespace, m.name, m.memory_type, m.description, m.body,
176            m.body_hash, m.session_id, validated_source,
177            serde_json::to_string(&m.metadata)?
178        ],
179    )?;
180    Ok(conn.last_insert_rowid())
181}
182
183/// Updates an existing memory optionally guarded by optimistic concurrency.
184///
185/// When `expected_updated_at` is `Some(ts)` the row is only updated if its
186/// current `updated_at` equals `ts`. This protects concurrent `edit` calls
187/// from silently clobbering each other.
188///
189/// # Returns
190///
191/// `Ok(true)` when exactly one row was updated, `Ok(false)` when the
192/// optimistic check failed or the memory does not exist.
193///
194/// # Errors
195///
196/// Returns `Err(AppError::Database)` on any `rusqlite` failure.
197pub fn update(
198    conn: &Connection,
199    id: i64,
200    m: &NewMemory,
201    expected_updated_at: Option<i64>,
202) -> Result<bool, AppError> {
203    // G29 Passo 2 (v1.0.69): runtime guard for the CHECK constraint on
204    // `source`. Mirrors `insert` so `body-enrich` and other mutations
205    // cannot reintroduce the historical "enrich" literal that broke
206    // `body-enrich` in v1.0.55 - v1.0.68.
207    let validated_source = crate::memory_source::validate_source(&m.source)?;
208    let affected = if let Some(ts) = expected_updated_at {
209        conn.execute(
210            "UPDATE memories SET type=?2, description=?3, body=?4, body_hash=?5,
211             session_id=?6, source=?7, metadata=?8
212             WHERE id=?1 AND updated_at=?9 AND deleted_at IS NULL",
213            params![
214                id,
215                m.memory_type,
216                m.description,
217                m.body,
218                m.body_hash,
219                m.session_id,
220                validated_source,
221                serde_json::to_string(&m.metadata)?,
222                ts
223            ],
224        )?
225    } else {
226        conn.execute(
227            "UPDATE memories SET type=?2, description=?3, body=?4, body_hash=?5,
228             session_id=?6, source=?7, metadata=?8
229             WHERE id=?1 AND deleted_at IS NULL",
230            params![
231                id,
232                m.memory_type,
233                m.description,
234                m.body,
235                m.body_hash,
236                m.session_id,
237                validated_source,
238                serde_json::to_string(&m.metadata)?
239            ],
240        )?
241    };
242    Ok(affected == 1)
243}
244
245/// Replaces the vector row for a memory in `memory_embeddings`.
246///
247/// v1.0.76: sqlite-vec was removed. Embeddings live in a regular BLOB-backed
248/// table; cosine similarity is computed in pure Rust on demand. The
249/// `memory_type`, `name`, and `snippet` arguments are accepted for API
250/// compatibility but are not stored — the FTS5 shadow table is the
251/// source of truth for textual metadata.
252///
253/// # Errors
254///
255/// Returns `Err(AppError::Database)` on any `rusqlite` failure.
256pub fn upsert_vec(
257    conn: &Connection,
258    memory_id: i64,
259    namespace: &str,
260    _memory_type: &str,
261    embedding: &[f32],
262    _name: &str,
263    _snippet: &str,
264) -> Result<(), AppError> {
265    // v1.1.1 (P1): skip empty vectors so the memory stays visible to the
266    // re-embed backfill scanner instead of persisting a vector-less row.
267    if embedding.is_empty() {
268        tracing::debug!(
269            memory_id,
270            "empty memory embedding: skipping memory_embeddings row (backfill via enrich re-embed)"
271        );
272        return Ok(());
273    }
274    let embedding_bytes = f32_to_bytes(embedding);
275    with_busy_retry(|| {
276        conn.execute(
277            "DELETE FROM memory_embeddings WHERE memory_id = ?1",
278            params![memory_id],
279        )?;
280        conn.execute(
281            "INSERT INTO memory_embeddings(memory_id, namespace, embedding, source, model, dim)
282             VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
283            params![
284                memory_id,
285                namespace,
286                &embedding_bytes,
287                "llm-headless",
288                crate::constants::SQLITE_GRAPHRAG_VERSION,
289                crate::constants::embedding_dim() as i64,
290            ],
291        )?;
292        Ok(())
293    })
294}
295
296/// Deletes the vector row for `memory_id` from `memory_embeddings`.
297///
298/// Called during `forget` and `purge` to keep the embeddings table
299/// consistent with the logical state of `memories`. FK CASCADE on
300/// `memory_embeddings.memory_id` handles the common case, but this
301/// function exists so callers can delete the embedding first
302/// (preserving the row in `memories` for audit).
303///
304/// # Errors
305///
306/// Returns `Err(AppError::Database)` on any `rusqlite` failure.
307pub fn delete_vec(conn: &Connection, memory_id: i64) -> Result<(), AppError> {
308    conn.execute(
309        "DELETE FROM memory_embeddings WHERE memory_id = ?1",
310        params![memory_id],
311    )?;
312    Ok(())
313}
314
315/// Fetches a live memory by `(namespace, name)` and returns all columns.
316///
317/// # Returns
318///
319/// `Ok(Some(row))` when found, `Ok(None)` when missing or soft-deleted.
320///
321/// # Errors
322///
323/// Returns `Err(AppError::Database)` on any `rusqlite` failure.
324pub fn read_by_name(
325    conn: &Connection,
326    namespace: &str,
327    name: &str,
328) -> Result<Option<MemoryRow>, AppError> {
329    let mut stmt = conn.prepare_cached(
330        "SELECT id, namespace, name, type, description, body, body_hash,
331                session_id, source, metadata, created_at, updated_at, deleted_at
332         FROM memories WHERE namespace=?1 AND name=?2 AND deleted_at IS NULL",
333    )?;
334    match stmt.query_row(params![namespace, name], |r| {
335        Ok(MemoryRow {
336            id: r.get(0)?,
337            namespace: r.get(1)?,
338            name: r.get(2)?,
339            memory_type: r.get(3)?,
340            description: r.get(4)?,
341            body: r.get(5)?,
342            body_hash: r.get(6)?,
343            session_id: r.get(7)?,
344            source: r.get(8)?,
345            metadata: r.get(9)?,
346            created_at: r.get(10)?,
347            updated_at: r.get(11)?,
348            deleted_at: r.get(12)?,
349        })
350    }) {
351        Ok(m) => Ok(Some(m)),
352        Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
353        Err(e) => Err(AppError::Database(e)),
354    }
355}
356
357/// Lists live memories in a namespace ordered by `updated_at` descending.
358///
359/// # Arguments
360///
361/// - `memory_type` — optional filter on the `type` column.
362/// - `limit` / `offset` — standard pagination controls in rows.
363///
364/// # Errors
365///
366/// Returns `Err(AppError::Database)` on any `rusqlite` failure.
367pub fn list(
368    conn: &Connection,
369    namespace: &str,
370    memory_type: Option<&str>,
371    limit: usize,
372    offset: usize,
373    include_deleted: bool,
374) -> Result<Vec<MemoryRow>, AppError> {
375    if let Some(mt) = memory_type {
376        let sql = if include_deleted {
377            "SELECT id, namespace, name, type, description, body, body_hash,
378                    session_id, source, metadata, created_at, updated_at, deleted_at
379             FROM memories WHERE namespace=?1 AND type=?2
380             ORDER BY updated_at DESC LIMIT ?3 OFFSET ?4"
381        } else {
382            "SELECT id, namespace, name, type, description, body, body_hash,
383                    session_id, source, metadata, created_at, updated_at, deleted_at
384             FROM memories WHERE namespace=?1 AND type=?2 AND deleted_at IS NULL
385             ORDER BY updated_at DESC LIMIT ?3 OFFSET ?4"
386        };
387        let mut stmt = conn.prepare_cached(sql)?;
388        let rows = stmt
389            .query_map(params![namespace, mt, limit as i64, offset as i64], |r| {
390                Ok(MemoryRow {
391                    id: r.get(0)?,
392                    namespace: r.get(1)?,
393                    name: r.get(2)?,
394                    memory_type: r.get(3)?,
395                    description: r.get(4)?,
396                    body: r.get(5)?,
397                    body_hash: r.get(6)?,
398                    session_id: r.get(7)?,
399                    source: r.get(8)?,
400                    metadata: r.get(9)?,
401                    created_at: r.get(10)?,
402                    updated_at: r.get(11)?,
403                    deleted_at: r.get(12)?,
404                })
405            })?
406            .collect::<Result<Vec<_>, _>>()?;
407        Ok(rows)
408    } else {
409        let sql = if include_deleted {
410            "SELECT id, namespace, name, type, description, body, body_hash,
411                    session_id, source, metadata, created_at, updated_at, deleted_at
412             FROM memories WHERE namespace=?1
413             ORDER BY updated_at DESC LIMIT ?2 OFFSET ?3"
414        } else {
415            "SELECT id, namespace, name, type, description, body, body_hash,
416                    session_id, source, metadata, created_at, updated_at, deleted_at
417             FROM memories WHERE namespace=?1 AND deleted_at IS NULL
418             ORDER BY updated_at DESC LIMIT ?2 OFFSET ?3"
419        };
420        let mut stmt = conn.prepare_cached(sql)?;
421        let rows = stmt
422            .query_map(params![namespace, limit as i64, offset as i64], |r| {
423                Ok(MemoryRow {
424                    id: r.get(0)?,
425                    namespace: r.get(1)?,
426                    name: r.get(2)?,
427                    memory_type: r.get(3)?,
428                    description: r.get(4)?,
429                    body: r.get(5)?,
430                    body_hash: r.get(6)?,
431                    session_id: r.get(7)?,
432                    source: r.get(8)?,
433                    metadata: r.get(9)?,
434                    created_at: r.get(10)?,
435                    updated_at: r.get(11)?,
436                    deleted_at: r.get(12)?,
437                })
438            })?
439            .collect::<Result<Vec<_>, _>>()?;
440        Ok(rows)
441    }
442}
443
444/// Count.
445pub fn count(
446    conn: &Connection,
447    namespace: &str,
448    memory_type: Option<&str>,
449    include_deleted: bool,
450) -> Result<usize, AppError> {
451    let (sql, params_vec): (&str, Vec<Box<dyn rusqlite::types::ToSql>>) = match (
452        memory_type,
453        include_deleted,
454    ) {
455        (Some(mt), true) => (
456            "SELECT COUNT(*) FROM memories WHERE namespace=?1 AND type=?2",
457            vec![
458                Box::new(namespace.to_string()) as Box<dyn rusqlite::types::ToSql>,
459                Box::new(mt.to_string()),
460            ],
461        ),
462        (Some(mt), false) => (
463            "SELECT COUNT(*) FROM memories WHERE namespace=?1 AND type=?2 AND deleted_at IS NULL",
464            vec![
465                Box::new(namespace.to_string()) as Box<dyn rusqlite::types::ToSql>,
466                Box::new(mt.to_string()),
467            ],
468        ),
469        (None, true) => (
470            "SELECT COUNT(*) FROM memories WHERE namespace=?1",
471            vec![Box::new(namespace.to_string()) as Box<dyn rusqlite::types::ToSql>],
472        ),
473        (None, false) => (
474            "SELECT COUNT(*) FROM memories WHERE namespace=?1 AND deleted_at IS NULL",
475            vec![Box::new(namespace.to_string()) as Box<dyn rusqlite::types::ToSql>],
476        ),
477    };
478    let params_refs: Vec<&dyn rusqlite::types::ToSql> =
479        params_vec.iter().map(|b| b.as_ref()).collect();
480    let n: i64 = conn.query_row(sql, params_refs.as_slice(), |r| r.get(0))?;
481    Ok(n as usize)
482}
483
484/// Runs a KNN search over `memory_embeddings`, optionally restricted to namespaces.
485///
486/// # Arguments
487///
488/// - `embedding` — query vector of length [`crate::constants::embedding_dim()`].
489/// - `namespaces` — namespaces to search. Empty slice means "all namespaces".
490/// - `memory_type` — optional filter on the `type` column.
491/// - `k` — maximum number of hits to return.
492///
493/// # Returns
494///
495/// A vector of `(memory_id, distance)` pairs sorted by ascending distance.
496///
497/// # Errors
498///
499/// Returns `Err(AppError::Database)` on any `rusqlite` failure.
500pub fn knn_search(
501    conn: &Connection,
502    embedding: &[f32],
503    namespaces: &[String],
504    memory_type: Option<&str>,
505    k: usize,
506) -> Result<Vec<(i64, f32)>, AppError> {
507    if embedding.len() != crate::constants::embedding_dim() {
508        return Err(AppError::Embedding(
509            crate::i18n::validation::embedding_knn_search_dim_mismatch(
510                embedding.len(),
511                crate::constants::embedding_dim(),
512            ),
513        ));
514    }
515    // v1.0.76: full table scan + in-process cosine similarity. The
516    // `memory_embeddings` table no longer has a `distance` column or a
517    // `type` column (the namespace/type filters were dropped for the
518    // BLOB-backed table — they live on the `memories` table). The
519    // cosine result is converted to a "distance" so callers that read
520    // `distance` keep working unchanged.
521
522    // Build the SQL once with the namespace IN clause shape.
523    let placeholders = (0..namespaces.len())
524        .map(|_| "?")
525        .collect::<Vec<_>>()
526        .join(",");
527    let sql = if namespaces.is_empty() {
528        "SELECT memory_id, embedding, namespace FROM memory_embeddings".to_string()
529    } else {
530        format!(
531            "SELECT memory_id, embedding, namespace FROM memory_embeddings \
532             WHERE namespace IN ({placeholders})"
533        )
534    };
535    let mut stmt = conn.prepare(&sql)?;
536    let mut raw_params: Vec<Box<dyn rusqlite::ToSql>> = Vec::new();
537    for ns in namespaces {
538        raw_params.push(Box::new(ns.clone()));
539    }
540    let param_refs: Vec<&dyn rusqlite::ToSql> = raw_params.iter().map(|b| b.as_ref()).collect();
541    let rows = stmt.query_map(param_refs.as_slice(), |r| {
542        let id: i64 = r.get(0)?;
543        let bytes: Vec<u8> = r.get(1)?;
544        let ns: String = r.get(2)?;
545        Ok((id, bytes, ns))
546    })?;
547
548    // Optionally restrict to a memory type by joining against the
549    // `memories` table on the fly.
550    let type_filter = memory_type.map(|t| t.to_string());
551    let mut candidates: Vec<(i64, f32)> = Vec::new();
552    for row in rows {
553        let (id, bytes, ns) = row?;
554        let stored = crate::embedder::bytes_to_f32(&bytes);
555        if stored.len() != embedding.len() {
556            continue;
557        }
558        let sim = crate::similarity::cosine_similarity(embedding, &stored);
559        let dist = crate::similarity::similarity_to_distance(sim);
560        if let Some(mt) = &type_filter {
561            // Look up the memory's type via a per-row check. For very
562            // large candidate sets this should be batched; for the
563            // v1.0.76 default namespace size (<10k memories) the
564            // per-row lookup is acceptable.
565            let actual: Option<String> = conn
566                .query_row(
567                    "SELECT type FROM memories WHERE id = ?1",
568                    params![id],
569                    |r| r.get(0),
570                )
571                .ok();
572            if actual.as_deref() != Some(mt.as_str()) {
573                continue;
574            }
575        }
576        let _ = ns; // namespace already filtered at SQL level
577        candidates.push((id, dist));
578    }
579    // Sort by distance ascending (best matches first).
580    candidates.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal));
581    candidates.truncate(k);
582    Ok(candidates)
583}
584
585/// Fetches a live memory by `(namespace, name)` and returns all columns.
586/// Fetches a live memory by primary key and returns all columns.
587///
588/// Mirrors [`read_by_name`] but keyed on `rowid` for use after a KNN search.
589///
590/// # Errors
591///
592/// Returns `Err(AppError::Database)` on any `rusqlite` failure.
593pub fn read_full(conn: &Connection, memory_id: i64) -> Result<Option<MemoryRow>, AppError> {
594    let mut stmt = conn.prepare_cached(
595        "SELECT id, namespace, name, type, description, body, body_hash,
596                session_id, source, metadata, created_at, updated_at, deleted_at
597         FROM memories WHERE id=?1 AND deleted_at IS NULL",
598    )?;
599    match stmt.query_row(params![memory_id], |r| {
600        Ok(MemoryRow {
601            id: r.get(0)?,
602            namespace: r.get(1)?,
603            name: r.get(2)?,
604            memory_type: r.get(3)?,
605            description: r.get(4)?,
606            body: r.get(5)?,
607            body_hash: r.get(6)?,
608            session_id: r.get(7)?,
609            source: r.get(8)?,
610            metadata: r.get(9)?,
611            created_at: r.get(10)?,
612            updated_at: r.get(11)?,
613            deleted_at: r.get(12)?,
614        })
615    }) {
616        Ok(m) => Ok(Some(m)),
617        Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
618        Err(e) => Err(AppError::Database(e)),
619    }
620}
621
622/// Preprocesses a raw user query for FTS5 `MATCH`.
623///
624/// Technical separators (`-`, `.`, `_`, `/`) are treated as word boundaries by
625/// the `unicode61` tokenizer.  When the query contains any of these characters
626/// the function builds a compound FTS5 expression:
627///   1. A phrase query with the separated tokens (exact compound matching).
628///   2. Individual prefix terms joined with OR (broader recall).
629///
630/// Queries without separators keep the original `term*` prefix behaviour.
631fn preprocess_fts_query(raw: &str) -> String {
632    const SEPARATORS: &[char] = &['-', '.', '_', '/'];
633    const FTS5_SYNTAX: &[char] = &['"', '*', '(', ')', '^', ':'];
634    const FTS5_KEYWORDS: &[&str] = &["OR", "AND", "NOT", "NEAR"];
635
636    let sanitized: String = raw.chars().filter(|c| !FTS5_SYNTAX.contains(c)).collect();
637    let trimmed = sanitized.trim();
638    if trimmed.is_empty() {
639        return String::new();
640    }
641
642    let is_fts_keyword = |t: &str| FTS5_KEYWORDS.iter().any(|kw| kw.eq_ignore_ascii_case(t));
643
644    if !trimmed.chars().any(|c| SEPARATORS.contains(&c)) {
645        return trimmed
646            .split_whitespace()
647            .filter(|t| !is_fts_keyword(t))
648            .map(|t| format!("{t}*"))
649            .collect::<Vec<_>>()
650            .join(" ");
651    }
652    let tokens: Vec<&str> = trimmed
653        .split(|c: char| SEPARATORS.contains(&c) || c.is_whitespace())
654        .filter(|t| !t.is_empty() && !is_fts_keyword(t))
655        .collect();
656    if tokens.is_empty() {
657        return String::new();
658    }
659    let phrase = format!("\"{}\"", tokens.join(" "));
660    let prefix_terms: Vec<String> = tokens.iter().map(|t| format!("{t}*")).collect();
661    format!("{phrase} OR {}", prefix_terms.join(" OR "))
662}
663
664/// Executes an FTS5 search against `fts_memories` with query preprocessing.
665///
666/// Technical separators in the query are converted to phrase + prefix OR
667/// expressions so compound terms like `graphrag-precompact.sh` match correctly.
668///
669/// # Errors
670///
671/// Returns `Err(AppError::Database)` on any `rusqlite` failure.
672pub fn fts_search(
673    conn: &Connection,
674    query: &str,
675    namespace: &str,
676    memory_type: Option<&str>,
677    limit: usize,
678) -> Result<Vec<MemoryRow>, AppError> {
679    let fts_query = preprocess_fts_query(query);
680    if let Some(mt) = memory_type {
681        let mut stmt = conn.prepare_cached(
682            "SELECT m.id, m.namespace, m.name, m.type, m.description, m.body, m.body_hash,
683                    m.session_id, m.source, m.metadata, m.created_at, m.updated_at, m.deleted_at
684             FROM fts_memories fts
685             JOIN memories m ON m.id = fts.rowid
686             WHERE fts_memories MATCH ?1 AND m.namespace = ?2 AND m.type = ?3 AND m.deleted_at IS NULL
687             ORDER BY rank LIMIT ?4",
688        )?;
689        let rows = stmt
690            .query_map(params![fts_query, namespace, mt, limit as i64], |r| {
691                Ok(MemoryRow {
692                    id: r.get(0)?,
693                    namespace: r.get(1)?,
694                    name: r.get(2)?,
695                    memory_type: r.get(3)?,
696                    description: r.get(4)?,
697                    body: r.get(5)?,
698                    body_hash: r.get(6)?,
699                    session_id: r.get(7)?,
700                    source: r.get(8)?,
701                    metadata: r.get(9)?,
702                    created_at: r.get(10)?,
703                    updated_at: r.get(11)?,
704                    deleted_at: r.get(12)?,
705                })
706            })?
707            .collect::<Result<Vec<_>, _>>()?;
708        Ok(rows)
709    } else {
710        let mut stmt = conn.prepare_cached(
711            "SELECT m.id, m.namespace, m.name, m.type, m.description, m.body, m.body_hash,
712                    m.session_id, m.source, m.metadata, m.created_at, m.updated_at, m.deleted_at
713             FROM fts_memories fts
714             JOIN memories m ON m.id = fts.rowid
715             WHERE fts_memories MATCH ?1 AND m.namespace = ?2 AND m.deleted_at IS NULL
716             ORDER BY rank LIMIT ?3",
717        )?;
718        let rows = stmt
719            .query_map(params![fts_query, namespace, limit as i64], |r| {
720                Ok(MemoryRow {
721                    id: r.get(0)?,
722                    namespace: r.get(1)?,
723                    name: r.get(2)?,
724                    memory_type: r.get(3)?,
725                    description: r.get(4)?,
726                    body: r.get(5)?,
727                    body_hash: r.get(6)?,
728                    session_id: r.get(7)?,
729                    source: r.get(8)?,
730                    metadata: r.get(9)?,
731                    created_at: r.get(10)?,
732                    updated_at: r.get(11)?,
733                    deleted_at: r.get(12)?,
734                })
735            })?
736            .collect::<Result<Vec<_>, _>>()?;
737        Ok(rows)
738    }
739}
740
741/// Syncs FTS5 external-content index after an UPDATE on the memories table.
742///
743/// The AFTER UPDATE trigger (`trg_fts_au`) is intentionally absent because
744/// sqlite-vec loaded via `sqlite3_auto_extension` conflicts with FTS5 inside
745/// UPDATE triggers. This function performs the equivalent sync in Rust:
746/// DELETE the old entry, then INSERT the new one (external-content FTS5
747/// tables do not support in-place UPDATE).
748#[allow(clippy::too_many_arguments)]
749pub fn sync_fts_after_update(
750    conn: &Connection,
751    memory_id: i64,
752    old_name: &str,
753    old_desc: &str,
754    old_body: &str,
755    new_name: &str,
756    new_desc: &str,
757    new_body: &str,
758) -> Result<(), AppError> {
759    conn.execute(
760        "INSERT INTO fts_memories(fts_memories, rowid, name, description, body)
761         VALUES('delete', ?1, ?2, ?3, ?4)",
762        params![memory_id, old_name, old_desc, old_body],
763    )?;
764    conn.execute(
765        "INSERT INTO fts_memories(rowid, name, description, body)
766         VALUES(?1, ?2, ?3, ?4)",
767        params![memory_id, new_name, new_desc, new_body],
768    )?;
769    Ok(())
770}
771#[cfg(test)]
772mod tests;
sqlite_graphrag/storage/memories/mod.rs

sqlite_graphrag/storage/memories/
mod.rs
