Skip to main content

second_brain_core/sqlite_store/
chunks.rs

1use anyhow::Result;
2use rusqlite::{OptionalExtension, params};
3use uuid::Uuid;
4
5use super::{SqliteStore, blob_to_embedding, cosine_blob, embedding_to_blob, parse_uuid};
6use crate::chunk;
7use crate::schema::MemoryType;
8
9/// A live memory whose stored chunk set is missing or no longer matches its
10/// content: the chunks to embed and the hash `store_chunks` must be handed
11/// back so a set split from since-changed content is refused.
12#[derive(Debug, Clone, PartialEq)]
13pub struct ChunkWork {
14    pub memory_id: Uuid,
15    pub content_hash: String,
16    pub chunks: Vec<String>,
17}
18
19#[derive(Debug, Clone, PartialEq)]
20pub struct MemoryChunk {
21    pub ord: usize,
22    pub text: String,
23    pub embedding: Vec<f32>,
24    pub content_hash: String,
25}
26
27impl SqliteStore {
28    /// Live memories that split into two or more chunks and whose stored set
29    /// is absent or was split from other content, oldest first. Splitting
30    /// happens here in Rust, so every candidate row is split on each call;
31    /// the length floor only skips rows too short to yield two chunks.
32    pub fn chunks_needing_embedding(&self, limit: usize) -> Result<Vec<ChunkWork>> {
33        Ok(self.scan_chunk_work(limit)?.0)
34    }
35
36    /// `chunks_needing_embedding`, plus the total number of memories needing
37    /// chunks. Counting past `limit` keeps scanning the whole table but
38    /// holds on to nothing beyond the first `limit`.
39    pub fn scan_chunk_work(&self, limit: usize) -> Result<(Vec<ChunkWork>, usize)> {
40        let conn = self.reader();
41        let mut stmt = conn.prepare(
42            "SELECT m.id, m.content, \
43             (SELECT c.content_hash FROM memory_chunks c WHERE c.memory_id = m.id LIMIT 1) \
44             FROM memories m \
45             WHERE m.tier = 'hot' AND m.superseded_by IS NULL AND length(m.content) >= ?1 \
46             ORDER BY m.rowid",
47        )?;
48        let mut rows = stmt.query(params![(2 * chunk::MIN_CHUNK_CHARS) as i64])?;
49        let mut out = Vec::new();
50        let mut pending = 0;
51        while let Some(row) = rows.next()? {
52            let content: String = row.get(1)?;
53            let stored: Option<String> = row.get(2)?;
54            let content_hash = chunk::content_hash(&content);
55            if stored.as_deref() == Some(content_hash.as_str()) {
56                continue;
57            }
58            let chunks = chunk::split(&content);
59            if chunks.is_empty() {
60                continue;
61            }
62            pending += 1;
63            if out.len() < limit {
64                out.push(ChunkWork {
65                    memory_id: parse_uuid(&row.get::<_, String>(0)?)?,
66                    content_hash,
67                    chunks,
68                });
69            }
70        }
71        Ok((out, pending))
72    }
73
74    /// (live memories holding a chunk set, memories still needing one as of
75    /// the last chunk backfill). The second is None before any backfill has
76    /// run, because counting it fresh means splitting every live row.
77    pub fn chunk_progress(&self) -> Result<(usize, Option<usize>)> {
78        let conn = self.reader();
79        let chunked: i64 = conn.query_row(
80            "SELECT COUNT(DISTINCT c.memory_id) FROM memory_chunks c \
81             JOIN memories m ON m.id = c.memory_id \
82             WHERE m.tier = 'hot' AND m.superseded_by IS NULL",
83            [],
84            |r| r.get(0),
85        )?;
86        let pending: Option<String> = conn
87            .query_row(
88                "SELECT value FROM index_meta WHERE key = 'chunks_pending'",
89                [],
90                |r| r.get(0),
91            )
92            .optional()?;
93        Ok((chunked as usize, pending.and_then(|v| v.parse().ok())))
94    }
95
96    pub(crate) fn set_chunks_pending(&self, pending: usize) -> Result<()> {
97        let conn = self.conn();
98        conn.execute(
99            "INSERT OR REPLACE INTO index_meta (key, value) VALUES ('chunks_pending', ?1)",
100            params![pending.to_string()],
101        )?;
102        Ok(())
103    }
104
105    /// Replaces the memory's whole chunk set and queues an index upsert for
106    /// it, in one transaction. Returns false, writing nothing, when the
107    /// memory is no longer live or its content no longer hashes to
108    /// `content_hash`. Chunks are derived locally on every peer, so nothing
109    /// here touches sync_log.
110    pub fn store_chunks(
111        &self,
112        memory_id: Uuid,
113        content_hash: &str,
114        chunks: &[(usize, String, Vec<f32>)],
115    ) -> Result<bool> {
116        let id = memory_id.to_string();
117        self.with_txn(|tx| {
118            let content: Option<String> = tx
119                .query_row(
120                    "SELECT content FROM memories \
121                     WHERE id = ?1 AND tier = 'hot' AND superseded_by IS NULL",
122                    params![id],
123                    |r| r.get(0),
124                )
125                .optional()?;
126            if content.is_none_or(|c| chunk::content_hash(&c) != content_hash) {
127                return Ok(false);
128            }
129            tx.execute(
130                "DELETE FROM memory_chunks WHERE memory_id = ?1",
131                params![id],
132            )?;
133            let mut insert = tx.prepare(
134                "INSERT INTO memory_chunks (memory_id, ord, text, embedding, content_hash) \
135                 VALUES (?1, ?2, ?3, ?4, ?5)",
136            )?;
137            for (ord, text, embedding) in chunks {
138                let blob = (!embedding.is_empty()).then(|| embedding_to_blob(embedding));
139                insert.execute(params![id, *ord as i64, text, blob, content_hash])?;
140            }
141            Self::append_outbox(tx, memory_id, "upsert")?;
142            Ok(true)
143        })
144    }
145
146    pub fn chunks_for(&self, memory_id: Uuid) -> Result<Vec<MemoryChunk>> {
147        let conn = self.reader();
148        let mut stmt = conn.prepare(
149            "SELECT ord, text, embedding, content_hash FROM memory_chunks \
150             WHERE memory_id = ?1 ORDER BY ord",
151        )?;
152        let rows = stmt.query_map(params![memory_id.to_string()], |r| {
153            Ok(MemoryChunk {
154                ord: r.get::<_, i64>(0)? as usize,
155                text: r.get(1)?,
156                embedding: r
157                    .get::<_, Option<Vec<u8>>>(2)?
158                    .map(|b| blob_to_embedding(&b))
159                    .unwrap_or_default(),
160                content_hash: r.get(3)?,
161            })
162        })?;
163        Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
164    }
165
166    pub fn delete_chunks(&self, memory_id: Uuid) -> Result<()> {
167        let conn = self.conn();
168        conn.execute(
169            "DELETE FROM memory_chunks WHERE memory_id = ?1",
170            params![memory_id.to_string()],
171        )?;
172        Ok(())
173    }
174
175    /// Stable u64 key in `chunks.usearch` for one chunk, allocated on first
176    /// use, mirroring `vec_key`.
177    pub fn chunk_vec_key(&self, memory_id: Uuid, ord: usize) -> Result<u64> {
178        let conn = self.conn();
179        let id = memory_id.to_string();
180        conn.execute(
181            "INSERT OR IGNORE INTO chunk_vec_keys (memory_id, ord) VALUES (?1, ?2)",
182            params![id, ord as i64],
183        )?;
184        let key: i64 = conn.query_row(
185            "SELECT key FROM chunk_vec_keys WHERE memory_id = ?1 AND ord = ?2",
186            params![id, ord as i64],
187            |r| r.get(0),
188        )?;
189        Ok(key as u64)
190    }
191
192    /// (ord, key) pairs allocated for the memory, by ord.
193    pub fn chunk_vec_keys_for(&self, memory_id: Uuid) -> Result<Vec<(usize, u64)>> {
194        let conn = self.reader();
195        let mut stmt =
196            conn.prepare("SELECT ord, key FROM chunk_vec_keys WHERE memory_id = ?1 ORDER BY ord")?;
197        let rows = stmt.query_map(params![memory_id.to_string()], |r| {
198            Ok((r.get::<_, i64>(0)? as usize, r.get::<_, i64>(1)? as u64))
199        })?;
200        Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
201    }
202
203    /// Drops the key row once its vector is gone from `chunks.usearch`, so
204    /// `chunk_vec_keys` only holds keys the index can still resolve.
205    pub fn delete_chunk_vec_key(&self, memory_id: Uuid, ord: usize) -> Result<()> {
206        let conn = self.conn();
207        conn.execute(
208            "DELETE FROM chunk_vec_keys WHERE memory_id = ?1 AND ord = ?2",
209            params![memory_id.to_string(), ord as i64],
210        )?;
211        Ok(())
212    }
213
214    pub fn chunk_for_vec_key(&self, key: u64) -> Result<Option<(Uuid, usize)>> {
215        let conn = self.reader();
216        let hit: Option<(String, i64)> = conn
217            .query_row(
218                "SELECT memory_id, ord FROM chunk_vec_keys WHERE key = ?1",
219                params![key as i64],
220                |r| Ok((r.get(0)?, r.get(1)?)),
221            )
222            .optional()?;
223        hit.map(|(id, ord)| Ok((Uuid::parse_str(&id)?, ord as usize)))
224            .transpose()
225    }
226
227    /// Chunk rows of live memories whose embedding has exactly `dim`
228    /// components: what `chunks.usearch` should hold once drained.
229    pub fn embedded_chunk_count(&self, dim: usize) -> Result<usize> {
230        let conn = self.reader();
231        let count: i64 = conn.query_row(
232            "SELECT COUNT(*) FROM memory_chunks c JOIN memories m ON m.id = c.memory_id \
233             WHERE m.tier = 'hot' AND m.superseded_by IS NULL \
234             AND c.embedding IS NOT NULL AND length(c.embedding) = ?1",
235            params![(dim * 4) as i64],
236            |r| r.get(0),
237        )?;
238        Ok(count as usize)
239    }
240
241    /// Live memories holding a chunk set, each with the hash of its current
242    /// content, which the set must match to still be indexed.
243    pub fn live_chunked_memories(&self) -> Result<Vec<(Uuid, String)>> {
244        let conn = self.reader();
245        let mut stmt = conn.prepare(
246            "SELECT m.id, m.content FROM memories m \
247             WHERE m.tier = 'hot' AND m.superseded_by IS NULL \
248             AND EXISTS (SELECT 1 FROM memory_chunks c WHERE c.memory_id = m.id)",
249        )?;
250        let rows = stmt.query_map([], |r| {
251            let id = parse_uuid(&r.get::<_, String>(0)?)?;
252            let content: String = r.get(1)?;
253            Ok((id, chunk::content_hash(&content)))
254        })?;
255        Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
256    }
257
258    /// Deletes chunk rows and chunk keys of every memory outside the live
259    /// set. Only a full rebuild calls this, because it resets the chunk
260    /// index first and so has no vectors to remove per key.
261    pub fn purge_chunks_outside_live_set(&self) -> Result<()> {
262        const LIVE: &str = "SELECT id FROM memories WHERE tier = 'hot' AND superseded_by IS NULL";
263        self.with_txn(|tx| {
264            tx.execute(
265                &format!("DELETE FROM memory_chunks WHERE memory_id NOT IN ({LIVE})"),
266                [],
267            )?;
268            tx.execute(
269                &format!("DELETE FROM chunk_vec_keys WHERE memory_id NOT IN ({LIVE})"),
270                [],
271            )?;
272            Ok(())
273        })
274    }
275
276    pub(crate) fn chunk_window_text(&self, memory_id: Uuid, ord: usize) -> Result<Option<String>> {
277        let conn = self.reader();
278        let mut stmt = conn.prepare(
279            "SELECT text FROM memory_chunks WHERE memory_id = ?1 AND ord BETWEEN ?2 AND ?3 \
280             ORDER BY ord",
281        )?;
282        let texts = stmt
283            .query_map(
284                params![
285                    memory_id.to_string(),
286                    ord.saturating_sub(1) as i64,
287                    ord as i64 + 1
288                ],
289                |r| r.get::<_, String>(0),
290            )?
291            .collect::<rusqlite::Result<Vec<_>>>()?;
292        Ok((!texts.is_empty()).then(|| texts.join(" ")))
293    }
294
295    /// The scoped counterpart of `Indexes::chunk_search`: a brute-force
296    /// cosine scan of the embedded chunks of live in-scope memories,
297    /// returning each memory's best (memory id, cosine, ord) for the top
298    /// `limit` memories, best first. Rows arrive grouped by memory (the
299    /// primary key's order), so only the running best of one memory and the
300    /// top `limit` are ever held, however wide the scope.
301    pub fn filtered_chunk_search(
302        &self,
303        embedding: &[f32],
304        source: Option<&str>,
305        memory_type: Option<MemoryType>,
306        project_path: Option<&str>,
307        limit: usize,
308    ) -> Result<Vec<(Uuid, f32, usize)>> {
309        if limit == 0 {
310            return Ok(Vec::new());
311        }
312        let (predicate, args) = Self::scope_predicate(source, memory_type, project_path);
313        let conn = self.reader();
314        let mut stmt = conn.prepare(&format!(
315            "SELECT memory_id, ord, embedding FROM memory_chunks \
316             WHERE embedding IS NOT NULL AND memory_id IN ( \
317               SELECT id FROM memories \
318               WHERE tier = 'hot' AND superseded_by IS NULL{predicate}) \
319             ORDER BY memory_id, ord"
320        ))?;
321        let bound: Vec<&dyn rusqlite::ToSql> =
322            args.iter().map(|a| a as &dyn rusqlite::ToSql).collect();
323        let query_norm = embedding.iter().map(|x| x * x).sum::<f32>().sqrt();
324        let mut rows = stmt.query(bound.as_slice())?;
325        let mut top: Vec<(f32, String, usize)> = Vec::with_capacity(limit + 1);
326        let mut current: Option<(f32, String, usize)> = None;
327        let place = |best: (f32, String, usize), top: &mut Vec<(f32, String, usize)>| {
328            let at = top.partition_point(|(s, _, _)| *s >= best.0);
329            if at < limit {
330                top.insert(at, best);
331                top.truncate(limit);
332            }
333        };
334        while let Some(row) = rows.next()? {
335            let Some(blob) = row.get_ref(2)?.as_blob_or_null()? else {
336                continue;
337            };
338            let similarity = cosine_blob(embedding, query_norm, blob);
339            let memory_id = row.get_ref(0)?.as_str()?;
340            let ord = row.get::<_, i64>(1)? as usize;
341            match &mut current {
342                Some(best) if best.1 == memory_id => {
343                    if similarity > best.0 {
344                        best.0 = similarity;
345                        best.2 = ord;
346                    }
347                }
348                _ => {
349                    if let Some(done) = current.take() {
350                        place(done, &mut top);
351                    }
352                    current = Some((similarity, memory_id.to_string(), ord));
353                }
354            }
355        }
356        if let Some(done) = current.take() {
357            place(done, &mut top);
358        }
359        top.into_iter()
360            .map(|(similarity, id, ord)| Ok((Uuid::parse_str(&id)?, similarity, ord)))
361            .collect()
362    }
363}