Skip to main content

second_brain_core/sqlite_store/
chunks.rs

1use anyhow::Result;
2use rusqlite::{OptionalExtension, Transaction, params};
3use uuid::Uuid;
4
5use super::{SqliteStore, blob_to_embedding, cosine_blob, embedding_to_blob, parse_uuid};
6use crate::chunk;
7use crate::schema::MemoryType;
8
9/// A live memory whose stored chunk set is missing or no longer matches its
10/// content: the chunks to embed and the hash `store_chunks` must be handed
11/// back so a set split from since-changed content is refused.
12#[derive(Debug, Clone, PartialEq)]
13pub struct ChunkWork {
14    pub memory_id: Uuid,
15    pub content_hash: String,
16    pub chunks: Vec<String>,
17}
18
19#[derive(Debug, Clone, PartialEq)]
20pub struct MemoryChunk {
21    pub ord: usize,
22    pub text: String,
23    pub embedding: Vec<f32>,
24    pub content_hash: String,
25}
26
27impl SqliteStore {
28    /// Live memories that split into two or more chunks and whose stored set
29    /// is absent or was split from other content, oldest first. Splitting
30    /// happens here in Rust, so every candidate row is split on each call;
31    /// the length floor only skips rows too short to yield two chunks.
32    pub fn chunks_needing_embedding(&self, limit: usize) -> Result<Vec<ChunkWork>> {
33        Ok(self.scan_chunk_work(limit)?.0)
34    }
35
36    /// `chunks_needing_embedding`, plus the total number of memories needing
37    /// chunks. Counting past `limit` keeps scanning the whole table but
38    /// holds on to nothing beyond the first `limit`.
39    pub fn scan_chunk_work(&self, limit: usize) -> Result<(Vec<ChunkWork>, usize)> {
40        let conn = self.reader();
41        let mut stmt = conn.prepare(
42            "SELECT m.id, m.content, \
43             (SELECT c.content_hash FROM memory_chunks c WHERE c.memory_id = m.id LIMIT 1) \
44             FROM memories m \
45             WHERE m.tier = 'hot' AND m.superseded_by IS NULL AND length(m.content) >= ?1 \
46             ORDER BY m.rowid",
47        )?;
48        let mut rows = stmt.query(params![(2 * chunk::MIN_CHUNK_CHARS) as i64])?;
49        let mut out = Vec::new();
50        let mut pending = 0;
51        while let Some(row) = rows.next()? {
52            let content: String = row.get(1)?;
53            let stored: Option<String> = row.get(2)?;
54            let content_hash = chunk::content_hash(&content);
55            if stored.as_deref() == Some(content_hash.as_str()) {
56                continue;
57            }
58            let chunks = chunk::split(&content);
59            if chunks.is_empty() {
60                continue;
61            }
62            pending += 1;
63            if out.len() < limit {
64                out.push(ChunkWork {
65                    memory_id: parse_uuid(&row.get::<_, String>(0)?)?,
66                    content_hash,
67                    chunks,
68                });
69            }
70        }
71        Ok((out, pending))
72    }
73
74    /// (live memories holding a chunk set, memories still needing one as of
75    /// the last chunk backfill). The second is None before any backfill has
76    /// run, because counting it fresh means splitting every live row.
77    pub fn chunk_progress(&self) -> Result<(usize, Option<usize>)> {
78        let conn = self.reader();
79        let chunked: i64 = conn.query_row(
80            "SELECT COUNT(DISTINCT c.memory_id) FROM memory_chunks c \
81             JOIN memories m ON m.id = c.memory_id \
82             WHERE m.tier = 'hot' AND m.superseded_by IS NULL",
83            [],
84            |r| r.get(0),
85        )?;
86        let pending: Option<String> = conn
87            .query_row(
88                "SELECT value FROM index_meta WHERE key = 'chunks_pending'",
89                [],
90                |r| r.get(0),
91            )
92            .optional()?;
93        Ok((chunked as usize, pending.and_then(|v| v.parse().ok())))
94    }
95
96    pub(crate) fn set_chunks_pending(&self, pending: usize) -> Result<()> {
97        let conn = self.conn();
98        conn.execute(
99            "INSERT OR REPLACE INTO index_meta (key, value) VALUES ('chunks_pending', ?1)",
100            params![pending.to_string()],
101        )?;
102        Ok(())
103    }
104
105    /// Replaces the memory's whole chunk set and queues an index upsert for
106    /// it, in one transaction. Returns false, writing nothing, when the
107    /// memory is no longer live or its content no longer hashes to
108    /// `content_hash`. Chunks are derived locally on every peer, so nothing
109    /// here touches sync_log.
110    pub fn store_chunks(
111        &self,
112        memory_id: Uuid,
113        content_hash: &str,
114        chunks: &[(usize, String, Vec<f32>)],
115    ) -> Result<bool> {
116        let id = memory_id.to_string();
117        self.with_txn(|tx| {
118            let content: Option<String> = tx
119                .query_row(
120                    "SELECT content FROM memories \
121                     WHERE id = ?1 AND tier = 'hot' AND superseded_by IS NULL",
122                    params![id],
123                    |r| r.get(0),
124                )
125                .optional()?;
126            if content.is_none_or(|c| chunk::content_hash(&c) != content_hash) {
127                return Ok(false);
128            }
129            Self::replace_chunk_rows(
130                tx,
131                &id,
132                content_hash,
133                chunks
134                    .iter()
135                    .map(|(ord, text, embedding)| (*ord, text.as_str(), embedding.as_slice())),
136            )?;
137            Self::append_outbox(tx, memory_id, "upsert")?;
138            Ok(true)
139        })
140    }
141
142    /// Replaces the memory's chunk rows; an empty vector is stored as NULL.
143    pub(crate) fn replace_chunk_rows<'a>(
144        tx: &Transaction<'_>,
145        memory_id: &str,
146        content_hash: &str,
147        chunks: impl IntoIterator<Item = (usize, &'a str, &'a [f32])>,
148    ) -> Result<()> {
149        tx.execute(
150            "DELETE FROM memory_chunks WHERE memory_id = ?1",
151            params![memory_id],
152        )?;
153        let mut insert = tx.prepare(
154            "INSERT INTO memory_chunks (memory_id, ord, text, embedding, content_hash) \
155             VALUES (?1, ?2, ?3, ?4, ?5)",
156        )?;
157        for (ord, text, embedding) in chunks {
158            let blob = (!embedding.is_empty()).then(|| embedding_to_blob(embedding));
159            insert.execute(params![memory_id, ord as i64, text, blob, content_hash])?;
160        }
161        Ok(())
162    }
163
164    pub fn chunks_for(&self, memory_id: Uuid) -> Result<Vec<MemoryChunk>> {
165        let conn = self.reader();
166        let mut stmt = conn.prepare(
167            "SELECT ord, text, embedding, content_hash FROM memory_chunks \
168             WHERE memory_id = ?1 ORDER BY ord",
169        )?;
170        let rows = stmt.query_map(params![memory_id.to_string()], |r| {
171            Ok(MemoryChunk {
172                ord: r.get::<_, i64>(0)? as usize,
173                text: r.get(1)?,
174                embedding: r
175                    .get::<_, Option<Vec<u8>>>(2)?
176                    .map(|b| blob_to_embedding(&b))
177                    .unwrap_or_default(),
178                content_hash: r.get(3)?,
179            })
180        })?;
181        Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
182    }
183
184    pub fn delete_chunks(&self, memory_id: Uuid) -> Result<()> {
185        let conn = self.conn();
186        conn.execute(
187            "DELETE FROM memory_chunks WHERE memory_id = ?1",
188            params![memory_id.to_string()],
189        )?;
190        Ok(())
191    }
192
193    /// Stable u64 key in `chunks.usearch` for one chunk, allocated on first
194    /// use, mirroring `vec_key`.
195    pub fn chunk_vec_key(&self, memory_id: Uuid, ord: usize) -> Result<u64> {
196        let conn = self.conn();
197        let id = memory_id.to_string();
198        conn.execute(
199            "INSERT OR IGNORE INTO chunk_vec_keys (memory_id, ord) VALUES (?1, ?2)",
200            params![id, ord as i64],
201        )?;
202        let key: i64 = conn.query_row(
203            "SELECT key FROM chunk_vec_keys WHERE memory_id = ?1 AND ord = ?2",
204            params![id, ord as i64],
205            |r| r.get(0),
206        )?;
207        Ok(key as u64)
208    }
209
210    /// (ord, key) pairs allocated for the memory, by ord.
211    pub fn chunk_vec_keys_for(&self, memory_id: Uuid) -> Result<Vec<(usize, u64)>> {
212        let conn = self.reader();
213        let mut stmt =
214            conn.prepare("SELECT ord, key FROM chunk_vec_keys WHERE memory_id = ?1 ORDER BY ord")?;
215        let rows = stmt.query_map(params![memory_id.to_string()], |r| {
216            Ok((r.get::<_, i64>(0)? as usize, r.get::<_, i64>(1)? as u64))
217        })?;
218        Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
219    }
220
221    /// Drops the key row once its vector is gone from `chunks.usearch`, so
222    /// `chunk_vec_keys` only holds keys the index can still resolve.
223    pub fn delete_chunk_vec_key(&self, memory_id: Uuid, ord: usize) -> Result<()> {
224        let conn = self.conn();
225        conn.execute(
226            "DELETE FROM chunk_vec_keys WHERE memory_id = ?1 AND ord = ?2",
227            params![memory_id.to_string(), ord as i64],
228        )?;
229        Ok(())
230    }
231
232    pub fn chunk_for_vec_key(&self, key: u64) -> Result<Option<(Uuid, usize)>> {
233        let conn = self.reader();
234        let hit: Option<(String, i64)> = conn
235            .query_row(
236                "SELECT memory_id, ord FROM chunk_vec_keys WHERE key = ?1",
237                params![key as i64],
238                |r| Ok((r.get(0)?, r.get(1)?)),
239            )
240            .optional()?;
241        hit.map(|(id, ord)| Ok((Uuid::parse_str(&id)?, ord as usize)))
242            .transpose()
243    }
244
245    /// Chunk rows of live memories whose embedding has exactly `dim`
246    /// components: what `chunks.usearch` should hold once drained.
247    pub fn embedded_chunk_count(&self, dim: usize) -> Result<usize> {
248        let conn = self.reader();
249        let count: i64 = conn.query_row(
250            "SELECT COUNT(*) FROM memory_chunks c JOIN memories m ON m.id = c.memory_id \
251             WHERE m.tier = 'hot' AND m.superseded_by IS NULL \
252             AND c.embedding IS NOT NULL AND length(c.embedding) = ?1",
253            params![(dim * 4) as i64],
254            |r| r.get(0),
255        )?;
256        Ok(count as usize)
257    }
258
259    /// Live memories holding a chunk set, each with the hash of its current
260    /// content, which the set must match to still be indexed.
261    pub fn live_chunked_memories(&self) -> Result<Vec<(Uuid, String)>> {
262        let conn = self.reader();
263        let mut stmt = conn.prepare(
264            "SELECT m.id, m.content FROM memories m \
265             WHERE m.tier = 'hot' AND m.superseded_by IS NULL \
266             AND EXISTS (SELECT 1 FROM memory_chunks c WHERE c.memory_id = m.id)",
267        )?;
268        let rows = stmt.query_map([], |r| {
269            let id = parse_uuid(&r.get::<_, String>(0)?)?;
270            let content: String = r.get(1)?;
271            Ok((id, chunk::content_hash(&content)))
272        })?;
273        Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
274    }
275
276    /// Deletes chunk rows and chunk keys of every memory outside the live
277    /// set. Only a full rebuild calls this, because it resets the chunk
278    /// index first and so has no vectors to remove per key.
279    pub fn purge_chunks_outside_live_set(&self) -> Result<()> {
280        const LIVE: &str = "SELECT id FROM memories WHERE tier = 'hot' AND superseded_by IS NULL";
281        self.with_txn(|tx| {
282            tx.execute(
283                &format!("DELETE FROM memory_chunks WHERE memory_id NOT IN ({LIVE})"),
284                [],
285            )?;
286            tx.execute(
287                &format!("DELETE FROM chunk_vec_keys WHERE memory_id NOT IN ({LIVE})"),
288                [],
289            )?;
290            Ok(())
291        })
292    }
293
294    pub(crate) fn chunk_window_text(&self, memory_id: Uuid, ord: usize) -> Result<Option<String>> {
295        let conn = self.reader();
296        let mut stmt = conn.prepare(
297            "SELECT text FROM memory_chunks WHERE memory_id = ?1 AND ord BETWEEN ?2 AND ?3 \
298             ORDER BY ord",
299        )?;
300        let texts = stmt
301            .query_map(
302                params![
303                    memory_id.to_string(),
304                    ord.saturating_sub(1) as i64,
305                    ord as i64 + 1
306                ],
307                |r| r.get::<_, String>(0),
308            )?
309            .collect::<rusqlite::Result<Vec<_>>>()?;
310        Ok((!texts.is_empty()).then(|| texts.join(" ")))
311    }
312
313    /// The scoped counterpart of `Indexes::chunk_search`: a brute-force
314    /// cosine scan of the embedded chunks of live in-scope memories,
315    /// returning each memory's best (memory id, cosine, ord) for the top
316    /// `limit` memories, best first. Rows arrive grouped by memory (the
317    /// primary key's order), so only the running best of one memory and the
318    /// top `limit` are ever held, however wide the scope.
319    pub fn filtered_chunk_search(
320        &self,
321        embedding: &[f32],
322        source: Option<&str>,
323        memory_type: Option<MemoryType>,
324        project_path: Option<&str>,
325        limit: usize,
326    ) -> Result<Vec<(Uuid, f32, usize)>> {
327        if limit == 0 {
328            return Ok(Vec::new());
329        }
330        let (predicate, args) = Self::scope_predicate(source, memory_type, project_path);
331        let conn = self.reader();
332        let mut stmt = conn.prepare(&format!(
333            "SELECT memory_id, ord, embedding FROM memory_chunks \
334             WHERE embedding IS NOT NULL AND memory_id IN ( \
335               SELECT id FROM memories \
336               WHERE tier = 'hot' AND superseded_by IS NULL{predicate}) \
337             ORDER BY memory_id, ord"
338        ))?;
339        let bound: Vec<&dyn rusqlite::ToSql> =
340            args.iter().map(|a| a as &dyn rusqlite::ToSql).collect();
341        let query_norm = embedding.iter().map(|x| x * x).sum::<f32>().sqrt();
342        let mut rows = stmt.query(bound.as_slice())?;
343        let mut top: Vec<(f32, String, usize)> = Vec::with_capacity(limit + 1);
344        let mut current: Option<(f32, String, usize)> = None;
345        let place = |best: (f32, String, usize), top: &mut Vec<(f32, String, usize)>| {
346            let at = top.partition_point(|(s, _, _)| *s >= best.0);
347            if at < limit {
348                top.insert(at, best);
349                top.truncate(limit);
350            }
351        };
352        while let Some(row) = rows.next()? {
353            let Some(blob) = row.get_ref(2)?.as_blob_or_null()? else {
354                continue;
355            };
356            let similarity = cosine_blob(embedding, query_norm, blob);
357            let memory_id = row.get_ref(0)?.as_str()?;
358            let ord = row.get::<_, i64>(1)? as usize;
359            match &mut current {
360                Some(best) if best.1 == memory_id => {
361                    if similarity > best.0 {
362                        best.0 = similarity;
363                        best.2 = ord;
364                    }
365                }
366                _ => {
367                    if let Some(done) = current.take() {
368                        place(done, &mut top);
369                    }
370                    current = Some((similarity, memory_id.to_string(), ord));
371                }
372            }
373        }
374        if let Some(done) = current.take() {
375            place(done, &mut top);
376        }
377        top.into_iter()
378            .map(|(similarity, id, ord)| Ok((Uuid::parse_str(&id)?, similarity, ord)))
379            .collect()
380    }
381}